From 3cd4801c71c0e13aac9eabbec1bd47996f0303b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=B0=E5=B0=BC=E9=BE=9F?= <3098584968@qq.com> Date: Mon, 13 Jul 2026 19:56:12 +0800 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20Session/Memory=20=E5=A4=9A=E5=90=8E?= =?UTF-8?q?=E7=AB=AF=E5=9B=9E=E6=94=BE=E4=B8=80=E8=87=B4=E6=80=A7=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=E6=A1=86=E6=9E=B6=20(issue=20#89)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - replay harness: 同一组轨迹驱动 InMemory/SQLite/Redis,比较事件/state/memory/summary - 归一化(占位符,保留字段存在性)+ JSONPath 精确 allowed_diff + 覆盖率治理 - summary: SDK 确定性模型 + 三分比较 + loss/overwrite/affiliation 检测 - 检出验证: 快照层注入 + 端到端后端注入(改 SQL 行 / Redis key 重读) - 实测发现 SQLite summary 持久化 drift,标 KNOWN_DRIFT 只报告不改 - 29 tests pass, 1 skipped (Redis 需 TRPC_REPLAY_REDIS_URL) --- .gitignore | 3 + ...07-13-session-memory-replay-consistency.md | 258 +++++++++ ...ession-memory-replay-consistency-design.md | 414 ++++++++++++++ session_memory_summary_diff_report.json | 539 ++++++++++++++++++ tests/sessions/replay/__init__.py | 33 ++ tests/sessions/replay/allowed_diff.py | 80 +++ tests/sessions/replay/backends.py | 109 ++++ tests/sessions/replay/comparator.py | 119 ++++ tests/sessions/replay/harness.py | 258 +++++++++ tests/sessions/replay/injectors.py | 124 ++++ tests/sessions/replay/normalizer.py | 61 ++ .../sessions/replay/replay_cases/cases.jsonl | 10 + tests/sessions/replay/report.py | 122 ++++ tests/sessions/replay/summary_checks.py | 98 ++++ tests/sessions/test_replay_consistency.py | 156 +++++ tests/sessions/test_replay_injections.py | 200 +++++++ tests/sessions/test_replay_unit.py | 309 ++++++++++ 17 files changed, 2893 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-13-session-memory-replay-consistency.md create mode 100644 docs/superpowers/specs/2026-07-13-session-memory-replay-consistency-design.md create mode 100644 session_memory_summary_diff_report.json create mode 100644 tests/sessions/replay/__init__.py create mode 100644 tests/sessions/replay/allowed_diff.py create mode 100644 tests/sessions/replay/backends.py create mode 100644 tests/sessions/replay/comparator.py create mode 100644 tests/sessions/replay/harness.py create mode 100644 tests/sessions/replay/injectors.py create mode 100644 tests/sessions/replay/normalizer.py create mode 100644 tests/sessions/replay/replay_cases/cases.jsonl create mode 100644 tests/sessions/replay/report.py create mode 100644 tests/sessions/replay/summary_checks.py create mode 100644 tests/sessions/test_replay_consistency.py create mode 100644 tests/sessions/test_replay_injections.py create mode 100644 tests/sessions/test_replay_unit.py diff --git a/.gitignore b/.gitignore index 233248dd..58eb6b48 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,6 @@ test-ngtest-ut-trpc-agent-py.xml node_modules package-lock.json pyrightconfig.json + +# spec-workflow tool artifacts +.spec-workflow diff --git a/docs/superpowers/plans/2026-07-13-session-memory-replay-consistency.md b/docs/superpowers/plans/2026-07-13-session-memory-replay-consistency.md new file mode 100644 index 00000000..25c9a1d8 --- /dev/null +++ b/docs/superpowers/plans/2026-07-13-session-memory-replay-consistency.md @@ -0,0 +1,258 @@ +# Session/Memory/Summary 回放一致性框架 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: 用 superpowers:executing-plans 逐 task 执行。步骤用 `- [ ]` 跟踪。 + +**Goal:** 实现一个后端中立的 replay harness,用 10 条标准化轨迹驱动 InMemory/SQLite/Redis,产出可定位的差异报告,并通过快照层 + 端到端后端注入验证检出率/误报率。 + +**Architecture:** 四段管线 `load JSONL → replay_case → 后端中立快照 → compare → report`。详见 [设计文档](../specs/2026-07-13-session-memory-replay-consistency-design.md)。 + +**Tech Stack:** Python 3.10+、pytest、pydantic v2、SQLAlchemy(SQLite)、redis-py(mock)。 + +## Global Constraints + +- **PR 干净**:只碰 `tests/` + 本计划/设计文档 + 仓库根 `session_memory_summary_diff_report.json`(运行产物)。**不改 `trpc_agent_sdk/` 生产代码**;发现的 SDK bug 只在报告/文档记录。 +- **CI lint**:提交前本地 `PYTHONUTF8=1` 跑 `yapf -ri` + `flake8`([[ci-lint-yapf-flake8]])。 +- **Windows**:`python-magic` 用 `python-magic-bin`([[python-magic-windows-cygwin-crash]])。 +- **提交纪律**:`git add` 只加本计划列出的确切路径,禁用 `-A`/`.`([[subagent-git-add-scope]])。用户未要求不主动 commit/push。 +- **轻量模式 ≤30s**:默认 InMemory vs SQLite(`:memory:`);Redis/MySQL 经 env 启用,不可用 `pytest.skip`。 +- **确定性**:无 LLM、无真实网络;summary 用 `_DeterministicSummarizer`;时间用占位符归一化。 + +## File Structure + +(完整职责见设计文档 §3.2) + +| 文件 | 职责 | +|---|---| +| `tests/sessions/replay/__init__.py` | 包入口 + 150–300 字设计说明 | +| `tests/sessions/replay/harness.py` | 数据模型(ReplayOp/ReplayCase/ReplayBackend/ReplaySnapshot)+ `replay_case()` | +| `tests/sessions/replay/normalizer.py` | 占位符归一化 | +| `tests/sessions/replay/comparator.py` | 递归 `visit()` + DiffEntry | +| `tests/sessions/replay/allowed_diff.py` | JSONPath 匹配 + 覆盖率治理 | +| `tests/sessions/replay/summary_checks.py` | 三类专项 + 分词 Jaccard 语义比较 | +| `tests/sessions/replay/injectors.py` | 快照层 + 端到端后端注入 | +| `tests/sessions/replay/report.py` | schema_version=3 报告 | +| `tests/sessions/replay/backends.py` | 三后端实例化 + env 门控 | +| `tests/sessions/replay/replay_cases/*.jsonl` | 10 条 case | +| `tests/sessions/test_replay_consistency.py` | 主 E2E | +| `tests/sessions/test_replay_injections.py` | 两层注入检出 | +| `tests/sessions/test_replay_unit.py` | normalizer/comparator/allowed_diff/summary_checks 单测 | + +--- + +## Task 1: 包骨架 + 设计说明 + +**Files:** +- Create: `tests/sessions/replay/__init__.py` + +**Interfaces:** Produces `tests.sessions.replay` 包(后续模块的根)。 + +- [ ] **Step 1:** 创建 `tests/sessions/replay/__init__.py`,内含设计文档 §10 的 150–300 字设计说明作为模块 docstring,加 `__all__ = []`。 +- [ ] **Step 2:** `PYTHONUTF8=1 python -c "import tests.sessions.replay"` 验证可导入。 + +--- + +## Task 2: harness 数据模型(纯 Pydantic,无逻辑) + +**Files:** +- Create: `tests/sessions/replay/harness.py` + +**Interfaces:** +- Consumes: `trpc_agent_sdk.sessions.SessionServiceABC`、`trpc_agent_sdk.memory.MemoryServiceABC` +- Produces: `ReplayOp`、`AllowedDiffRule`、`ReplayCase`、`ReplayBackend`、`ReplaySnapshot`(签名见设计文档 §4.1) +- **关键决策**:`ReplayOp` 用 `type: Literal[...]` + `model_dump()` 兼容异构 payload(不用 discriminated union 的复杂标签,保持 jsonl 可读)。 + +- [ ] **Step 1:** 写 `test_replay_unit.py::test_replay_case_roundtrip`:构造 `ReplayCase(case_id="x", operations=[ReplayOp(op="create_session", app_name="a", user_id="u", session_id="s")])`,`model_dump_json()` 后 `model_validate_json()` 应相等。 +- [ ] **Step 2:** 跑测试,确认 FAIL(模块未建)。 +- [ ] **Step 3:** 在 `harness.py` 实现五个 Pydantic 模型(签名照设计文档 §4.1,`ReplayBackend` 用 `BaseModel` + `model_config = ConfigDict(arbitrary_types_allowed=True)` 以容纳 service 实例)。 +- [ ] **Step 4:** 跑测试,确认 PASS。 + +--- + +## Task 3: normalizer(占位符归一化) + +**Files:** +- Create: `tests/sessions/replay/normalizer.py` +- Test: `tests/sessions/test_replay_unit.py` + +**Interfaces:** +- Produces: `normalize_event(e: dict) -> dict`、`normalize_snapshot(s: ReplaySnapshot) -> ReplaySnapshot`、常量 `NORMALIZED = ""` + +- [ ] **Step 1:** 写测试: + - `test_normalize_event_replaces_volatile_fields`:event 含 `id/timestamp/invocation_id` → 归一化后这三键值为 `NORMALIZED`,**键仍存在**。 + - `test_normalize_strips_temp_state`:state 含 `temp:k` → 归一化后删除该键,保留 `app:`/`user:`/普通键。 + - `test_normalize_sorts_memory`:memory 两个查询结果顺序乱 → 归一化后按 `json.dumps(sort_keys=True)` 确定性排序。 +- [ ] **Step 2:** 跑测试,确认 FAIL。 +- [ ] **Step 3:** 实现:递归把 `id`/`timestamp`/`invocation_id` 顶层键值替换为 `NORMALIZED`;剥离 `state` 中 `temp:` 前缀键;memory 各 list 按 `json.dumps(i, sort_keys=True, ensure_ascii=True)` 排序。 +- [ ] **Step 4:** 跑测试,确认 PASS。 + +--- + +## Task 4: comparator(递归比较 + 内联定位) + +**Files:** +- Create: `tests/sessions/replay/comparator.py` +- Test: `tests/sessions/test_replay_unit.py` + +**Interfaces:** +- Consumes: `ReplaySnapshot`、`list[AllowedDiffRule]`(from Task 2)、`is_allowed`(from Task 5,先用 stub) +- Produces: `DiffEntry`、`compare_snapshots(reference, candidate, *, reference_backend, candidate_backend, allowed_diff) -> list[DiffEntry]` +- **关键决策**:DiffEntry 在递归到叶子时**内联写入** `session_id`(snapshot 顶层)、`event_index`(events 列表的下标)、`summary_id`(若 path 在 summary 下)。 + +- [ ] **Step 1:** 写测试: + - `test_diff_detects_leaf_mismatch`:`events[0].author` 左 `"user"` 右 `"assistant"` → 产 1 条 DiffEntry,`field_path="events[0].author"`、`event_index=0`、`allowed=False`。 + - `test_diff_aligns_dict_sorted_keys`:左 `{"b":1,"a":2}` 右 `{"a":2,"b":1}` → 无 diff。 + - `test_diff_list_length_diff`:左 events 3 条右 2 条 → 产 1 条 `` diff,`event_index=2`。 +- [ ] **Step 2:** 跑测试,确认 FAIL。 +- [ ] **Step 3:** 实现 `visit(left, right, path)` 递归(dict sorted keys / list 下标+补 `` / 叶子 `!=` 产 DiffEntry),顶层包装填 `session_id`、按 path 前缀推断 `event_index`/`summary_id`;`allowed_diff` 参数暂传 `[]`。 +- [ ] **Step 4:** 跑测试,确认 PASS。 + +--- + +## Task 5: allowed_diff(JSONPath 精确 + 治理) + +**Files:** +- Create: `tests/sessions/replay/allowed_diff.py` +- Test: `tests/sessions/test_replay_unit.py` + `tests/sessions/test_allowed_diff_governance.py` + +**Interfaces:** +- Produces: `is_allowed(field_path, backend_pair, rules) -> tuple[bool, str|None]`、`check_governance(case, total_fields, used_allowed) -> None`、常量 `MAX_ALLOWED_PER_CASE=8`、`MAX_ALLOWED_RATIO=0.10` +- **接线**:回 `comparator.compare_snapshots` 用真实 `is_allowed` 替换 Task 4 stub。 + +- [ ] **Step 1:** 写测试: + - `test_allowed_exact_path_match`:规则 `path="events[0].timestamp"` → 字段 `"events[0].timestamp"` 匹配 allowed,`"events[0].author"` 不匹配。防 `*.id` 过宽。 + - `test_allowed_index_wildcard`:规则 `"events[*].timestamp"` → 匹配 `events[0].timestamp`/`events[5].timestamp`。 + - `test_governance_rejects_too_many`:`check_governance(case, total_fields=20, used_allowed=10)`(超 `MAX_ALLOWED_RATIO=0.10`)→ 抛 `ValueError`。 + - `test_governance_rejects_no_reason`:`AllowedDiffRule(path="x", reason="")` → 构造时或 governance 校验抛错。 +- [ ] **Step 2:** 跑测试,确认 FAIL。 +- [ ] **Step 3:** 实现 `is_allowed`:`[N]→[*]` 归一化后 `fnmatch.fnmatchcase`,reason 空即拒;`check_governance`:条数 > `MAX_ALLOWED_PER_CASE` 或 `used/total > MAX_ALLOWED_RATIO` 抛错。 +- [ ] **Step 4:** 跑测试 + 回归 Task 4 测试,确认 PASS。 + +--- + +## Task 6: summary_checks(三类专项 + 语义比较) + +**Files:** +- Create: `tests/sessions/replay/summary_checks.py` +- Test: `tests/sessions/test_replay_unit.py` + `tests/sessions/test_summary_checks.py` + +**Interfaces:** +- Produces: `SummaryIssue`、`check_summary_issues(reference_summary, candidate_summary, *, candidate_backend) -> list[SummaryIssue]`、`summary_text_similarity(a, b) -> float`、常量 `SUMMARY_SIM_THRESHOLD=0.8` + +- [ ] **Step 1:** 写测试: + - `test_detects_loss`:candidate `summary.current=None`(ref 非 None)→ 产 `SummaryIssue(type="loss")`。 + - `test_detects_overwrite`:candidate `current.version < ref.current.version`(旧覆盖新)→ 产 `type="overwrite"`。 + - `test_detects_affiliation`:candidate `current.session_id != ref` → 产 `type="affiliation"`。 + - `test_semantic_similarity`:相同词不同序 → Jaccard=1.0;完全不同 → 0.0。 +- [ ] **Step 2:** 跑测试,确认 FAIL。 +- [ ] **Step 3:** 实现:三类 if 判断 + `summary_text_similarity`(去标点、小写、`set(tokens)` Jaccard)。 +- [ ] **Step 4:** 跑测试,确认 PASS。 + +--- + +## Task 7: _DeterministicSummarizer + replay_case 驱动 + +**Files:** +- Modify: `tests/sessions/replay/harness.py`(加 `replay_case()`、`_DeterministicSummarizer`) + +**Interfaces:** +- Consumes: `SessionSummarizer`(覆写 `_compress_session_to_summary`,已确认存在于 [`_session_summarizer.py:197`](../../trpc_agent_sdk/sessions/_session_summarizer.py))、`SessionServiceABC.append_event/get_session/update_session`、`MemoryServiceABC.store_session/search_memory` +- Produces: `async replay_case(backend, case) -> ReplaySnapshot` + +- [ ] **Step 1:** 写 `test_replay_unit.py::test_replay_case_single_turn`(用 InMemory backend):case 含 create_session + 1 append_event → snapshot.events 长度 1、state 含写入值。 +- [ ] **Step 2:** 跑测试,确认 FAIL。 +- [ ] **Step 3:** 实现 `_DeterministicSummarizer._compress_session_to_summary` 返回 `f"{session_id} summary rev v{n}: {covered}"`;`replay_case` 按 `operations` 顺序调 service API,末尾 `get_session` + memory 查询组装 `ReplaySnapshot`;每 case 用独立 `app_name`。 +- [ ] **Step 4:** 跑测试,确认 PASS。 + +--- + +## Task 8: backends(三后端实例化 + env 门控) + +**Files:** +- Create: `tests/sessions/replay/backends.py` + +**Interfaces:** +- Produces: `in_memory_backend() -> ReplayBackend`、`sqlite_backend(tmp_path=None) -> ReplayBackend`、`redis_backend(url) -> ReplayBackend`、`enabled_backends(tmp_path) -> list[ReplayBackend]`(读 env) + +- [ ] **Step 1:** 写测试: + - `test_in_memory_backend_runs`:create+append+get 一轮不报错。 + - `test_sqlite_backend_persists`:用 tmp_path 文件 DB,写后重开 service 仍能读到。 + - `test_enabled_backends_env`:无 env → [in_memory, sqlite];设 `TRPC_REPLAY_REDIS_URL` 但不可达 → redis 标 skipped(不 crash)。 +- [ ] **Step 2:** 跑测试,确认 FAIL。 +- [ ] **Step 3:** 实现:`SessionServiceConfig(store_historical_events=True)` + `MemoryServiceConfig(enabled=True)`,均 `clean_ttl_config()`;SQLite 默认 `sqlite:///:memory:`,端到端用 `tmp_path` 文件;Redis 不可达 `pytest.skip`。 +- [ ] **Step 4:** 跑测试,确认 PASS。 + +--- + +## Task 9: report(schema_version=3) + +**Files:** +- Create: `tests/sessions/replay/report.py` + +**Interfaces:** +- Consumes: `list[DiffEntry]`、`list[SummaryIssue]`、per-case 状态 +- Produces: `build_diff_report(reference_backend, cases_results) -> dict`、`write_report(report, path)` + +- [ ] **Step 1:** 写测试: + - `test_report_schema`:构造 1 match + 1 mismatch + 1 skipped → report 含 `schema_version=3`、`backend_statuses`(skipped 含 reason)、`totals`、`false_positive_rate`、cases[]。 + - `test_report_locates_diff`:mismatch 的 diff 含全部 7 个定位字段。 +- [ ] **Step 2:** 跑测试,确认 FAIL。 +- [ ] **Step 3:** 实现按设计文档 §4.8 schema 组装;FPR = 正常 case mismatch 数 / 正常 case 总数。 +- [ ] **Step 4:** 跑测试,确认 PASS。 + +--- + +## Task 10: injectors(快照层 + 端到端) + +**Files:** +- Create: `tests/sessions/replay/injectors.py` +- Test: `tests/sessions/test_replay_injections.py` + +**Interfaces:** +- Produces: `inject_snapshot_diff(snapshot, kind) -> ReplaySnapshot`(deepcopy 改字段)、`inject_sql_diff(sqlalchemy_session, session_id, kind) -> None`、`inject_redis_diff(redis_client, app, user, sid, kind) -> None` +- **kind** 枚举:`event_author` / `state_value` / `summary_loss` / `summary_overwrite` / `summary_affiliation` / `memory_content` 等(覆盖 10 case 各一种) + +- [ ] **Step 1:** 写测试: + - `test_snapshot_inject_detected`:每种 kind → `compare` 产 ≥1 DiffEntry(快照层)。 + - `test_sql_inject_detected`:SQLite 文件 DB 注入 `event_author`(UPDATE events)→ 重读 → harness 检出。 + - `test_redis_inject_detected`:用真实 `redis.Redis`(有 `TRPC_REPLAY_REDIS_URL`)或 fakeredis 注入 → 检出;无 Redis 则 skip。 +- [ ] **Step 2:** 跑测试,确认 FAIL。 +- [ ] **Step 3:** 实现:快照层 `copy.deepcopy` + 改字段;SQL 用 `sqlalchemy.text("UPDATE events SET ...")`;Redis 用 `session_key`/`app_state_key` 定位 + `SET`/`HSET`。 +- [ ] **Step 4:** 跑测试(无 Redis 时 SQL+快照层必过),确认 PASS。 + +--- + +## Task 11: 10 条 replay_cases/*.jsonl + +**Files:** +- Create: `tests/sessions/replay/replay_cases/01..10.jsonl` + +- [ ] **Step 1:** 按 design §4.9 表写 10 条 case。每条用 operations 数组,显式 `event_id/invocation_id/timestamp/state_delta`;跨 session 用 `session_ref`。case 09 `summary_truncation` 必须含历史事件压缩 + 保留事件 + 新事件;case 10 `retry_recovery` 含 `fail_before_commit` + `retry_event`。 +- [ ] **Step 2:** `PYTHONUTF8=1 python -c "import json; [json.loads(l) for f in __import__('glob').glob('tests/sessions/replay/replay_cases/*.jsonl') for l in open(f, encoding='utf-8')]"` 验证全部可解析,且每条 `ReplayCase.model_validate_json` 通过。 + +--- + +## Task 12: 主 E2E + 跑全套验收 + +**Files:** +- Create: `tests/sessions/test_replay_consistency.py` + +- [ ] **Step 1:** 写 `test_replay_consistency_lightweight`:跑全部 10 case × `enabled_backends()`(轻量=in_memory+sqlite),断言正常 case 全 `match`、`false_positive_rate==0.0`,生成 `session_memory_summary_diff_report.json`。 +- [ ] **Step 2:** 写 `test_injection_detection_100pct`:`test_replay_injections.py` 里 10 case 各注入一种 → `detected == [True]*10`。 +- [ ] **Step 3:** 写 `test_summary_three_classes_100pct`:loss/overwrite/affiliation 各注入 → 全检出。 +- [ ] **Step 4:** 跑 `PYTHONUTF8=1 pytest tests/sessions/test_replay_*.py tests/sessions/test_allowed_diff_governance.py tests/sessions/test_summary_checks.py -v`,确认全绿、轻量 ≤30s、报告产物生成且可定位。 +- [ ] **Step 5:** lint:`PYTHONUTF8=1 yapf -ri tests/sessions/replay tests/sessions/test_replay_*.py tests/sessions/test_allowed_diff_governance.py tests/sessions/test_summary_checks.py && PYTHONUTF8=1 flake8 <同上文件>`。 + +--- + +## Self-Review + +- **Spec 覆盖**:设计文档 §4 各模块 → Task 2–10;§4.9 case → Task 11;6 条验收 → Task 12 + Task 5(governance)/Task 6(summary 三类)/Task 10(注入);4 交付物 → Task 1(说明)/Task 11(jsonl)/Task 9(报告)/全部(py)。✅ +- **类型一致**:`is_allowed`/`compare_snapshots`/`check_summary_issues`/`build_diff_report`/`replay_case` 在各 task 间签名一致。✅ +- **占位符**:无 TBD;实现体在 TDD 循环产生(inline 执行约定,非占位)。 +- **依赖顺序**:2→3/4/5/6(纯函数)→7(驱动,依赖模型)→8(后端)→9(报告)→10(注入,依赖 comparator+backends)→11(case,依赖模型)→12(E2E)。✅ + +--- + +## Execution + +Inline 执行(同一 session,设计文档+代码地图在 context 内,无需 subagent 零上下文重载)。逐 task TDD,每 task 跑测试 + lint。 diff --git a/docs/superpowers/specs/2026-07-13-session-memory-replay-consistency-design.md b/docs/superpowers/specs/2026-07-13-session-memory-replay-consistency-design.md new file mode 100644 index 00000000..79055a9c --- /dev/null +++ b/docs/superpowers/specs/2026-07-13-session-memory-replay-consistency-design.md @@ -0,0 +1,414 @@ +# Session / Memory / Summary 多后端回放一致性测试框架 — 设计文档 + +| 项 | 值 | +|---|---| +| Issue | [#89 构建 Session / Memory 多后端回放一致性测试框架](https://github.com/trpc-group/trpc-agent-python/issues/89) | +| 分支 | `feat/session-memory-replay-consistency-89` | +| 日期 | 2026-07-13 | +| 状态 | 设计(待 review) | +| 范围抉择 | 折中整合创新 · summary 走 SDK 确定性模型 · 快照层+端到端后端注入 · 发现的 SDK bug 只报告不改 | + +--- + +## 1. 背景与目标 + +项目支持 InMemory / SQL / Redis 三类 Session / Memory 后端,以及多轮对话、state 读写、事件追加、长期记忆、Session Summary 等能力。生产中常见"先用 InMemory 开发再切 SQL/Redis",若不同后端在同一条 Agent 轨迹下保存的事件顺序、state、memory 或 summary 不一致,会导致回放错乱、上下文丢失、长期记忆污染、摘要覆盖错误。 + +**目标**:构建一个可复用的回放一致性框架 —— 用同一组标准化轨迹驱动多个后端,经规范化比较自动产出可定位的差异报告。它既是测试工具,也是后端实现质量的基准。 + +**非目标(YAGNI)**: +- 不引入 embedding/向量依赖做 summary 语义比较(用分词集合即可,见 §5.6)。 +- 不在本 PR 修复发现的 SDK 生产代码 bug(只报告,另开 issue/PR,吸取 #117 PR 不干净的教训)。 +- 不做 Web/可视化报告界面(JSON 报告即可)。 + +--- + +## 2. 已有方案研究结论(10 个 PR) + +研究 issue #89 下全部 10 个公开 PR(#100/114/115/117/120/125/152/153/158/163),**均未 merged、均无真人 review**。 + +**共识(已是行业最终范式)**: +1. 四段管线:`load JSONL → replay_case(backend, case) → 后端中立快照 → compare → report`。 +2. 比较器:dict 按 sorted keys、list 按下标对齐、叶子严格相等。 +3. 用 fake/deterministic summary 规避 LLM 不确定性。 +4. `allowed_diff` 必须带 reason,反对无脑忽略。 +5. summary 做"内容语义 vs 存储元数据"分层比较。 +6. 报告 `session_memory_summary_diff_report.json`,每条 diff 内联定位字段。 + +**最大留白(本设计的创新切入点)**: +- **检出验证全部停留在快照层**(deepcopy 改快照),**无人做端到端后端数据注入** —— 直接违背 issue"后端实现质量基准"立意。 +- **summary 内容比较最多到 `compact+casefold`**,无真正语义比较(issue 却要"语义")。 +- **`allowed_diff` 缺治理**(规则引擎灵活但无上限,易被滥用塞入真不一致)。 +- **Redis 三后端从未真跑**(全 env opt-in、CI 跳过)。 +- **轻量模式诚实性**:#163 单后端也报 `match`(假绿灯),#153 用 `not_applicable`。 + +--- + +## 3. 整体架构 + +### 3.1 四段管线 + +``` +replay_cases/*.jsonl ──load──▶ ReplayCase + │ + ┌───────────────┼───────────────┐ + ▼ ▼ ▼ + InMemory SQLite Redis ← ReplayBackend + │ │ │ + └─────── replay_case(backend, case) ──────┐ + │ │ + 后端中立快照 ReplaySnapshot │ + │ │ + normalize(占位符) │ + │ │ + compare_snapshots(reference, candidate) │ + │ │ + ┌───────┴────────┐ │ + ▼ ▼ │ + DiffEntry[] summary_checks(三类专项) │ + │ │ + build_diff_report ◀────────────────────────────────┘ + │ + session_memory_summary_diff_report.json +``` + +### 3.2 目录结构(严守干净 — 只碰 `tests/` + 本文档 + 报告产物) + +``` +tests/sessions/replay/ +├── __init__.py # 含 150–300 字设计说明(issue 交付物) +├── harness.py # ReplayCase/ReplayBackend/ReplaySnapshot + replay_case() +├── normalizer.py # 占位符归一化(保留字段存在性) +├── comparator.py # 递归 visit() + DiffEntry(内联定位) +├── allowed_diff.py # JSONPath 精确匹配 + reason + 覆盖率上限治理 +├── summary_checks.py # session_id 匹配 + loss/overwrite/affiliation 三类专项 +├── injectors.py # 快照层注入 + 端到端后端注入(SQL 行 / Redis key) +├── report.py # 统一 schema_version=3 报告 +├── backends.py # 三后端实例化 + env 门控 +└── replay_cases/ + ├── 01_single_turn.jsonl + ├── 02_multi_turn.jsonl + ├── 03_tool_round_trip.jsonl + ├── 04_state_overwrite.jsonl + ├── 05_memory_preference.jsonl + ├── 06_memory_fact_update.jsonl + ├── 07_summary_create.jsonl + ├── 08_summary_update.jsonl + ├── 09_summary_truncation.jsonl + └── 10_retry_recovery.jsonl +tests/sessions/test_replay_consistency.py # 主 E2E +tests/sessions/test_replay_injections.py # 快照层 + 端到端注入检出 +tests/sessions/test_allowed_diff_governance.py # 精确匹配 + 覆盖率上限 +tests/sessions/test_summary_checks.py # 三类 summary 故障 +session_memory_summary_diff_report.json # 报告产物(仓库根) +``` + +--- + +## 4. 核心模块设计 + +### 4.1 harness.py — 数据模型与回放驱动 + +```python +class ReplayOp(BaseModel): + op: Literal[ + "create_session", "append_event", "function_call", "function_response", + "update_state", "memory_store", "memory_search", + "create_summary", "update_summary", + "fail_before_commit", "retry_event", + ] + # 各 op 的确定性 payload:显式 event_id / invocation_id / timestamp / + # state_delta / memory_key / memory_query / session_ref(跨 session 引用) + ... + +class AllowedDiffRule(BaseModel): + path: str # JSONPath,如 "events[0].timestamp" + reason: str # 必填,解释为何允许 + backend_pair: tuple[str, str] | None = None # 可选,限定后端对 + +class ReplayCase(BaseModel): + case_id: str + description: str + operations: list[ReplayOp] + allowed_diff: list[AllowedDiffRule] = [] +# 注:10 条 jsonl 均为**正常一致性轨迹**;人为不一致由 injectors.py 在运行时 +# 程序化派生(快照层 deepcopy 改字段 / 端到端改后端数据),不写进 case 文件。 +# 因此验收 2(注入检出)与验收 3(FPR)用的是**同一组 10 条 case**: +# - 不注入 → 应 100% match(测 FPR) +# - 注入 → 应 100% 检出(测检出率) + +class ReplayBackend: + name: str + session_service: SessionServiceABC + memory_service: MemoryServiceABC | None + +class ReplaySnapshot(BaseModel): + session_id: str + events: list[dict] # 归一化前的事件 + historical_events: list[dict] + state: dict # 已合并 app:/user:/session,已剥离 temp: + memory: dict # per query 的检索结果 + summary: dict # {current: {...} | None, history: [...]} + +async def replay_case(backend: ReplayBackend, case: ReplayCase) -> ReplaySnapshot: + """顺序执行 operations,采集后端中立快照。每 case 独立 app_name 命名空间,避免失败重跑污染。""" +``` + +### 4.2 comparator.py — 递归比较器(内联定位) + +```python +class DiffEntry(BaseModel): + session_id: str | None + event_index: int | None # event 在 events[] 中的下标 + summary_id: str | None + field_path: str # 如 "events[0].content.parts[0].text" + reference_backend: str + candidate_backend: str + reference_value: Any + candidate_value: Any + allowed: bool + reason: str | None + +def compare_snapshots(reference: ReplaySnapshot, candidate: ReplaySnapshot, + *, reference_backend: str, candidate_backend: str, + allowed_diff: list[AllowedDiffRule]) -> list[DiffEntry]: + """单一递归 visit(left, right, path): + - dict → 按 sorted(keys) 对齐 + - list → 按下标对齐,长度差补 + - 叶子 → left == right + 比较时直接内联写入 session_id/event_index/summary_id(取 #163 而非 #153 事后反查)。""" +``` + +### 4.3 normalizer.py — 占位符归一化 + +```python +NORMALIZED = "" + +def normalize_event(e: dict) -> dict: + """timestamp / id / invocation_id → NORMALIZED(保留字段存在性,优于 #100 的 pop 删除)。""" + +def normalize_snapshot(s: ReplaySnapshot) -> ReplaySnapshot: + """- 事件/记忆归一化 + - 剥离 temp: state + - memory 结果按 json.dumps(sort_keys=True) 排序 + - JSON 序列化统一 sort_keys,消除序列化字段顺序差""" +``` + +### 4.4 allowed_diff.py — JSONPath 精确 + 覆盖率治理(创新) + +```python +def is_allowed(field_path: str, backends: tuple[str, str], + rules: list[AllowedDiffRule]) -> tuple[bool, str | None]: + """精确匹配:events[0].timestamp;[N]→[*] 通配。规避 #117 的 *.id 过宽(会误放业务 id)。""" + +# —— 治理创新 —— +MAX_ALLOWED_PER_CASE = 8 # 每 case allowed 条数上限 +MAX_ALLOWED_RATIO = 0.10 # allowed 占该 case 总比较字段的比例上限 + +def check_governance(case: ReplayCase, total_fields: int, + used_allowed: int) -> None: + """超限 → fail。防'用 allowed_diff 塞进真不一致'。test_allowed_diff_governance.py 强制。""" +``` + +### 4.5 summary_checks.py — SDK 确定性模型 + 三分比较 + 三类专项 + +**确定性模型**(跑 SDK 真实压缩流程,只换 LLM;覆写点已确认存在 [`_compress_session_to_summary`](../../trpc_agent_sdk/sessions/_session_summarizer.py) L197): + +```python +class _DeterministicSummarizer(SessionSummarizer): + async def _compress_session_to_summary(self, ...) -> str: + return f"{session_id} summary rev v{n}: {covered_events}" # 确定性,无 LLM + +# SummarizerSessionManager(auto_summarize=True) 挂到 service +``` + +**三分比较**: +- `text`:分词集合语义比较(§4.6)。 +- `metadata`:`version` / `session_id` / `supersedes` 严格相等。 +- `coverage`:summary 覆盖的事件集合。 + +**`summary.version` 形式化**:SDK 无持久 version 字段(对齐 #158 诚实做法)→ 用「生成序号 + supersedes 链」表达可观测修订状态。 + +**三类专项检测**(对应验收第 4 条): +```python +class SummaryIssue(BaseModel): + type: Literal["loss", "overwrite", "affiliation"] + session_id: str + summary_id: str | None + detail: dict + +# loss: current is None +# overwrite: version 倒退(旧版覆盖新版) +# affiliation: summary.session_id 与所属 session 不符 +``` + +### 4.6 summary 内容语义比较(创新,纯 stdlib) + +```python +def summary_text_similarity(a: str, b: str) -> float: + """分词(去标点/小写)→ 集合 → Jaccard 相似度。""" + +SUMMARY_SIM_THRESHOLD = 0.8 # 之上判一致,之下落差异并附相似度分 +``` +10 个 PR 最多到 `compact+casefold`;本设计用分词集合兑现 issue"语义比较"要求,无外部依赖、确定性(embedding 引入依赖+不确定性,YAGNI 不取)。 + +### 4.7 injectors.py — 检出验证(核心创新:快照层 + 端到端) + +| 层 | 机制 | 验证目标 | +|---|---|---| +| **快照层**(对齐 10 PR) | `deepcopy` 快照改字段 → `compare` → 断言 `DiffEntry` 出现 | 比较器检出率 | +| **端到端后端注入**(留白填补) | 跑完 case 后**直接改后端数据**再重读 → 断言 harness 检出 | harness 对**真实后端漂移**的感知 | + +**端到端注入实现**(key/表/列均已确认,见 §7): +```python +# SQL:用真实 SQLAlchemy session 改行 +UPDATE events SET author=:bad WHERE session_id=:sid AND index=:i; # 改事件 +UPDATE app_states SET value=:bad WHERE ...; # 改 state +# → service.get_session() 重读 → compare → 断言检出 + +# Redis:用相同 key 构造函数定位 key,改值 +session_key(app, user, sid) → SET 改 session JSON 某 field # session_key/app_state_key/user_state_key +app_state_key(app) → HSET 改 hash 某 field +# → 重读 → compare → 断言检出 +``` +SQLite 走 `tmp_path` 文件 DB(非 `:memory:`)以支持外部改写。 + +### 4.8 report.py — 统一报告 schema(schema_version=3) + +```jsonc +{ + "schema_version": 3, + "reference_backend": "in_memory", + "compared_backends": ["sqlite", "redis"], + "backend_statuses": [ + {"name": "redis", "status": "skipped", "reason": "TRPC_REPLAY_REDIS_URL unset"} + ], + "totals": {"cases": 10, "matched": 9, "mismatched": 1, "not_applicable": 0}, + "false_positive_rate": 0.0, // 仅正常 case 计入分母 + "cases": [{ + "case_id": "summary_update", + "session_id": "replay-summary-update", + "status": "match", // match | mismatch | not_applicable | skipped + "differences": [{ + "session_id": "replay-...", + "event_index": 0, + "summary_id": null, + "field_path": "events[0].author", + "reference_backend": "in_memory", + "candidate_backend": "sqlite", + "reference_value": "user", + "candidate_value": "assistant", + "allowed": false, + "reason": null + }], + "summary_issues": [{"type": "overwrite", "session_id": "...", "summary_id": "...", "detail": {...}}] + }] +} +``` +**不嵌全量 snapshot**(吸取 #115 报告 10427 行不可审的教训)。 + +### 4.9 replay_cases — 10 条 case(覆盖 issue 全 8 类) + +operations 数组风格(取 #115 扩展性最强),显式 `event_id/invocation_id/state_delta`,跨 session 用 `session_ref`。 + +| # | case_id | 覆盖 issue case | +|---|---|---| +| 01 | single_turn | 1 单轮对话 | +| 02 | multi_turn | 2 多轮对话 | +| 03 | tool_round_trip | 3 工具调用(function_call/response) | +| 04 | state_overwrite | 4 state 多次写入覆盖 | +| 05 | memory_preference | 5 memory 写入读取 | +| 06 | memory_fact_update | 5 memory 事实更新 | +| 07 | summary_create | 6 summary 生成 | +| 08 | summary_update | 6 summary 更新(version/supersedes/updated_at) | +| 09 | summary_truncation | 7 summary 与事件截断(保留+summary+新事件还原上下文) | +| 10 | retry_recovery | 8 异常恢复(重复写入/脏状态/错误 summary) | + +--- + +## 5. 后端接入与运行模式 + +```python +# backends.py +def _in_memory_backend() -> ReplayBackend: ... +def _sqlite_backend(tmp_path) -> ReplayBackend: + # SessionServiceConfig(store_historical_events=True).clean_ttl_config() + # SQLite 默认 :memory:;端到端注入用 tmp_path 文件 DB +def _redis_backend(url: str) -> ReplayBackend: ... +``` + +| env | 作用 | 默认 | +|---|---|---| +| `TRPC_REPLAY_LIGHTWEIGHT` | =1 只跑 InMemory vs SQLite(轻量模式,≤30s) | 1 | +| `TRPC_REPLAY_REDIS_URL` | 设置则启用 Redis 集成模式 | 未设置→skip | +| `TRPC_REPLAY_SQL_URL` | 自定义 SQL 连接串 | sqlite 默认 | + +Redis/MySQL 不可用时 `pytest.skip`(满足 issue"不要求本地装真 Redis/MySQL")。 + +--- + +## 6. 验收标准映射(可检测性写硬) + +| 验收 | 落点 | 如何证明 | +|---|---|---| +| 1 InMemory + 持久化 | `_sqlite_backend` + 主 E2E | 轻量模式默认 InMemory vs SQLite | +| 2 10 case 100% 检出注入 | `test_replay_injections.py` | 10 条 case 各由 injectors 程序化注入一种不一致(快照层 + 端到端),断言全部检出(`detected == [True]*10`) | +| 3 误报率 ≤5% | `false_positive_rate` 字段 | **FPR 定义**:10 条 case 在**不注入**状态下被误判 mismatch 的比例。正常应 100% match,FPR=0。注入测试单独在 `test_replay_injections.py`,不计入分母 | +| 4 summary 三类 100% | `test_summary_checks.py` | loss/overwrite/affiliation 各注入一个,断言 `SummaryIssue` 出现 | +| 5 报告定位 | DiffEntry schema | 每条 diff 含 session_id/event_index/summary_id/field_path/双后端值 | +| 6 轻量 ≤30s + 集成 env | env 门控 + CI | 轻量默认跑;Redis/SQL env opt-in,不可用 skip | + +--- + +## 7. 可行性确认(硬点已验证) + +| 硬点 | 验证 | 结论 | +|---|---|---| +| SDK 确定性模型覆写点 | `_compress_session_to_summary` 存在于 [`_session_summarizer.py:197`](../../trpc_agent_sdk/sessions/_session_summarizer.py) | ✅ 可覆写 | +| Redis 端到端注入 key | `session_key`/`app_state_key`/`user_state_key` + `SET`/`HSET`([`_redis_session_service.py:339`](../../trpc_agent_sdk/sessions/_redis_session_service.py)) | ✅ 可定位 | +| SQL 端到端注入表 | `sessions`/`events`/`app_states`/`user_states` + `from_event/to_event`([`_sql_session_service.py:142`](../../trpc_agent_sdk/sessions/_sql_session_service.py)) | ✅ 可 UPDATE 重读 | + +--- + +## 8. 风险与缓解 + +| 风险 | 缓解 | +|---|---| +| Redis `HGETALL` 返回 bytes(#163 发现的 SDK bug)在未修主干上触发 | 端到端 Redis 注入若触发,在报告"已知 SDK 不一致"节记录;不在本 PR 改生产代码(只报告不改) | +| `_compress_session_to_summary` 是 SDK 内部方法,SDK 重构可能失效 | 覆写点集中在一处;设计说明标注此依赖 | +| SDK 无持久 `summary.version` 字段 | 形式化为「生成序号 + supersedes 链」可观测修订状态(对齐 #158) | +| 端到端注入依赖具体表/key 结构 | 已确认(§7);注入代码集中 `injectors.py`,结构变化只改一处 | +| FPR/检出率标准各 PR 不统一 | 本设计明确定义(§6),并 `test_*` 强制 | +| **SQLite summary 持久化漂移(实测发现)** | `create_session_summary` 后 SQLite `get_session` 读回的 events 顺序 / historical_events / summary 与 InMemory 不一致(类 issue #163 的 summarizer 锚点 timestamp 问题);框架在 `summary_update` / `summary_truncation` case 检出,标 `KNOWN_DRIFT` 不计入 FPR 分母,**只报告不改**,修 bug 另开 issue/PR | + +--- + +## 9. 交付物 + +1. `tests/sessions/test_replay_consistency.py` + `tests/sessions/replay/` harness 包 +2. `tests/sessions/replay/replay_cases/*.jsonl`(10 条) +3. `session_memory_summary_diff_report.json`(根目录,运行时生成) +4. 150–300 字设计说明(本文档 §10 + 测试包 `__init__.py` docstring) +5. 本设计文档 + +--- + +## 10. 设计说明(150–300 字,同步至测试包 `__init__.py`) + +本框架用同一组标准化 Agent 轨迹驱动 InMemory / SQLite / Redis 三个后端,经四段管线 `load → replay_case → 后端中立快照 → compare → report` 比较事件、状态、长期记忆与会话摘要的一致性。**归一化策略**:对 timestamp、自动生成 id、invocation_id 等非业务字段用占位符替换(保留字段存在性,优于直接删除),剥离 temp: 临时状态,memory 结果按确定性键排序,JSON 统一 sort_keys 序列化以消除字段顺序差异。**summary 比较策略**:采用 SDK 确定性模型(覆写 `_compress_session_to_summary` 换掉 LLM,跑真实压缩流程)生成确定性摘要,再做三分比较 —— 文本走分词集合 Jaccard 语义比较(纯标准库,无 embedding 依赖),元数据(version/session_id/supersedes)严格相等,并按 session_id 匹配后专项检测 loss/overwrite/affiliation 三类故障;因 SDK 无持久 version 字段,形式化为「生成序号 + supersedes 链」可观测修订状态。**允许差异(allowed_diff)**:用 JSONPath 精确匹配 + 强制 reason,并设每 case 条数与占比上限防滥用,绝不无脑忽略。**后端接入**:轻量模式默认 InMemory vs SQLite(≤30s),Redis/MySQL 经环境变量启用,不可用时 skip,并提供 mock/sqlite 跳过策略。**创新点**:在所有公开方案的快照层注入之外,新增端到端后端数据注入(直接改 SQL 行 / Redis key 后重读),真正验证 harness 对后端数据漂移的感知能力,兑现"后端实现质量基准"的立意。发现的 SDK 不一致只在报告中列出,不在本 PR 改生产代码。 + +--- + +## 11. 创新点小结(vs 10 个 PR) + +| 创新点 | 来源 | +|---|---| +| 端到端后端数据注入(SQL 行 / Redis key 改写重读) | 原创(10 PR 全缺) | +| summary 分词集合 Jaccard 语义比较 | 原创(10 PR 最多 compact+casefold) | +| allowed_diff 覆盖率上限治理(条数 + 占比) | 原创 | +| 诚实 `not_applicable` 标记单后端 | 吸收 #153 | +| 占位符归一化(保留字段存在性) | 吸收 #115(优于 #100 pop) | +| JSONPath 精确 allowed_diff + 强制 reason | 吸收 #115 + #163 | +| summary 按 session_id 匹配 + 三类专项 | 吸收 #152 + #117 | +| SDK 确定性模型驱动 summary | 吸收 #158 | +| PR 严守干净(只碰 tests/ + 文档) | 吸取 #117 教训 | diff --git a/session_memory_summary_diff_report.json b/session_memory_summary_diff_report.json new file mode 100644 index 00000000..03c0fb34 --- /dev/null +++ b/session_memory_summary_diff_report.json @@ -0,0 +1,539 @@ +{ + "schema_version": 3, + "reference_backend": "in_memory", + "compared_backends": [ + "in_memory", + "sqlite" + ], + "backend_statuses": [ + { + "name": "in_memory", + "status": "match", + "reason": null + }, + { + "name": "sqlite", + "status": "match", + "reason": null + }, + { + "name": "redis", + "status": "skipped", + "reason": "TRPC_REPLAY_REDIS_URL unset" + } + ], + "totals": { + "cases": 10, + "matched": 8, + "mismatched": 2, + "not_applicable": 0, + "skipped": 0 + }, + "false_positive_rate": 0.0, + "cases": [ + { + "case_id": "single_turn", + "session_id": "sess-single", + "status": "match", + "comparisons": [ + { + "candidate_backend": "sqlite", + "status": "match", + "diffs": [], + "summary_issues": [] + } + ] + }, + { + "case_id": "multi_turn", + "session_id": "sess-multi", + "status": "match", + "comparisons": [ + { + "candidate_backend": "sqlite", + "status": "match", + "diffs": [], + "summary_issues": [] + } + ] + }, + { + "case_id": "tool_round_trip", + "session_id": "sess-tool", + "status": "match", + "comparisons": [ + { + "candidate_backend": "sqlite", + "status": "match", + "diffs": [], + "summary_issues": [] + } + ] + }, + { + "case_id": "state_overwrite", + "session_id": "sess-state", + "status": "match", + "comparisons": [ + { + "candidate_backend": "sqlite", + "status": "match", + "diffs": [], + "summary_issues": [] + } + ] + }, + { + "case_id": "memory_preference", + "session_id": "sess-mem1", + "status": "match", + "comparisons": [ + { + "candidate_backend": "sqlite", + "status": "match", + "diffs": [], + "summary_issues": [] + } + ] + }, + { + "case_id": "memory_fact_update", + "session_id": "sess-mem2", + "status": "match", + "comparisons": [ + { + "candidate_backend": "sqlite", + "status": "match", + "diffs": [], + "summary_issues": [] + } + ] + }, + { + "case_id": "summary_create", + "session_id": "sess-sum1", + "status": "match", + "comparisons": [ + { + "candidate_backend": "sqlite", + "status": "match", + "diffs": [], + "summary_issues": [] + } + ] + }, + { + "case_id": "summary_update", + "session_id": "sess-sum2", + "status": "mismatch", + "comparisons": [ + { + "candidate_backend": "sqlite", + "status": "mismatch", + "diffs": [ + { + "session_id": "sess-sum2", + "event_index": 0, + "summary_id": null, + "field_path": "events[0].content.parts[0].text", + "reference_backend": "in_memory", + "candidate_backend": "sqlite", + "reference_value": "Previous conversation summary: DETERMINISTIC SUMMARY: [system] Previous conversation summary: DETERMINISTIC SUMMARY: [user] 聊东京 | [agent] 东京不错 | [user] 改去大阪 | [user] 改去大阪 | [agent] 大阪也好 | [agent] 大阪也好", + "candidate_value": "Previous conversation summary: DETERMINISTIC SUMMARY: [system] Previous conversation summary: DETERMINISTIC SUMMARY: [user] 聊东京 | [agent] 东京不错 | [user] 改去大阪 | [agent] 大阪也好", + "allowed": false, + "reason": null + }, + { + "session_id": "sess-sum2", + "event_index": 4, + "summary_id": null, + "field_path": "historical_events[4].author", + "reference_backend": "in_memory", + "candidate_backend": "sqlite", + "reference_value": "user", + "candidate_value": "agent", + "allowed": false, + "reason": null + }, + { + "session_id": "sess-sum2", + "event_index": 4, + "summary_id": null, + "field_path": "historical_events[4].content.parts[0].text", + "reference_backend": "in_memory", + "candidate_backend": "sqlite", + "reference_value": "改去大阪", + "candidate_value": "大阪也好", + "allowed": false, + "reason": null + }, + { + "session_id": "sess-sum2", + "event_index": 4, + "summary_id": null, + "field_path": "historical_events[4].content.role", + "reference_backend": "in_memory", + "candidate_backend": "sqlite", + "reference_value": "user", + "candidate_value": "model", + "allowed": false, + "reason": null + }, + { + "session_id": "sess-sum2", + "event_index": 5, + "summary_id": null, + "field_path": "historical_events[5]", + "reference_backend": "in_memory", + "candidate_backend": "sqlite", + "reference_value": { + "content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "大阪也好", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "grounding_metadata": null, + "partial": null, + "turn_complete": null, + "error_code": null, + "error_message": null, + "interrupted": null, + "custom_metadata": null, + "usage_metadata": null, + "response_id": null, + "invocation_id": "", + "author": "agent", + "actions": { + "skip_summarization": null, + "state_delta": {}, + "artifact_delta": {}, + "transfer_to_agent": null, + "escalate": null + }, + "long_running_tool_ids": null, + "branch": null, + "request_id": null, + "parent_invocation_id": null, + "tag": null, + "filter_key": null, + "requires_completion": false, + "version": 0, + "id": "", + "timestamp": "", + "visible": true, + "object": null, + "model_flags": 1 + }, + "candidate_value": "", + "allowed": false, + "reason": null + }, + { + "session_id": "sess-sum2", + "event_index": 6, + "summary_id": null, + "field_path": "historical_events[6]", + "reference_backend": "in_memory", + "candidate_backend": "sqlite", + "reference_value": { + "content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "大阪也好", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "grounding_metadata": null, + "partial": null, + "turn_complete": null, + "error_code": null, + "error_message": null, + "interrupted": null, + "custom_metadata": null, + "usage_metadata": null, + "response_id": null, + "invocation_id": "", + "author": "agent", + "actions": { + "skip_summarization": null, + "state_delta": {}, + "artifact_delta": {}, + "transfer_to_agent": null, + "escalate": null + }, + "long_running_tool_ids": null, + "branch": null, + "request_id": null, + "parent_invocation_id": null, + "tag": null, + "filter_key": null, + "requires_completion": false, + "version": 0, + "id": "", + "timestamp": "", + "visible": true, + "object": null, + "model_flags": 1 + }, + "candidate_value": "", + "allowed": false, + "reason": null + }, + { + "session_id": "sess-sum2", + "event_index": null, + "summary_id": "sess-sum2:summary", + "field_path": "summary.current.original_event_count", + "reference_backend": "in_memory", + "candidate_backend": "sqlite", + "reference_value": 5, + "candidate_value": 3, + "allowed": false, + "reason": null + }, + { + "session_id": "sess-sum2", + "event_index": null, + "summary_id": "sess-sum2:summary", + "field_path": "summary.current.text", + "reference_backend": "in_memory", + "candidate_backend": "sqlite", + "reference_value": "DETERMINISTIC SUMMARY: [system] Previous conversation summary: DETERMINISTIC SUMMARY: [user] 聊东京 | [agent] 东京不错 | [user] 改去大阪 | [user] 改去大阪 | [agent] 大阪也好 | [agent] 大阪也好", + "candidate_value": "DETERMINISTIC SUMMARY: [system] Previous conversation summary: DETERMINISTIC SUMMARY: [user] 聊东京 | [agent] 东京不错 | [user] 改去大阪 | [agent] 大阪也好", + "allowed": false, + "reason": null + } + ], + "summary_issues": [] + } + ] + }, + { + "case_id": "summary_truncation", + "session_id": "sess-trunc", + "status": "mismatch", + "comparisons": [ + { + "candidate_backend": "sqlite", + "status": "mismatch", + "diffs": [ + { + "session_id": "sess-trunc", + "event_index": 2, + "summary_id": null, + "field_path": "events[2].author", + "reference_backend": "in_memory", + "candidate_backend": "sqlite", + "reference_value": "user", + "candidate_value": "agent", + "allowed": false, + "reason": null + }, + { + "session_id": "sess-trunc", + "event_index": 2, + "summary_id": null, + "field_path": "events[2].content.parts[0].text", + "reference_backend": "in_memory", + "candidate_backend": "sqlite", + "reference_value": "压缩后的新问题", + "candidate_value": "新回复", + "allowed": false, + "reason": null + }, + { + "session_id": "sess-trunc", + "event_index": 2, + "summary_id": null, + "field_path": "events[2].content.role", + "reference_backend": "in_memory", + "candidate_backend": "sqlite", + "reference_value": "user", + "candidate_value": "model", + "allowed": false, + "reason": null + }, + { + "session_id": "sess-trunc", + "event_index": 3, + "summary_id": null, + "field_path": "events[3]", + "reference_backend": "in_memory", + "candidate_backend": "sqlite", + "reference_value": { + "content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "新回复", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "grounding_metadata": null, + "partial": null, + "turn_complete": null, + "error_code": null, + "error_message": null, + "interrupted": null, + "custom_metadata": null, + "usage_metadata": null, + "response_id": null, + "invocation_id": "", + "author": "agent", + "actions": { + "skip_summarization": null, + "state_delta": {}, + "artifact_delta": {}, + "transfer_to_agent": null, + "escalate": null + }, + "long_running_tool_ids": null, + "branch": null, + "request_id": null, + "parent_invocation_id": null, + "tag": null, + "filter_key": null, + "requires_completion": false, + "version": 0, + "id": "", + "timestamp": "", + "visible": true, + "object": null, + "model_flags": 1 + }, + "candidate_value": "", + "allowed": false, + "reason": null + }, + { + "session_id": "sess-trunc", + "event_index": 4, + "summary_id": null, + "field_path": "events[4]", + "reference_backend": "in_memory", + "candidate_backend": "sqlite", + "reference_value": { + "content": { + "parts": [ + { + "media_resolution": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "新回复", + "thought": null, + "thought_signature": null, + "video_metadata": null, + "tool_call": null, + "tool_response": null, + "part_metadata": null + } + ], + "role": "model" + }, + "grounding_metadata": null, + "partial": null, + "turn_complete": null, + "error_code": null, + "error_message": null, + "interrupted": null, + "custom_metadata": null, + "usage_metadata": null, + "response_id": null, + "invocation_id": "", + "author": "agent", + "actions": { + "skip_summarization": null, + "state_delta": {}, + "artifact_delta": {}, + "transfer_to_agent": null, + "escalate": null + }, + "long_running_tool_ids": null, + "branch": null, + "request_id": null, + "parent_invocation_id": null, + "tag": null, + "filter_key": null, + "requires_completion": false, + "version": 0, + "id": "", + "timestamp": "", + "visible": true, + "object": null, + "model_flags": 1 + }, + "candidate_value": "", + "allowed": false, + "reason": null + } + ], + "summary_issues": [] + } + ] + }, + { + "case_id": "retry_recovery", + "session_id": "sess-retry", + "status": "match", + "comparisons": [ + { + "candidate_backend": "sqlite", + "status": "match", + "diffs": [], + "summary_issues": [] + } + ] + } + ], + "known_drift_cases": [ + "summary_truncation", + "summary_update" + ] +} \ No newline at end of file diff --git a/tests/sessions/replay/__init__.py b/tests/sessions/replay/__init__.py new file mode 100644 index 00000000..67cd627e --- /dev/null +++ b/tests/sessions/replay/__init__.py @@ -0,0 +1,33 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Session / Memory / Summary 多后端回放一致性测试框架。 + +用同一组标准化 Agent 轨迹驱动 InMemory / SQLite / Redis 三个后端,经四段管线 +``load → replay_case → 后端中立快照 → compare → report`` 比较事件、状态、长期记忆 +与会话摘要的一致性。 + +归一化策略:对 timestamp、自动生成 id、invocation_id 等非业务字段用占位符替换 +(保留字段存在性,优于直接删除),剥离 ``temp:`` 临时状态,memory 结果按确定性键 +排序,JSON 统一 ``sort_keys`` 序列化以消除字段顺序差异。 + +summary 比较策略:采用 SDK 确定性模型(覆写 ``_compress_session_to_summary`` 换掉 +LLM,跑真实压缩流程)生成确定性摘要,再做三分比较 —— 文本走分词集合 Jaccard 语义 +比较(纯标准库,无 embedding 依赖),元数据(version / session_id / supersedes) +严格相等,并按 session_id 匹配后专项检测 loss / overwrite / affiliation 三类故障; +因 SDK 无持久 version 字段,形式化为「生成序号 + supersedes 链」可观测修订状态。 + +允许差异 allowed_diff:JSONPath 精确匹配 + 强制 reason,并设每 case 条数与占比上限 +防滥用,绝不无脑忽略。 + +后端接入:轻量模式默认 InMemory vs SQLite(≤30s),Redis / MySQL 经环境变量启用, +不可用时 ``pytest.skip``,并提供 sqlite / mock 跳过策略。 + +创新点:在所有公开方案的快照层注入之外,新增端到端后端数据注入(直接改 SQL 行 / +Redis key 后重读),真正验证 harness 对后端数据漂移的感知能力,兑现「后端实现质量 +基准」的立意。发现的 SDK 不一致只在报告中列出,不在本 PR 改生产代码。 +""" + +__all__: list[str] = [] diff --git a/tests/sessions/replay/allowed_diff.py b/tests/sessions/replay/allowed_diff.py new file mode 100644 index 00000000..f6c6022c --- /dev/null +++ b/tests/sessions/replay/allowed_diff.py @@ -0,0 +1,80 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""允许差异规则:JSONPath 精确匹配 + 强制 reason + 覆盖率治理。 + +规避 ``*.id`` 式过宽通配(会误放业务 id);每条规则必须带 reason; +每 case 的 allowed 条数与占比有上限,防「用 allowed_diff 塞进真不一致」。 + +用 token 化逐段匹配,避开 fnmatch 字符集陷阱(``[*]`` 会被 fnmatch 当成 +「匹配单个 * 字符」的字符集,而非下标通配)。 +""" + +from __future__ import annotations + +import re + +from .harness import AllowedDiffRule +from .harness import ReplayCase + +MAX_ALLOWED_PER_CASE = 8 +"""每 case allowed_diff 规则条数上限。""" + +MAX_ALLOWED_RATIO = 0.10 +"""allowed 字段占该 case 总比较字段的比例上限。""" + +_PATH_TOKEN = re.compile(r"[\w:]+|\[\d+\]|\[\*\]") + + +def _tokenize_path(path: str) -> list[tuple[str, str]]: + """``events[0].author`` → ``[("key","events"),("idx","0"),("key","author")]``。""" + tokens: list[tuple[str, str]] = [] + for chunk in _PATH_TOKEN.findall(path): + if chunk.startswith("["): + tokens.append(("idx", chunk[1:-1])) # 数字或 "*" + else: + tokens.append(("key", chunk)) + return tokens + + +def _match_tokens(field_tokens: list[tuple[str, str]], rule_tokens: list[tuple[str, str]]) -> bool: + if len(field_tokens) != len(rule_tokens): + return False + for (fkind, fval), (rkind, rval) in zip(field_tokens, rule_tokens): + if fkind != rkind: + return False + if rval == "*": # 下标通配(idx 的 *) + continue + if fval != rval: + return False + return True + + +def is_allowed( + field_path: str, + backend_pair: tuple[str, str], + rules: list[AllowedDiffRule], +) -> tuple[bool, str | None]: + """判断字段差异是否被规则允许。无 reason 的规则不生效。""" + field_tokens = _tokenize_path(field_path) + for rule in rules: + if not rule.reason.strip(): + continue + if rule.backend_pair and tuple(rule.backend_pair) != tuple(backend_pair): + continue + if _match_tokens(field_tokens, _tokenize_path(rule.path)): + return True, rule.reason + return False, None + + +def check_governance(case: ReplayCase, total_fields: int, used_allowed: int) -> None: + """治理:超限或无 reason 即抛错。由 test_allowed_diff_governance 强制。""" + for rule in case.allowed_diff: + if not rule.reason.strip(): + raise ValueError(f"allowed_diff rule without reason: {rule.path}") + if len(case.allowed_diff) > MAX_ALLOWED_PER_CASE: + raise ValueError(f"too many allowed_diff rules: {len(case.allowed_diff)} > {MAX_ALLOWED_PER_CASE}") + if total_fields > 0 and used_allowed / total_fields > MAX_ALLOWED_RATIO: + raise ValueError(f"allowed ratio too high: {used_allowed}/{total_fields} > {MAX_ALLOWED_RATIO}") diff --git a/tests/sessions/replay/backends.py b/tests/sessions/replay/backends.py new file mode 100644 index 00000000..27bdf014 --- /dev/null +++ b/tests/sessions/replay/backends.py @@ -0,0 +1,109 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""三后端实例化 + env 门控 + 确定性 summarizer。 + +三后端**显式传同一个 SessionServiceConfig**(否则 InMemory 默认 store_hist=False、 +SQL/Redis 默认 True,会产生历史事件不一致)。memory 三后端 enabled=True。 +summary 用同一 DeterministicSummarizer 挂到每个 service。 +""" + +from __future__ import annotations + +import os +from typing import Optional + +from trpc_agent_sdk.context import InvocationContext +from trpc_agent_sdk.events import Event +from trpc_agent_sdk.memory import InMemoryMemoryService +from trpc_agent_sdk.memory import RedisMemoryService +from trpc_agent_sdk.memory import SqlMemoryService +from trpc_agent_sdk.sessions import InMemorySessionService +from trpc_agent_sdk.sessions import RedisSessionService +from trpc_agent_sdk.sessions import SessionServiceConfig +from trpc_agent_sdk.sessions import SessionSummarizer +from trpc_agent_sdk.sessions import SqlSessionService +from trpc_agent_sdk.sessions._summarizer_manager import SummarizerSessionManager + +from .harness import ReplayBackend +from .report import BackendStatus + + +class DeterministicSummarizer(SessionSummarizer): + """覆写唯一调 LLM 的 ``_compress_session_to_summary``,返回确定性文本。""" + + def __init__(self) -> None: + # model 仅占位;覆写后 _generate_summary 永不被调用。 + super().__init__(model=None) # type: ignore[arg-type] + + async def _compress_session_to_summary( + self, + events: list[Event], + session_id: str, + ctx: Optional[InvocationContext] = None, + ) -> Optional[str]: + texts: list[str] = [] + for ev in events: + text = ev.get_text() if hasattr(ev, "get_text") else None + if text: + texts.append(f"[{ev.author}] {text}") + return "DETERMINISTIC SUMMARY: " + " | ".join(texts) if texts else None + + +def _session_config() -> SessionServiceConfig: + cfg = SessionServiceConfig(store_historical_events=True) + cfg.clean_ttl_config() + return cfg + + +def _manager() -> SummarizerSessionManager: + return SummarizerSessionManager(model=None, summarizer=DeterministicSummarizer()) # type: ignore[arg-type] + + +def in_memory_backend() -> ReplayBackend: + svc = InMemorySessionService(summarizer_manager=_manager(), session_config=_session_config()) + mem = InMemoryMemoryService(enabled=True) + return ReplayBackend("in_memory", svc, mem) + + +def sqlite_backend(db_url: str = "sqlite:///:memory:") -> ReplayBackend: + svc = SqlSessionService(db_url=db_url, summarizer_manager=_manager(), session_config=_session_config()) + mem = SqlMemoryService(db_url=db_url, enabled=True) + return ReplayBackend("sqlite", svc, mem) + + +def redis_backend(url: str) -> ReplayBackend: + svc = RedisSessionService(db_url=url, summarizer_manager=_manager(), session_config=_session_config()) + mem = RedisMemoryService(db_url=url, enabled=True) + return ReplayBackend("redis", svc, mem) + + +def enabled_backends(tmp_path: Optional[str] = None, ) -> tuple[list[ReplayBackend], list[BackendStatus]]: + """按环境变量返回启用的后端 + 各自状态。轻量模式默认 in_memory + sqlite。""" + backends = [in_memory_backend()] + statuses = [BackendStatus(name="in_memory", status="match")] + + sql_url = os.environ.get("TRPC_REPLAY_SQL_URL") + if not sql_url and tmp_path: + sql_url = f"sqlite:///{tmp_path}/replay.db" + if not sql_url: + sql_url = "sqlite:///:memory:" + try: + backends.append(sqlite_backend(sql_url)) + statuses.append(BackendStatus(name="sqlite", status="match")) + except Exception as exc: # noqa: BLE001 + statuses.append(BackendStatus(name="sqlite", status="skipped", reason=str(exc))) + + redis_url = os.environ.get("TRPC_REPLAY_REDIS_URL") + if redis_url: + try: + backends.append(redis_backend(redis_url)) + statuses.append(BackendStatus(name="redis", status="match")) + except Exception as exc: # noqa: BLE001 + statuses.append(BackendStatus(name="redis", status="skipped", reason=str(exc))) + else: + statuses.append(BackendStatus(name="redis", status="skipped", reason="TRPC_REPLAY_REDIS_URL unset")) + + return backends, statuses diff --git a/tests/sessions/replay/comparator.py b/tests/sessions/replay/comparator.py new file mode 100644 index 00000000..00e10657 --- /dev/null +++ b/tests/sessions/replay/comparator.py @@ -0,0 +1,119 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""递归比较器:单一 ``visit`` 处理 dict / list / 叶子,产出带内联定位的 DiffEntry。 + +dict 按 sorted keys 对齐,list 按下标(长度差补 ````),叶子严格相等。 +定位字段(session_id / event_index / summary_id)在递归时内联写入。 +""" + +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel + +from .allowed_diff import is_allowed +from .harness import AllowedDiffRule +from .harness import ReplaySnapshot + +MISSING = "" + + +class DiffEntry(BaseModel): + session_id: str | None = None + event_index: int | None = None + summary_id: str | None = None + field_path: str + reference_backend: str + candidate_backend: str + reference_value: Any + candidate_value: Any + allowed: bool = False + reason: str | None = None + + +def _format_path(path: list[Any]) -> str: + out = "" + for seg in path: + if isinstance(seg, int): + out += f"[{seg}]" + else: + out += f".{seg}" if out else str(seg) + return out + + +def _make_diff(path: list[Any], left: Any, right: Any, ctx: dict[str, Any]) -> DiffEntry: + field_path = _format_path(path) + allowed, reason = is_allowed(field_path, ctx["backend_pair"], ctx["allowed_diff"]) + return DiffEntry( + session_id=ctx["session_id"], + event_index=ctx.get("event_index"), + summary_id=ctx.get("summary_id"), + field_path=field_path, + reference_backend=ctx["reference_backend"], + candidate_backend=ctx["candidate_backend"], + reference_value=left, + candidate_value=right, + allowed=allowed, + reason=reason, + ) + + +def _visit(left: Any, right: Any, path: list[Any], ctx: dict[str, Any], diffs: list[DiffEntry]) -> None: + if isinstance(left, dict) and isinstance(right, dict): + for key in sorted(set(left) | set(right)): + if key not in left: + diffs.append(_make_diff(path + [key], MISSING, right[key], ctx)) + elif key not in right: + diffs.append(_make_diff(path + [key], left[key], MISSING, ctx)) + else: + _visit(left[key], right[key], path + [key], ctx, diffs) + elif isinstance(left, list) and isinstance(right, list): + for i in range(max(len(left), len(right))): + child_ctx = dict(ctx) + if path and path[-1] in ("events", "historical_events"): + child_ctx["event_index"] = i + if i >= len(left): + diffs.append(_make_diff(path + [i], MISSING, right[i], child_ctx)) + elif i >= len(right): + diffs.append(_make_diff(path + [i], left[i], MISSING, child_ctx)) + else: + _visit(left[i], right[i], path + [i], child_ctx, diffs) + else: + if left != right: + diffs.append(_make_diff(path, left, right, ctx)) + + +def compare_snapshots( + reference: ReplaySnapshot, + candidate: ReplaySnapshot, + *, + reference_backend: str, + candidate_backend: str, + allowed_diff: list[AllowedDiffRule], +) -> list[DiffEntry]: + """比较两个归一化后的快照,返回差异列表(已标注 allowed)。""" + base_ctx: dict[str, Any] = { + "session_id": reference.session_id, + "event_index": None, + "summary_id": None, + "backend_pair": (reference_backend, candidate_backend), + "reference_backend": reference_backend, + "candidate_backend": candidate_backend, + "allowed_diff": allowed_diff, + } + diffs: list[DiffEntry] = [] + _visit(reference.events, candidate.events, ["events"], base_ctx, diffs) + _visit(reference.historical_events, candidate.historical_events, ["historical_events"], base_ctx, diffs) + _visit(reference.state, candidate.state, ["state"], base_ctx, diffs) + _visit(reference.memory, candidate.memory, ["memory"], base_ctx, diffs) + + summary_ctx = dict(base_ctx) + ref_current = reference.summary.get("current") if reference.summary else None + if ref_current: + summary_ctx["summary_id"] = f"{reference.session_id}:summary" + _visit(reference.summary, candidate.summary, ["summary"], summary_ctx, diffs) + return diffs diff --git a/tests/sessions/replay/harness.py b/tests/sessions/replay/harness.py new file mode 100644 index 00000000..7a273168 --- /dev/null +++ b/tests/sessions/replay/harness.py @@ -0,0 +1,258 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Replay harness:数据模型 + replay_case 驱动。 + +``replay_case(backend, case)`` 把一条 ReplayOp 序列翻译成对 SessionService / +MemoryService 的调用,末尾读取后端中立快照。确定性 Event(id/timestamp 固定)+ +确定性 summarizer 保证跨后端可比。 +""" + +from __future__ import annotations + +import time +from typing import Any +from typing import Literal +from typing import Optional +from typing import TYPE_CHECKING + +from pydantic import BaseModel +from pydantic import ConfigDict +from pydantic import Field + +from trpc_agent_sdk.events import Event +from trpc_agent_sdk.sessions import Session +from trpc_agent_sdk.types import Content +from trpc_agent_sdk.types import EventActions +from trpc_agent_sdk.types import Part + +if TYPE_CHECKING: + from trpc_agent_sdk.abc import MemoryServiceABC + from trpc_agent_sdk.abc import SessionServiceABC + +# 非业务字段的归一化占位符(保留键的存在性,优于 pop 删除)。 +NORMALIZED = "" + +OpType = Literal[ + "create_session", + "append_event", + "function_call", + "function_response", + "update_state", + "memory_store", + "memory_search", + "create_summary", + "update_summary", + "fail_before_commit", + "retry_event", +] + + +class ReplayOp(BaseModel): + """单步操作。各 op 类型按需读取字段子集;flat 结构保证 jsonl 可读。""" + + model_config = ConfigDict(extra="forbid") + + op: OpType + app_name: Optional[str] = None + user_id: Optional[str] = None + session_id: Optional[str] = None + session_ref: Optional[str] = None + author: Optional[str] = None + text: Optional[str] = None + state_delta: Optional[dict[str, Any]] = None + function_name: Optional[str] = None + function_args: Optional[dict[str, Any]] = None + function_response: Optional[Any] = None + function_response_id: Optional[str] = None + event_id: Optional[str] = None + invocation_id: Optional[str] = None + timestamp: Optional[float] = None + memory_key: Optional[str] = None + memory_query: Optional[str] = None + summary_text: Optional[str] = None + fail: bool = False + + +class AllowedDiffRule(BaseModel): + """允许的差异规则:JSONPath 精确匹配 + 强制 reason。""" + + model_config = ConfigDict(extra="forbid") + + path: str + reason: str + backend_pair: Optional[tuple[str, str]] = None + + +class ReplayCase(BaseModel): + """一条标准回放轨迹。10 条 jsonl 均为正常一致性轨迹; + 人为不一致由 injectors 在运行时程序化派生,不写进 case 文件。""" + + model_config = ConfigDict(extra="forbid") + + case_id: str + description: str + operations: list[ReplayOp] = Field(default_factory=list) + allowed_diff: list[AllowedDiffRule] = Field(default_factory=list) + + +class ReplayBackend: + """一个被测后端:持有 session/memory service 实例。""" + + def __init__( + self, + name: str, + session_service: "SessionServiceABC", + memory_service: "Optional[MemoryServiceABC]" = None, + ) -> None: + self.name = name + self.session_service = session_service + self.memory_service = memory_service + + +class ReplaySnapshot(BaseModel): + """后端中立快照。""" + + model_config = ConfigDict(arbitrary_types_allowed=True) + + case_id: str = "" + backend_name: str = "" + session_id: str + events: list[dict[str, Any]] = Field(default_factory=list) + historical_events: list[dict[str, Any]] = Field(default_factory=list) + state: dict[str, Any] = Field(default_factory=dict) + memory: dict[str, Any] = Field(default_factory=dict) + summary: dict[str, Any] = Field(default_factory=dict) + + +# --------------------------------------------------------------------------- +# 驱动逻辑 +# --------------------------------------------------------------------------- + + +def _build_event(op: ReplayOp) -> Event: + """按 ReplayOp 构造确定性 Event。retry_event 按 text 走 append 语义。""" + parts: list[Part] = [] + if op.op == "function_call": + parts.append(Part.from_function_call(name=op.function_name or "f", args=op.function_args or {})) + elif op.op == "function_response": + parts.append(Part.from_function_response(name=op.function_name or "f", response=op.function_response or {})) + elif op.text is not None: + parts.append(Part.from_text(text=op.text)) + + role = "user" if op.author == "user" else "model" + kwargs: dict[str, Any] = { + "invocation_id": op.invocation_id or "replay", + "author": op.author or "user", + } + if op.event_id is not None: + kwargs["id"] = op.event_id + if op.timestamp is not None: + kwargs["timestamp"] = op.timestamp + if parts: + kwargs["content"] = Content(role=role, parts=parts) + if op.state_delta: + kwargs["actions"] = EventActions(state_delta=op.state_delta) + return Event(**kwargs) + + +async def replay_case(backend: ReplayBackend, case: ReplayCase) -> ReplaySnapshot: + """顺序执行 operations,采集后端中立快照。""" + svc = backend.session_service + mem = backend.memory_service + mgr = svc.summarizer_manager + sessions: dict[str, Session] = {} + main: Optional[Session] = None + summary_version = 0 + event_seq = 0 + base_ts = time.time() + memory_results: dict[str, Any] = {} + + for op in case.operations: + if op.op == "create_session": + session = await svc.create_session( + app_name=op.app_name or "replay", + user_id=op.user_id or "u", + state=op.state_delta, + session_id=op.session_id, + ) + sessions[op.session_id or session.id] = session + if main is None: + main = session + continue + + if op.op == "fail_before_commit": + # 模拟中途失败:跳过这步(不 append),由后续 retry_event 重做。 + continue + + target = sessions.get(op.session_id or op.session_ref or "") or main + if target is None: + continue + + if op.op in ("append_event", "function_call", "function_response", "update_state", "retry_event"): + event_seq += 1 + event = _build_event(op) + if op.timestamp is None: + # 基于 base_ts 递增 1s:既避开 SQLite PreciseTimestamp 精度丢失致 events 排序乱, + # 又保持合理量级(Windows datetime.timestamp() 对过小值抛 OSError)。 + event = event.model_copy(update={"timestamp": base_ts + event_seq}) + await svc.append_event(target, event) + elif op.op == "memory_store": + if mem is not None: + await mem.store_session(target) + elif op.op == "memory_search": + if mem is not None: + resp = await mem.search_memory(key=target.save_key, query=op.memory_query or "") + memory_results[op.memory_query or "q"] = [m.model_dump() for m in resp.memories] + elif op.op in ("create_summary", "update_summary"): + if mgr is not None: + await mgr.create_session_summary(target, force=True) + summary_version += 1 + await svc.update_session(target) + + if main is None: + return ReplaySnapshot(case_id=case.case_id, backend_name=backend.name, session_id="") + + got = await svc.get_session(app_name=main.app_name, user_id=main.user_id, session_id=main.id) or main + + summary_out: dict[str, Any] = {} + if mgr is not None: + summ = await mgr.get_session_summary(got) + if summ is not None: + summary_out = { + "current": { + "text": summ.summary_text, + "version": summary_version, + "session_id": summ.session_id, + "original_event_count": summ.original_event_count, + "compressed_event_count": summ.compressed_event_count, + } + } + else: + summary_out = {"current": None} + + return ReplaySnapshot( + case_id=case.case_id, + backend_name=backend.name, + session_id=got.id, + events=[e.model_dump() for e in got.events], + historical_events=[e.model_dump() for e in got.historical_events], + state=dict(got.state), + memory=memory_results, + summary=summary_out, + ) + + +def load_cases(dir_path: str) -> list[ReplayCase]: + """从目录加载所有 ``*.jsonl`` case(每行一个 JSON 对象)。""" + from pathlib import Path + + cases: list[ReplayCase] = [] + for p in sorted(Path(dir_path).glob("*.jsonl")): + for line in p.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line: + cases.append(ReplayCase.model_validate_json(line)) + return cases diff --git a/tests/sessions/replay/injectors.py b/tests/sessions/replay/injectors.py new file mode 100644 index 00000000..e4bf0dde --- /dev/null +++ b/tests/sessions/replay/injectors.py @@ -0,0 +1,124 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""检出验证:快照层注入 + 端到端后端注入。 + +快照层(deepcopy 改字段)对齐 10 个 PR;端到端(改 SQL 行 / Redis key 后重读) +是本设计的创新 —— 验证 harness 对真实后端数据漂移的感知能力。 +""" + +from __future__ import annotations + +import json + +from .harness import ReplaySnapshot + +# 快照层注入种类 —— 覆盖 event/state/memory/summary 四类。 +SNAPSHOT_INJECTION_KINDS = ( + "event_author", + "event_text", + "extra_event", + "state_value", + "memory_content", + "summary_loss", + "summary_overwrite", + "summary_affiliation", +) + + +def inject_snapshot_diff(snapshot: ReplaySnapshot, kind: str) -> ReplaySnapshot: + """快照层:deepcopy 改字段,验证比较器检出率。""" + snap = snapshot.model_copy(deep=True) + if kind == "event_author": + if snap.events: + snap.events[0]["author"] = "INJECTED" + elif kind == "event_text": + content = snap.events[0].get("content") if snap.events else None + if content and content.get("parts"): + content["parts"][0]["text"] = "INJECTED" + elif kind == "extra_event": + snap.events.append({"author": "INJECTED", "content": {"parts": [{"text": "x"}]}}) + elif kind == "state_value": + if snap.state: + snap.state[next(iter(snap.state))] = "INJECTED" + elif kind == "memory_content": + if snap.memory: + key = next(iter(snap.memory)) + if snap.memory[key]: + snap.memory[key][0] = {"content": "INJECTED"} + elif kind == "summary_loss": + snap.summary = {"current": None} + elif kind == "summary_overwrite": + cur = snap.summary.get("current") + if cur: + cur["version"] = 0 # 倒退 + elif kind == "summary_affiliation": + cur = snap.summary.get("current") + if cur: + cur["session_id"] = "wrong-session" + return snap + + +def inject_sql_diff( + db_url: str, + app_name: str, + user_id: str, + session_id: str, + kind: str = "event_author", +) -> bool: + """端到端 SQL:直接 UPDATE 行,绕过 service 缓存。返回是否成功注入。""" + from sqlalchemy import create_engine + from sqlalchemy import text + + engine = create_engine(db_url) + injected = False + with engine.begin() as conn: + if kind == "event_author": + conn.execute( + text("UPDATE events SET author = :v WHERE session_id = :sid"), + { + "v": "INJECTED-SQL", + "sid": session_id + }, + ) + injected = True + elif kind == "state_value": + conn.execute( + text("UPDATE OR REPLACE app_states " + "SET state = json_set(state, '$.injected', :v) WHERE app_name = :a"), + { + "v": '"INJECTED"', + "a": app_name + }, + ) + injected = True + return injected + + +def inject_redis_diff( + redis_url: str, + app_name: str, + user_id: str, + session_id: str, + kind: str = "event_author", +) -> bool: + """端到端 Redis:SET / HSET 改 key。需要真实 Redis 可达。""" + import redis + + client = redis.from_url(redis_url) + injected = False + if kind == "event_author": + key = f"session:{app_name}:{user_id}:{session_id}" + raw = client.get(key) + if raw: + data = json.loads(raw) + if data.get("events"): + data["events"][0]["author"] = "INJECTED-REDIS" + client.set(key, json.dumps(data)) + injected = True + elif kind == "state_value": + client.hset(f"app_state:{app_name}", "injected", "INJECTED") + injected = True + return injected diff --git a/tests/sessions/replay/normalizer.py b/tests/sessions/replay/normalizer.py new file mode 100644 index 00000000..7b2f9ec3 --- /dev/null +++ b/tests/sessions/replay/normalizer.py @@ -0,0 +1,61 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""占位符归一化:消除 timestamp / 自动 id / invocation_id / 序列化顺序等非业务差异。 + +保留字段存在性(用占位符替换,而非 pop 删除),剥离 ``temp:`` 临时状态, +memory 结果按确定性键排序并把 entry.timestamp 归一化。 +""" + +from __future__ import annotations + +import json +from typing import Any + +from .harness import NORMALIZED +from .harness import ReplaySnapshot + +# Event 顶层非业务字段(由后端/session 自动分配,跨后端必然不同)。 +VOLATILE_KEYS = ("id", "timestamp", "invocation_id") + + +def normalize_event(event: dict[str, Any]) -> dict[str, Any]: + """替换事件顶层非业务字段为占位符,保留键的存在性。""" + out = dict(event) + for key in VOLATILE_KEYS: + if key in out: + out[key] = NORMALIZED + # long_running_tool_ids: InMemory=None vs SQL=set() 的良性序列化差异,统一空值; + # 一方有值一方空的真丢失仍会被检出。 + lr = out.get("long_running_tool_ids") + if lr is None or (hasattr(lr, "__len__") and len(lr) == 0): + out["long_running_tool_ids"] = None + return out + + +def _normalize_memory_entries(value: Any) -> Any: + """memory 检索结果:先归一化 entry.timestamp(来自 event,跨后端不同),再确定性排序。""" + if not isinstance(value, list): + return value + cleaned: list[Any] = [] + for entry in value: + if isinstance(entry, dict): + entry = dict(entry) + if "timestamp" in entry: + entry["timestamp"] = NORMALIZED + cleaned.append(entry) + return sorted(cleaned, key=lambda i: json.dumps(i, sort_keys=True, ensure_ascii=True)) + + +def normalize_snapshot(snapshot: ReplaySnapshot) -> ReplaySnapshot: + """返回归一化后的快照副本(不改原对象)。""" + out = snapshot.model_copy(deep=True) + out.events = [normalize_event(e) for e in out.events] + out.historical_events = [normalize_event(e) for e in out.historical_events] + # 剥离 temp: 临时状态(不持久化,比较时排除)。 + out.state = {k: v for k, v in out.state.items() if not k.startswith("temp:")} + # memory 检索结果:归一化 timestamp + 确定性排序。 + out.memory = {k: _normalize_memory_entries(v) for k, v in out.memory.items()} + return out diff --git a/tests/sessions/replay/replay_cases/cases.jsonl b/tests/sessions/replay/replay_cases/cases.jsonl new file mode 100644 index 00000000..0cc649f5 --- /dev/null +++ b/tests/sessions/replay/replay_cases/cases.jsonl @@ -0,0 +1,10 @@ +{"case_id":"single_turn","description":"单轮对话:user 输入 + agent 文本输出","operations":[{"op":"create_session","app_name":"replay-single","user_id":"u1","session_id":"sess-single"},{"op":"append_event","author":"user","text":"你好"},{"op":"append_event","author":"agent","text":"你好,有什么可以帮你?"}],"allowed_diff":[]} +{"case_id":"multi_turn","description":"多轮对话:连续追加 user / assistant event","operations":[{"op":"create_session","app_name":"replay-multi","user_id":"u1","session_id":"sess-multi"},{"op":"append_event","author":"user","text":"查天气"},{"op":"append_event","author":"agent","text":"哪个城市"},{"op":"append_event","author":"user","text":"北京"},{"op":"append_event","author":"agent","text":"北京晴"}],"allowed_diff":[]} +{"case_id":"tool_round_trip","description":"工具调用对话:function_call + function_response","operations":[{"op":"create_session","app_name":"replay-tool","user_id":"u1","session_id":"sess-tool"},{"op":"append_event","author":"user","text":"查北京天气"},{"op":"function_call","author":"agent","function_name":"get_weather","function_args":{"city":"北京"}},{"op":"function_response","author":"agent","function_name":"get_weather","function_response":{"temp":26,"cond":"晴"}}],"allowed_diff":[]} +{"case_id":"state_overwrite","description":"state 多次写入与覆盖","operations":[{"op":"create_session","app_name":"replay-state","user_id":"u1","session_id":"sess-state"},{"op":"update_state","author":"agent","text":"init","state_delta":{"counter":1}},{"op":"update_state","author":"agent","text":"incr","state_delta":{"counter":2}},{"op":"update_state","author":"agent","text":"overwrite","state_delta":{"counter":3,"flag":true}}],"allowed_diff":[]} +{"case_id":"memory_preference","description":"memory 写入与读取:用户偏好","operations":[{"op":"create_session","app_name":"replay-mem1","user_id":"u1","session_id":"sess-mem1"},{"op":"append_event","author":"user","text":"我喜欢安静的工作环境"},{"op":"memory_store"},{"op":"memory_search","memory_query":"安静"}],"allowed_diff":[]} +{"case_id":"memory_fact_update","description":"memory 事实更新:多次写入后检索","operations":[{"op":"create_session","app_name":"replay-mem2","user_id":"u1","session_id":"sess-mem2"},{"op":"append_event","author":"user","text":"我住在北京"},{"op":"memory_store"},{"op":"append_event","author":"user","text":"我搬到上海了"},{"op":"memory_store"},{"op":"memory_search","memory_query":"上海"}],"allowed_diff":[]} +{"case_id":"summary_create","description":"summary 生成:长对话触发摘要写入","operations":[{"op":"create_session","app_name":"replay-sum1","user_id":"u1","session_id":"sess-sum1"},{"op":"append_event","author":"user","text":"我们讨论旅行计划"},{"op":"append_event","author":"agent","text":"好的你想去哪里"},{"op":"append_event","author":"user","text":"想去日本"},{"op":"create_summary"}],"allowed_diff":[]} +{"case_id":"summary_update","description":"summary 更新:version/supersedes 关联","operations":[{"op":"create_session","app_name":"replay-sum2","user_id":"u1","session_id":"sess-sum2"},{"op":"append_event","author":"user","text":"聊东京"},{"op":"append_event","author":"agent","text":"东京不错"},{"op":"create_summary"},{"op":"append_event","author":"user","text":"改去大阪"},{"op":"append_event","author":"agent","text":"大阪也好"},{"op":"update_summary"}],"allowed_diff":[]} +{"case_id":"summary_truncation","description":"summary 与事件截断:历史压缩后保留 summary + 新事件","operations":[{"op":"create_session","app_name":"replay-trunc","user_id":"u1","session_id":"sess-trunc"},{"op":"append_event","author":"user","text":"早期话题一"},{"op":"append_event","author":"agent","text":"早期回复一"},{"op":"append_event","author":"user","text":"早期话题二"},{"op":"create_summary"},{"op":"append_event","author":"user","text":"压缩后的新问题"},{"op":"append_event","author":"agent","text":"新回复"}],"allowed_diff":[]} +{"case_id":"retry_recovery","description":"异常恢复:中途失败 + 重试,不应产生重复/脏事件","operations":[{"op":"create_session","app_name":"replay-retry","user_id":"u1","session_id":"sess-retry"},{"op":"append_event","author":"user","text":"第一条"},{"op":"fail_before_commit","author":"user","text":"失败的那条"},{"op":"retry_event","author":"user","text":"失败的那条"},{"op":"append_event","author":"agent","text":"成功"}],"allowed_diff":[]} diff --git a/tests/sessions/replay/report.py b/tests/sessions/replay/report.py new file mode 100644 index 00000000..4e01f808 --- /dev/null +++ b/tests/sessions/replay/report.py @@ -0,0 +1,122 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""schema_version=3 差异报告组装。 + +每条 diff 内联 session_id / event_index / summary_id / field_path / 双后端值; +``false_positive_rate`` 仅按正常 case 计算(注入 case 不计入)。 +单后端(轻量只剩 InMemory)时用 ``not_applicable`` 诚实标记,而非假 ``match``。 +""" + +from __future__ import annotations + +import json +from typing import Any +from typing import Literal + +from pydantic import BaseModel +from pydantic import Field + +from .comparator import DiffEntry +from .summary_checks import SummaryIssue + +CaseStatus = Literal["match", "mismatch", "not_applicable", "skipped"] + + +class BackendStatus(BaseModel): + name: str + status: CaseStatus + reason: str | None = None + + +class Comparison(BaseModel): + candidate_backend: str + status: CaseStatus + diffs: list[DiffEntry] = Field(default_factory=list) + summary_issues: list[SummaryIssue] = Field(default_factory=list) + + +class CaseResult(BaseModel): + case_id: str + session_id: str + comparisons: list[Comparison] = Field(default_factory=list) + + +def _roll_up_status(comparisons: list[Comparison]) -> CaseStatus: + """一个 case 跨所有候选后端的汇总状态。""" + if not comparisons: + return "not_applicable" + statuses = {c.status for c in comparisons} + if "mismatch" in statuses: + return "mismatch" + if statuses == {"skipped"}: + return "skipped" + if statuses <= {"not_applicable"}: + return "not_applicable" + return "match" + + +def _compared_backends(statuses: list[BackendStatus], case_results: list[CaseResult]) -> list[str]: + compared = [b.name for b in statuses if b.status != "skipped"] + if compared: + return compared + seen: list[str] = [] + for cr in case_results: + for c in cr.comparisons: + if c.candidate_backend not in seen: + seen.append(c.candidate_backend) + return seen + + +def build_diff_report( + reference_backend: str, + case_results: list[CaseResult], + backend_statuses: list[BackendStatus] | None = None, +) -> dict[str, Any]: + """组装差异报告 dict(可 json.dump)。""" + statuses = backend_statuses or [] + totals = { + "cases": len(case_results), + "matched": 0, + "mismatched": 0, + "not_applicable": 0, + "skipped": 0, + } + cases_out: list[dict[str, Any]] = [] + normal_mismatch = 0 + for cr in case_results: + st = _roll_up_status(cr.comparisons) + if st == "match": + totals["matched"] += 1 + elif st == "mismatch": + totals["mismatched"] += 1 + normal_mismatch += 1 + elif st == "not_applicable": + totals["not_applicable"] += 1 + elif st == "skipped": + totals["skipped"] += 1 + cases_out.append({ + "case_id": cr.case_id, + "session_id": cr.session_id, + "status": st, + "comparisons": [c.model_dump() for c in cr.comparisons], + }) + + fpr = (normal_mismatch / len(case_results)) if case_results else 0.0 + return { + "schema_version": 3, + "reference_backend": reference_backend, + "compared_backends": _compared_backends(statuses, case_results), + "backend_statuses": [b.model_dump() for b in statuses], + "totals": totals, + "false_positive_rate": fpr, + "cases": cases_out, + } + + +def write_report(report: dict[str, Any], path: str) -> None: + """把报告写入 JSON 文件(仓库根 session_memory_summary_diff_report.json)。""" + with open(path, "w", encoding="utf-8") as f: + json.dump(report, f, ensure_ascii=False, indent=2) diff --git a/tests/sessions/replay/summary_checks.py b/tests/sessions/replay/summary_checks.py new file mode 100644 index 00000000..9135a37d --- /dev/null +++ b/tests/sessions/replay/summary_checks.py @@ -0,0 +1,98 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""summary 专项检测:loss / overwrite / affiliation 三类故障 + 文本语义相似度。 + +三分比较里的「内容语义」走分词集合 Jaccard(纯标准库,无 embedding 依赖); +「存储元数据」(version / session_id)严格相等,三类故障由本模块显式检测。 +""" + +from __future__ import annotations + +import re +from typing import Any +from typing import Literal + +from pydantic import BaseModel + +SUMMARY_SIM_THRESHOLD = 0.8 +"""summary 文本 Jaccard 相似度阈值,之上判内容一致。""" + + +class SummaryIssue(BaseModel): + type: Literal["loss", "overwrite", "affiliation"] + session_id: str + summary_id: str | None = None + detail: dict[str, Any] + + +def _tokenize(text: str) -> list[str]: + return re.findall(r"\w+", text.lower()) + + +def summary_text_similarity(a: str | None, b: str | None) -> float: + """分词集合 Jaccard 相似度。任一空串返回 0.0。""" + if not a or not b: + return 0.0 + ta = set(_tokenize(a)) + tb = set(_tokenize(b)) + if not ta or not tb: + return 0.0 + return len(ta & tb) / len(ta | tb) + + +def check_summary_issues( + reference_summary: dict[str, Any], + candidate_summary: dict[str, Any], + *, + candidate_backend: str, + session_id: str, +) -> list[SummaryIssue]: + """检测 summary 三类故障:loss / overwrite(version 倒退)/ affiliation(session 归属错)。""" + issues: list[SummaryIssue] = [] + ref_cur = reference_summary.get("current") if reference_summary else None + cand_cur = candidate_summary.get("current") if candidate_summary else None + + # loss:参考端有 summary,候选端丢失。 + if ref_cur and not cand_cur: + issues.append(SummaryIssue(type="loss", session_id=session_id, detail={"backend": candidate_backend})) + return issues + + if not (ref_cur and cand_cur): + return issues + + # overwrite:候选 version 倒退(旧版覆盖新版)。 + ref_ver = ref_cur.get("version") + cand_ver = cand_cur.get("version") + if ref_ver is not None and cand_ver is not None and cand_ver < ref_ver: + issues.append( + SummaryIssue( + type="overwrite", + session_id=session_id, + summary_id=cand_cur.get("id"), + detail={ + "ref_version": ref_ver, + "cand_version": cand_ver, + "backend": candidate_backend, + }, + )) + + # affiliation:summary 归属 session 错误。 + ref_sid = ref_cur.get("session_id") + cand_sid = cand_cur.get("session_id") + if ref_sid and cand_sid and ref_sid != cand_sid: + issues.append( + SummaryIssue( + type="affiliation", + session_id=session_id, + summary_id=cand_cur.get("id"), + detail={ + "ref_session": ref_sid, + "cand_session": cand_sid, + "backend": candidate_backend, + }, + )) + + return issues diff --git a/tests/sessions/test_replay_consistency.py b/tests/sessions/test_replay_consistency.py new file mode 100644 index 00000000..7cc0ea52 --- /dev/null +++ b/tests/sessions/test_replay_consistency.py @@ -0,0 +1,156 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Replay 一致性 E2E:同一组 case 驱动多后端,比较事件/state/memory/summary。 + +轻量模式默认 InMemory vs SQLite(:memory:);Redis 经 TRPC_REPLAY_REDIS_URL 启用。 +报告产物:仓库根 session_memory_summary_diff_report.json。 +""" + +from __future__ import annotations + +import time +from pathlib import Path + +from tests.sessions.replay.backends import enabled_backends +from tests.sessions.replay.backends import in_memory_backend +from tests.sessions.replay.backends import sqlite_backend +from tests.sessions.replay.comparator import compare_snapshots +from tests.sessions.replay.harness import load_cases +from tests.sessions.replay.harness import replay_case +from tests.sessions.replay.normalizer import normalize_snapshot +from tests.sessions.replay.report import CaseResult +from tests.sessions.replay.report import Comparison +from tests.sessions.replay.report import build_diff_report +from tests.sessions.replay.report import write_report +from tests.sessions.replay.summary_checks import check_summary_issues + +CASES_DIR = str(Path(__file__).parent / "replay" / "replay_cases") +REPORT_PATH = str(Path(__file__).parents[2] / "session_memory_summary_diff_report.json") +LIGHTWEIGHT_TIMEOUT = 30 # 验收第 6 条:轻量模式 ≤30s + +KNOWN_DRIFT = {"summary_update", "summary_truncation"} +"""已知 SQLite summary 持久化漂移:``create_session_summary`` 后 SQLite ``get_session`` +读回的 events 顺序 / historical_events / summary 与 InMemory 不一致(类 issue #163 的 +summarizer 锚点 timestamp 问题)。框架正确发现,按设计 §8「只报告不改」记录, +不计入误报率分母,修 bug 另开 issue/PR。""" + + +def _find(case_id: str): + for c in load_cases(CASES_DIR): + if c.case_id == case_id: + return c + raise AssertionError(f"case not found: {case_id}") + + +# --------------------------------------------------------------------------- +# 冒烟:replay_case 驱动单后端 +# --------------------------------------------------------------------------- + + +class TestReplaySmoke: + + async def test_single_turn_in_memory(self): + snap = await replay_case(in_memory_backend(), _find("single_turn")) + assert snap.session_id == "sess-single" + assert len(snap.events) >= 2 + + async def test_state_overwrite_cross_backend(self): + case = _find("state_overwrite") + snap_im = await replay_case(in_memory_backend(), case) + snap_sql = await replay_case(sqlite_backend(), case) + assert snap_im.state.get("counter") == 3 + assert snap_im.state == snap_sql.state + + async def test_multi_turn_in_memory_vs_sqlite_no_diff(self): + case = _find("multi_turn") + snap_im = normalize_snapshot(await replay_case(in_memory_backend(), case)) + snap_sql = normalize_snapshot(await replay_case(sqlite_backend(), case)) + diffs = compare_snapshots( + snap_im, + snap_sql, + reference_backend="in_memory", + candidate_backend="sqlite", + allowed_diff=case.allowed_diff, + ) + assert [d for d in diffs if not d.allowed] == [] + + +# --------------------------------------------------------------------------- +# 主 E2E:全 case × 多后端 + 报告(验收 1/3/5/6) +# --------------------------------------------------------------------------- + + +class TestReplayConsistencyE2E: + + async def test_all_cases_cross_backend(self): + start = time.time() + cases = load_cases(CASES_DIR) + backends, statuses = enabled_backends() + reference = backends[0] + candidates = backends[1:] + + case_results: list[CaseResult] = [] + for case in cases: + snap_ref = normalize_snapshot(await replay_case(reference, case)) + comparisons: list[Comparison] = [] + for cand in candidates: + snap_cand = normalize_snapshot(await replay_case(cand, case)) + diffs = compare_snapshots( + snap_ref, + snap_cand, + reference_backend=reference.name, + candidate_backend=cand.name, + allowed_diff=case.allowed_diff, + ) + issues = check_summary_issues( + snap_ref.summary, + snap_cand.summary, + candidate_backend=cand.name, + session_id=snap_ref.session_id, + ) + real = [d for d in diffs if not d.allowed] + status = "mismatch" if (real or issues) else "match" + comparisons.append( + Comparison( + candidate_backend=cand.name, + status=status, + diffs=diffs, + summary_issues=issues, + )) + case_results.append( + CaseResult(case_id=case.case_id, session_id=snap_ref.session_id, comparisons=comparisons)) + + # 误报率只算「正常 case」(排除已知 drift case —— 后者是框架发现的真 bug)。 + normal = [cr for cr in case_results if cr.case_id not in KNOWN_DRIFT] + normal_mismatch = sum(1 for cr in normal if any(c.status == "mismatch" for c in cr.comparisons)) + fpr = normal_mismatch / len(normal) if normal else 0.0 + + report = build_diff_report(reference.name, case_results, statuses) + report["false_positive_rate"] = fpr + report["known_drift_cases"] = sorted(KNOWN_DRIFT) + write_report(report, REPORT_PATH) + + # 验收 1:InMemory + 持久化(SQLite)对比。 + assert "sqlite" in report["compared_backends"] + # 验收 3:正常 case 误报率 0。 + bad = [cr.case_id for cr in normal if any(c.status == "mismatch" for c in cr.comparisons)] + assert fpr == 0.0, f"normal-case FPR>0: {bad}" + for cr in normal: + for comp in cr.comparisons: + assert comp.status == "match", (f"unexpected mismatch in {cr.case_id}: " + f"{[d.field_path for d in comp.diffs if not d.allowed]}") + # 已知 drift case:框架应检出 SQLite 漂移(这正是框架的价值)。 + for cr in case_results: + if cr.case_id in KNOWN_DRIFT: + sqlite_comp = [c for c in cr.comparisons if c.candidate_backend == "sqlite"] + assert sqlite_comp and sqlite_comp[0].status == "mismatch", ( + f"{cr.case_id} should be detected as drift") + # 验收 5:报告 schema。 + assert report["schema_version"] == 3 + assert report["totals"]["cases"] == len(cases) + # 验收 6:轻量模式 ≤30s。 + elapsed = time.time() - start + assert elapsed < LIGHTWEIGHT_TIMEOUT, f"lightweight too slow: {elapsed:.1f}s" diff --git a/tests/sessions/test_replay_injections.py b/tests/sessions/test_replay_injections.py new file mode 100644 index 00000000..af568f03 --- /dev/null +++ b/tests/sessions/test_replay_injections.py @@ -0,0 +1,200 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""检出验证:快照层注入(对齐 10 PR)+ 端到端后端注入(本设计创新)。 + +- 快照层:deepcopy 改字段,验证比较器 + summary_checks 检出率(8 种,覆盖 event/ + state/memory/summary 四类)。 +- 端到端:直接改 SQL 行 / Redis key 后重读,验证 harness 对真实后端漂移的感知。 +对应验收第 2 条(100% 检出)与第 4 条(summary 三类 100% 检出)。 +""" + +from __future__ import annotations + +import os + +import pytest + +from tests.sessions.replay.backends import in_memory_backend +from tests.sessions.replay.backends import sqlite_backend +from tests.sessions.replay.comparator import compare_snapshots +from tests.sessions.replay.harness import ReplaySnapshot +from tests.sessions.replay.harness import load_cases +from tests.sessions.replay.harness import replay_case +from tests.sessions.replay.injectors import inject_redis_diff +from tests.sessions.replay.injectors import inject_snapshot_diff +from tests.sessions.replay.injectors import inject_sql_diff +from tests.sessions.replay.normalizer import normalize_snapshot +from tests.sessions.replay.summary_checks import check_summary_issues + +CASES_DIR = "tests/sessions/replay/replay_cases" + +# 验收 2(字面):每条 case 配一种「该 case 结构支持」的注入,断言 100% 检出。 +_CASE_INJECTION = { + "single_turn": "event_author", + "multi_turn": "event_text", + "tool_round_trip": "event_author", + "state_overwrite": "state_value", + "memory_preference": "memory_content", + "memory_fact_update": "memory_content", + "summary_create": "summary_affiliation", + "summary_update": "summary_overwrite", + "summary_truncation": "extra_event", + "retry_recovery": "extra_event", +} + + +def _find(case_id: str): + for c in load_cases(CASES_DIR): + if c.case_id == case_id: + return c + raise AssertionError(f"case not found: {case_id}") + + +def _detected(base: ReplaySnapshot, injected: ReplaySnapshot) -> bool: + diffs = compare_snapshots( + base, + injected, + reference_backend="in_memory", + candidate_backend="in_memory", + allowed_diff=[], + ) + issues = check_summary_issues(base.summary, + injected.summary, + candidate_backend="in_memory", + session_id=base.session_id) + return bool([d for d in diffs if not d.allowed]) or bool(issues) + + +# --------------------------------------------------------------------------- +# 快照层注入:8 种 kind 全检出(验收 2 + 4) +# --------------------------------------------------------------------------- + + +class TestSnapshotInjection: + + async def test_all_eight_kinds_detected(self): + base_map = { + "single_turn": normalize_snapshot(await replay_case(in_memory_backend(), _find("single_turn"))), + "memory_preference": normalize_snapshot(await replay_case(in_memory_backend(), _find("memory_preference"))), + "summary_create": normalize_snapshot(await replay_case(in_memory_backend(), _find("summary_create"))), + "state_overwrite": normalize_snapshot(await replay_case(in_memory_backend(), _find("state_overwrite"))), + } + # (case_id, kind) —— kind 挂在结构匹配的快照上。 + plan = [ + ("single_turn", "event_author"), + ("single_turn", "event_text"), + ("single_turn", "extra_event"), + ("state_overwrite", "state_value"), + ("memory_preference", "memory_content"), + ("summary_create", "summary_loss"), + ("summary_create", "summary_overwrite"), + ("summary_create", "summary_affiliation"), + ] + not_detected = [ + f"{cid}/{kind}" for cid, kind in plan + if not _detected(base_map[cid], inject_snapshot_diff(base_map[cid], kind)) + ] + assert not_detected == [], f"injections not detected: {not_detected}" + + async def test_each_case_detects_injection(self): + """验收 2(字面):10 条 case 各注入一种不一致,必须 100% 检出。""" + not_detected = [] + for case in load_cases(CASES_DIR): + kind = _CASE_INJECTION.get(case.case_id) + if kind is None: + continue + base = normalize_snapshot(await replay_case(in_memory_backend(), case)) + if not _detected(base, inject_snapshot_diff(base, kind)): + not_detected.append(f"{case.case_id}/{kind}") + assert not_detected == [], f"injections not detected: {not_detected}" + + +# --------------------------------------------------------------------------- +# 端到端 SQL 注入(创新:验证真实后端漂移感知) +# --------------------------------------------------------------------------- + + +async def _read_sql_snapshot(db_url: str, app: str, user: str, sid: str) -> ReplaySnapshot: + """用全新 service 读 DB 文件(绕过缓存),组装快照。""" + backend = sqlite_backend(db_url) + got = await backend.session_service.get_session(app_name=app, user_id=user, session_id=sid) + return ReplaySnapshot( + backend_name="sqlite", + session_id=sid, + events=[e.model_dump() for e in got.events], + historical_events=[e.model_dump() for e in got.historical_events], + state=dict(got.state), + ) + + +class TestEndToEndSqlInjection: + + async def test_event_author_drift_detected(self, tmp_path): + db_url = f"sqlite:///{tmp_path.as_posix()}/inj.db" + case = _find("single_turn") + await replay_case(sqlite_backend(db_url), case) + + before = normalize_snapshot(await _read_sql_snapshot(db_url, "replay-single", "u1", "sess-single")) + assert inject_sql_diff(db_url, "replay-single", "u1", "sess-single", "event_author") + after = normalize_snapshot(await _read_sql_snapshot(db_url, "replay-single", "u1", "sess-single")) + + diffs = compare_snapshots( + before, + after, + reference_backend="sqlite", + candidate_backend="sqlite", + allowed_diff=[], + ) + real = [d for d in diffs if not d.allowed] + assert any("author" in d.field_path for d in real), f"author drift not detected: {real}" + + +# --------------------------------------------------------------------------- +# 端到端 Redis 注入(需要真实 Redis) +# --------------------------------------------------------------------------- + + +class TestEndToEndRedisInjection: + + async def test_event_author_drift_detected(self): + redis_url = os.environ.get("TRPC_REPLAY_REDIS_URL") + if not redis_url: + pytest.skip("TRPC_REPLAY_REDIS_URL unset") + from tests.sessions.replay.backends import redis_backend + + case = _find("single_turn") + backend = redis_backend(redis_url) + await replay_case(backend, case) + + got = await backend.session_service.get_session(app_name="replay-single", + user_id="u1", + session_id="sess-single") + before = normalize_snapshot( + ReplaySnapshot( + backend_name="redis", + session_id="sess-single", + events=[e.model_dump() for e in got.events], + )) + assert inject_redis_diff(redis_url, "replay-single", "u1", "sess-single", "event_author") + got2 = await backend.session_service.get_session(app_name="replay-single", + user_id="u1", + session_id="sess-single") + after = normalize_snapshot( + ReplaySnapshot( + backend_name="redis", + session_id="sess-single", + events=[e.model_dump() for e in got2.events], + )) + + diffs = compare_snapshots( + before, + after, + reference_backend="redis", + candidate_backend="redis", + allowed_diff=[], + ) + real = [d for d in diffs if not d.allowed] + assert any("author" in d.field_path for d in real), f"redis drift not detected: {real}" diff --git a/tests/sessions/test_replay_unit.py b/tests/sessions/test_replay_unit.py new file mode 100644 index 00000000..75c7acbc --- /dev/null +++ b/tests/sessions/test_replay_unit.py @@ -0,0 +1,309 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Replay harness 单元测试:模型 / normalizer / comparator / allowed_diff / summary_checks / report。""" + +from __future__ import annotations + +import pytest + +from tests.sessions.replay.allowed_diff import MAX_ALLOWED_PER_CASE +from tests.sessions.replay.allowed_diff import check_governance +from tests.sessions.replay.allowed_diff import is_allowed +from tests.sessions.replay.comparator import MISSING +from tests.sessions.replay.comparator import DiffEntry +from tests.sessions.replay.comparator import compare_snapshots +from tests.sessions.replay.harness import AllowedDiffRule +from tests.sessions.replay.harness import ReplayCase +from tests.sessions.replay.harness import ReplayOp +from tests.sessions.replay.harness import ReplaySnapshot +from tests.sessions.replay.normalizer import NORMALIZED +from tests.sessions.replay.normalizer import normalize_event +from tests.sessions.replay.normalizer import normalize_snapshot +from tests.sessions.replay.report import BackendStatus +from tests.sessions.replay.report import CaseResult +from tests.sessions.replay.report import Comparison +from tests.sessions.replay.report import build_diff_report +from tests.sessions.replay.summary_checks import check_summary_issues +from tests.sessions.replay.summary_checks import summary_text_similarity + +# --------------------------------------------------------------------------- +# Task 2: 数据模型 +# --------------------------------------------------------------------------- + + +class TestReplayModels: + + def test_replay_case_roundtrip(self): + case = ReplayCase( + case_id="single_turn", + description="one turn", + operations=[ + ReplayOp(op="create_session", app_name="a", user_id="u", session_id="s"), + ReplayOp(op="append_event", author="user", text="hi"), + ], + allowed_diff=[AllowedDiffRule(path="events[*].timestamp", reason="auto")], + ) + back = ReplayCase.model_validate_json(case.model_dump_json()) + assert back.case_id == "single_turn" + assert back.operations[0].op == "create_session" + assert back.operations[1].text == "hi" + assert back.allowed_diff[0].reason == "auto" + + def test_replay_op_rejects_unknown_field(self): + with pytest.raises(Exception): + ReplayOp(op="append_event", not_a_field=1) + + +def _snapshot(**kw) -> ReplaySnapshot: + return ReplaySnapshot(session_id="s1", **kw) + + +def _diff(**kw) -> DiffEntry: + return DiffEntry( + field_path="events[0].author", + reference_backend="in_memory", + candidate_backend="sqlite", + reference_value="user", + candidate_value="assistant", + **kw, + ) + + +# --------------------------------------------------------------------------- +# Task 3: normalizer +# --------------------------------------------------------------------------- + + +class TestNormalizer: + + def test_normalize_event_replaces_volatile_fields(self): + e = normalize_event({"id": "u1", "timestamp": 1.2, "invocation_id": "i", "author": "user"}) + assert e["id"] == NORMALIZED + assert e["timestamp"] == NORMALIZED + assert e["invocation_id"] == NORMALIZED + assert e["author"] == "user" + + def test_normalize_strips_temp_state(self): + snap = normalize_snapshot(_snapshot(state={"app:x": 1, "temp:skip": 2, "plain": 3})) + assert "temp:skip" not in snap.state + assert snap.state["app:x"] == 1 + assert snap.state["plain"] == 3 + + def test_normalize_sorts_memory(self): + snap = normalize_snapshot(_snapshot(memory={"q": [{"b": 2}, {"a": 1}, {"c": 3}]})) + keys = [list(m.keys())[0] for m in snap.memory["q"]] + assert keys == ["a", "b", "c"] + + +# --------------------------------------------------------------------------- +# Task 4: comparator +# --------------------------------------------------------------------------- + + +class TestComparator: + + def test_diff_detects_leaf_mismatch(self): + ref = _snapshot(events=[{"author": "user", "content": {"parts": [{"text": "hi"}]}}]) + cand = _snapshot(events=[{"author": "assistant", "content": {"parts": [{"text": "hi"}]}}]) + diffs = compare_snapshots(ref, cand, reference_backend="in_memory", candidate_backend="sqlite", allowed_diff=[]) + assert len(diffs) == 1 + d = diffs[0] + assert d.field_path == "events[0].author" + assert d.event_index == 0 + assert d.reference_value == "user" + assert d.candidate_value == "assistant" + assert d.allowed is False + + def test_diff_aligns_dict_sorted_keys(self): + ref = _snapshot(state={"b": 1, "a": 2}) + cand = _snapshot(state={"a": 2, "b": 1}) + assert compare_snapshots(ref, cand, reference_backend="in_memory", candidate_backend="sqlite", + allowed_diff=[]) == [] + + def test_diff_list_length_diff(self): + ref = _snapshot(events=[{"author": "a"}, {"author": "b"}, {"author": "c"}]) + cand = _snapshot(events=[{"author": "a"}, {"author": "b"}]) + diffs = compare_snapshots(ref, cand, reference_backend="in_memory", candidate_backend="sqlite", allowed_diff=[]) + assert len(diffs) == 1 + assert diffs[0].field_path == "events[2]" + assert diffs[0].event_index == 2 + assert diffs[0].candidate_value == MISSING + + def test_diff_marks_allowed(self): + ref = _snapshot(events=[{"timestamp": 1.0, "author": "user"}]) + cand = _snapshot(events=[{"timestamp": 2.0, "author": "user"}]) + rule = AllowedDiffRule(path="events[*].timestamp", reason="backend-assigned") + diffs = compare_snapshots(ref, + cand, + reference_backend="in_memory", + candidate_backend="sqlite", + allowed_diff=[rule]) + assert len(diffs) == 1 + assert diffs[0].allowed is True + assert diffs[0].reason == "backend-assigned" + + +# --------------------------------------------------------------------------- +# Task 5: allowed_diff +# --------------------------------------------------------------------------- + + +class TestAllowedDiff: + + def test_exact_path_match(self): + rule = AllowedDiffRule(path="events[0].timestamp", reason="auto") + assert is_allowed("events[0].timestamp", ("in_memory", "sqlite"), [rule])[0] is True + assert is_allowed("events[0].author", ("in_memory", "sqlite"), [rule])[0] is False + + def test_index_wildcard(self): + rule = AllowedDiffRule(path="events[*].timestamp", reason="auto") + assert is_allowed("events[0].timestamp", ("in_memory", "sqlite"), [rule])[0] is True + assert is_allowed("events[5].timestamp", ("in_memory", "sqlite"), [rule])[0] is True + + def test_backend_pair_filter(self): + rule = AllowedDiffRule(path="events[*].timestamp", reason="auto", backend_pair=("in_memory", "redis")) + assert is_allowed("events[0].timestamp", ("in_memory", "redis"), [rule])[0] is True + assert is_allowed("events[0].timestamp", ("in_memory", "sqlite"), [rule])[0] is False + + def test_governance_rejects_too_many(self): + rules = [AllowedDiffRule(path=f"events[{i}].timestamp", reason="r") for i in range(MAX_ALLOWED_PER_CASE + 1)] + case = ReplayCase(case_id="x", description="d", allowed_diff=rules) + with pytest.raises(ValueError): + check_governance(case, total_fields=100, used_allowed=0) + + def test_governance_rejects_ratio(self): + case = ReplayCase(case_id="x", description="d") + with pytest.raises(ValueError): + check_governance(case, total_fields=20, used_allowed=5) + + def test_governance_rejects_no_reason(self): + rule = AllowedDiffRule(path="events[0].timestamp", reason="") + case = ReplayCase(case_id="x", description="d", allowed_diff=[rule]) + with pytest.raises(ValueError): + check_governance(case, total_fields=100, used_allowed=0) + + +# --------------------------------------------------------------------------- +# Task 6: summary_checks +# --------------------------------------------------------------------------- + + +class TestSummaryChecks: + + def test_detects_loss(self): + ref = {"current": {"text": "s", "version": 1, "session_id": "s1"}} + issues = check_summary_issues(ref, {"current": None}, candidate_backend="sqlite", session_id="s1") + assert any(i.type == "loss" for i in issues) + + def test_detects_overwrite(self): + ref = {"current": {"text": "s", "version": 3, "session_id": "s1"}} + cand = {"current": {"text": "s", "version": 1, "session_id": "s1"}} + issues = check_summary_issues(ref, cand, candidate_backend="sqlite", session_id="s1") + assert any(i.type == "overwrite" for i in issues) + + def test_detects_affiliation(self): + ref = {"current": {"text": "s", "version": 1, "session_id": "s1"}} + cand = {"current": {"text": "s", "version": 1, "session_id": "replay-wrong-session"}} + issues = check_summary_issues(ref, cand, candidate_backend="sqlite", session_id="s1") + assert any(i.type == "affiliation" for i in issues) + + def test_no_issue_when_consistent(self): + ref = {"current": {"text": "s", "version": 1, "session_id": "s1"}} + assert check_summary_issues(ref, dict(ref), candidate_backend="sqlite", session_id="s1") == [] + + def test_semantic_similarity(self): + assert summary_text_similarity("hello world foo", "foo world hello") == 1.0 + assert summary_text_similarity("aaa", "zzz") == 0.0 + assert summary_text_similarity(None, "x") == 0.0 + + +# --------------------------------------------------------------------------- +# Task 9: report +# --------------------------------------------------------------------------- + + +class TestReport: + + def test_report_schema_and_totals(self): + results = [ + CaseResult( + case_id="c1", + session_id="s1", + comparisons=[Comparison(candidate_backend="sqlite", status="match")], + ), + CaseResult( + case_id="c2", + session_id="s2", + comparisons=[Comparison(candidate_backend="sqlite", status="mismatch", diffs=[_diff()])], + ), + ] + report = build_diff_report( + "in_memory", + results, + backend_statuses=[ + BackendStatus(name="sqlite", status="match"), + BackendStatus(name="redis", status="skipped", reason="no url"), + ], + ) + assert report["schema_version"] == 3 + assert report["reference_backend"] == "in_memory" + assert report["compared_backends"] == ["sqlite"] + assert report["totals"] == { + "cases": 2, + "matched": 1, + "mismatched": 1, + "not_applicable": 0, + "skipped": 0, + } + assert report["false_positive_rate"] == 0.5 + redis_status = [b for b in report["backend_statuses"] if b["name"] == "redis"][0] + assert redis_status["reason"] == "no url" + + def test_report_locates_diff(self): + results = [ + CaseResult( + case_id="c1", + session_id="s1", + comparisons=[ + Comparison( + candidate_backend="sqlite", + status="mismatch", + diffs=[ + DiffEntry( + session_id="s1", + event_index=0, + summary_id=None, + field_path="events[0].author", + reference_backend="in_memory", + candidate_backend="sqlite", + reference_value="user", + candidate_value="assistant", + ) + ], + ) + ], + ) + ] + report = build_diff_report("in_memory", results) + diff = report["cases"][0]["comparisons"][0]["diffs"][0] + for key in ( + "session_id", + "event_index", + "summary_id", + "field_path", + "reference_backend", + "candidate_backend", + "reference_value", + "candidate_value", + ): + assert key in diff + + def test_report_not_applicable_single_backend(self): + # 无 comparison 不误报 match(诚实标记,#153 vs #163) + results = [CaseResult(case_id="c1", session_id="s1", comparisons=[])] + report = build_diff_report("in_memory", results) + assert report["cases"][0]["status"] == "not_applicable" + assert report["totals"]["not_applicable"] == 1 From 189eeff543c1317950cf0a725e3ed5bacc7d3933 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=B0=E5=B0=BC=E9=BE=9F?= <3098584968@qq.com> Date: Tue, 14 Jul 2026 23:26:48 +0800 Subject: [PATCH 2/2] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=20eval=5Foptimiz?= =?UTF-8?q?e=5Floop=20=E8=AF=84=E6=B5=8B-=E4=BC=98=E5=8C=96=E8=87=AA?= =?UTF-8?q?=E5=8A=A8=E9=97=AD=E7=8E=AF=20example=20(issue=20#91)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 构建「评测→失败归因→prompt优化→验证集回归→接受决策→审计落盘」可复现闭环： - 分层失败归因（规则快通道 + 反事实深归因，本地 metric 零成本） - 过拟合三重检测（显式公式 + 泛化缺口 + 趋势背离） - 可配置 gate（三态 accept/reject/needs_review） - sha256 审计 + 三态成本 fail-closed - fake/trace/online 三模式，fake 无 API key 可跑通（<1s），9 pytest 全过 --- .../optimization/eval_optimize_loop/DESIGN.md | 593 ++++++++++++ .../optimization/eval_optimize_loop/README.md | 147 +++ .../eval_optimize_loop/agent/__init__.py | 1 + .../eval_optimize_loop/agent/agent.py | 73 ++ .../eval_optimize_loop/agent/config.py | 26 + .../agent/prompts/candidate_ineffective.md | 1 + .../agent/prompts/candidate_overfit.md | 11 + .../agent/prompts/candidate_robust.md | 13 + .../agent/prompts/system.md | 1 + .../eval_optimize_loop/agent/tools.py | 32 + .../data/train.evalset.json | 49 + .../eval_optimize_loop/data/val.evalset.json | 43 + .../optimization/eval_optimize_loop/gate.json | 16 + .../eval_optimize_loop/offline/__init__.py | 1 + .../eval_optimize_loop/offline/fixtures.py | 291 ++++++ .../eval_optimize_loop/optimizer.json | 50 + .../eval_optimize_loop/pipeline/__init__.py | 1 + .../pipeline/attribution.py | 192 ++++ .../eval_optimize_loop/pipeline/comparator.py | 77 ++ .../eval_optimize_loop/pipeline/config.py | 72 ++ .../eval_optimize_loop/pipeline/evaluator.py | 58 ++ .../eval_optimize_loop/pipeline/gate.py | 166 ++++ .../eval_optimize_loop/pipeline/models.py | 193 ++++ .../eval_optimize_loop/pipeline/reporting.py | 172 ++++ .../eval_optimize_loop/run_pipeline.py | 242 +++++ .../audit/environment.snapshot.json | 7 + .../sample_output/audit/gate_decisions.json | 262 ++++++ .../sample_output/audit/input.snapshot.json | 8 + .../sample_output/audit/proposals.json | 23 + .../sample_output/optimization_report.json | 872 ++++++++++++++++++ .../sample_output/optimization_report.md | 64 ++ .../tests/test_eval_optimize_loop.py | 255 +++++ 32 files changed, 4012 insertions(+) create mode 100644 examples/optimization/eval_optimize_loop/DESIGN.md create mode 100644 examples/optimization/eval_optimize_loop/README.md create mode 100644 examples/optimization/eval_optimize_loop/agent/__init__.py create mode 100644 examples/optimization/eval_optimize_loop/agent/agent.py create mode 100644 examples/optimization/eval_optimize_loop/agent/config.py create mode 100644 examples/optimization/eval_optimize_loop/agent/prompts/candidate_ineffective.md create mode 100644 examples/optimization/eval_optimize_loop/agent/prompts/candidate_overfit.md create mode 100644 examples/optimization/eval_optimize_loop/agent/prompts/candidate_robust.md create mode 100644 examples/optimization/eval_optimize_loop/agent/prompts/system.md create mode 100644 examples/optimization/eval_optimize_loop/agent/tools.py create mode 100644 examples/optimization/eval_optimize_loop/data/train.evalset.json create mode 100644 examples/optimization/eval_optimize_loop/data/val.evalset.json create mode 100644 examples/optimization/eval_optimize_loop/gate.json create mode 100644 examples/optimization/eval_optimize_loop/offline/__init__.py create mode 100644 examples/optimization/eval_optimize_loop/offline/fixtures.py create mode 100644 examples/optimization/eval_optimize_loop/optimizer.json create mode 100644 examples/optimization/eval_optimize_loop/pipeline/__init__.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/attribution.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/comparator.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/config.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/evaluator.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/gate.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/models.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/reporting.py create mode 100644 examples/optimization/eval_optimize_loop/run_pipeline.py create mode 100644 examples/optimization/eval_optimize_loop/sample_output/audit/environment.snapshot.json create mode 100644 examples/optimization/eval_optimize_loop/sample_output/audit/gate_decisions.json create mode 100644 examples/optimization/eval_optimize_loop/sample_output/audit/input.snapshot.json create mode 100644 examples/optimization/eval_optimize_loop/sample_output/audit/proposals.json create mode 100644 examples/optimization/eval_optimize_loop/sample_output/optimization_report.json create mode 100644 examples/optimization/eval_optimize_loop/sample_output/optimization_report.md create mode 100644 examples/optimization/eval_optimize_loop/tests/test_eval_optimize_loop.py diff --git a/examples/optimization/eval_optimize_loop/DESIGN.md b/examples/optimization/eval_optimize_loop/DESIGN.md new file mode 100644 index 00000000..9ddbc0c5 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/DESIGN.md @@ -0,0 +1,593 @@ +# Evaluation + Optimization 自动闭环 Pipeline 设计文档 + +> 对应 [issue #91](https://github.com/trpc-group/trpc-agent-python/issues/91) +> 主题：构建「评测 → 失败归因 → prompt 优化 → 验证集回归 → 接受决策 → 审计落盘」的自动闭环 +> 分支：`feat/eval-optimization-loop-91` + +--- + +## 0. 文档定位 + +本文是实现的**架构契约**：定义闭环的边界、各阶段职责、关键数据结构与决策算法。 +读完本文应能回答：每个文件放什么、issue 的 6 条验收标准分别落在哪段代码、为什么这样取舍。 + +本文**不是**最终 README。README 面向使用者（怎么跑），DESIGN 面向评审者（为什么这么设计）。 +issue 要求的「300–500 字方案设计说明」将从本文 §4 浓缩进 README。 + +--- + +## 1. 背景与目标 + +### 1.1 issue 核心诉求 + +把一次 prompt 优化从「分数变高了」升级为**可审计的发布决策**。核心不是再跑一次 `AgentOptimizer`， +而是回答四个问题： + +1. 优化**真的**提升了吗？（不能只信优化器自报的聚合分） +2. 提升是否**牺牲了其他指标**？（验证集逐 case 回归） +3. 是否**过拟合**？（train 升 val 降） +4. 改出来的 prompt**值得回写源文件**吗？（接受/拒绝 + 理由） + +### 1.2 与现有 SDK 能力的关系：编排层，不是重写 + +SDK 已提供的能力（**复用，不重写**）： + +| SDK 能力 | 位置 | 在闭环中的角色 | +|---|---|---| +| `AgentOptimizer.optimize()` | [_agent_optimizer.py:132](../../../trpc_agent_sdk/evaluation/_agent_optimizer.py#L132) | 优化引擎，内部已做 train/val 双集 + GEPA 反思 + 早停 | +| `OptimizeResult` / `RoundRecord` | [_optimize_result.py](../../../trpc_agent_sdk/evaluation/_optimize_result.py) | 审计原料（baseline/best 分数、每轮候选、cost、duration、seed）| +| `AgentEvaluator` (trace-only) | [_agent_evaluator.py](../../../trpc_agent_sdk/evaluation/_agent_evaluator.py) | 候选独立复评（不信任优化器聚合分）| +| `TargetPrompt` | [_target_prompt.py](../../../trpc_agent_sdk/evaluation/_target_prompt.py) | prompt 源注册 + 回写 | +| `eval_mode="trace"` | [_eval_case.py:170](../../../trpc_agent_sdk/evaluation/_eval_case.py#L170) | 无 API key 回放 | +| `LLM_EVALUATOR_REGISTRY` | [_llm_evaluator.py:178](../../../trpc_agent_sdk/evaluation/_llm_evaluator.py#L178) | fake judge 注入点 | +| `call_agent` 参数 | `AgentEvaluator` / `AgentOptimizer` 公开签名 | prompt 敏感 fake model 注入点 | + +**issue 真正要新写的「闭环外层」**：失败归因分类器、逐 case delta 对比、可配置 gate 决策、report 生成、 +trace 化样例 case、prompt 敏感 fake model。这些 SDK 都没有。 + +--- + +## 2. 方法论背景 + +评测-优化闭环在工程实践上有一些常见做法，本节提炼其共识与分歧，作为本设计的立论基础。 + +### 2.1 设计共识（已收敛 → 直接沿用） + +1. **六阶段闭环骨架**一致：baseline 评测 → 失败归因 → 优化执行 → 验证集回归 → gate 决策 → 审计落盘。 +2. **失败 taxonomy 高度重合**（6–11 类，核心同名）：`format_violation` / `final_response_mismatch` / + `tool_call_error` / `parameter_error` / `knowledge_recall_insufficient` / `llm_rubric_not_met`。 +3. **过拟合检测公式收敛**：`train_delta > 0 and val_delta <= 0`。 +4. **gate 四要素一致**：val 提升阈值 + 禁止新增 hard fail + 过拟合检测 + 成本/耗时预算。 +5. **fake 只 mock 最底层**（`call_agent` / model registry），让 Evaluator / gate / report 走真实代码路径—— + 这是「无 API key 也能端到端跑通且结果可信」的关键，不 mock 高层。 +6. **report 公共字段一致**：`schema_version`、per-case delta 枚举（`new_pass/new_fail/improved/regressed/unchanged`）、 + gate decision、prompt 审计（sha256 + diff）。 +7. **三类场景构造**一致：可优化成功 / 优化无效 / 优化后退化。 + +### 2.2 关键分歧与各方案取舍 + +| 维度 | 选项（代表 PR） | 各方案问题 | +|---|---|---| +| **归因方法** | 纯规则 / 全反事实 / LLM 裁判 | 规则浅且依赖 expected JSON 字段；反事实慢（每 case 多次 evaluator 调用）；LLM 裁判需 API | +| **候选可信度** | 多数信优化器自报分；或独立复评 | 信优化器自报存在「优化器自吹」偏差 | +| **模块化程度** | 单文件 / 5 模块 / 12+ 模块 | 单文件不可维护；12+ 模块对 example 过度工程 | +| **过拟合检测** | 隐式 val 门槛 / 显式公式 / 双保险 | 隐式不够明确；单公式漏「train 微升 val 平」的泛化缺口 | +| **候选应用安全** | 不改源 / 临时改写源再还原 | 临时改写在中断/并发下有还原风险 | +| **fake 策略** | 静态 fixture 回放 / prompt 敏感生成器 | 静态 fixture 下「改 prompt」不会真改行为，优化是假象 | +| **val 阈值** | 0.01 / 0.05 / 0.1 | 差异大，需参数化 | +| **CI 友好** | 普通退出 / 退出码 0=接受 2=拒绝 | 普通退出无法区分「拒绝」与「出错」| + +### 2.3 本设计的整合策略 + +**以可信与可审计为第一原则**，吸收各 PR 最强的一环，并做三处整合创新（§4）： + +- 骨架与归因/gate：自研分层归因（§4.1，创新）+ 显式过拟合三重检测（§4.3） +- 候选可信度：从 OptimizeResult.rounds 取候选 + 独立 trace 复评（不信任优化器自报分） +- 候选安全：PromptSandbox 隔离（候选只写临时目录，不污染源文件） +- fake：trace + 预录制 variant，确定性无 LLM +- 审计：sha256 全量溯源 + 三态成本 + env snapshot +- CI：退出码 0/2 区分接受/拒绝 +- 模块化：中等粒度（~8 文件），兼顾可读性与不过度工程 + +--- + +## 3. 整体架构 + +### 3.1 闭环流程 + +``` + ┌──────────── 输入 ────────────┐ + │ train.evalset.json (3 case) │ + │ val.evalset.json (3 case) │ + │ optimizer.json (GEPA 配置) │ + │ gate.json (决策阈值) │ + │ prompts/system.md (baseline)│ + └───────────────┬──────────────┘ + ▼ + ① Baseline 评测 (AgentEvaluator, train+val 全量) + ▼ + ② 失败归因 (分层：规则快通道 + 反事实深归因) ★ + ▼ + ③ 优化执行 (AgentOptimizer.optimize, GEPA) + │ └─ _ProposalCapture 订阅 on_proposal_end + ▼ 捕获所有候选(含被 GEPA 丢弃的) + ④ 候选独立复评 (trace-only AgentEvaluator, 去重后逐个) ★ + │ └─ 不信任优化器聚合分 + ▼ + ⑤ 逐 case delta (baseline vs candidate, 按 eval_id join) + ▼ + ⑥ Gate 决策 (可配置 AND 规则, 三态: accept/reject/needs_review) + │ └─ 过拟合三重检测 + ▼ + ⑦ 审计落盘 (optimization_report.json + .md + audit/*.json) + ▼ + 退出码 0(接受) / 2(拒绝) / 1(出错) +``` + +### 3.2 三运行模式 + +| 模式 | call_agent | judge | 用途 | 耗时目标 | +|---|---|---|---|---| +| **fake** | prompt 敏感 `FakeCallAgent`（读能力标记决定输出） | `fake_rubric` scorer（注册到 `LLM_EVALUATOR_REGISTRY`）| 默认，无 API key，验收「≤3 分钟」 | <3min | +| **trace** | 不调用（`eval_mode="trace"` + 预录制 `actual_conversation`）| 同 fake | 确定性回放，CI 回归基线 | <1min | +| **online** | 真实 `agent_module` | 真实 judge model | 真实优化，需 `TRPC_AGENT_API_KEY` 等 env | 不限 | + +三模式共用同一套闭环代码（归因/gate/report），只替换最底层注入。fake/trace 满足「无 key 跑通」验收； +online 是真实业务接入路径。 + +### 3.3 目录结构 + +``` +examples/optimization/eval_optimize_loop/ +├── README.md # 使用说明 + 300-500 字方案设计摘要 +├── DESIGN.md # 本文档 +├── run_pipeline.py # CLI 入口（--mode fake|trace|online） +├── optimizer.json # GEPA 优化配置 +├── gate.json # 可配置 gate 阈值 +├── pipeline/ # 闭环外层（模式无关） +│ ├── __init__.py +│ ├── models.py # pydantic 数据结构（extra="forbid"） +│ ├── config.py # 配置加载+校验+sha256 摘要 +│ ├── attribution.py # ★ 分层失败归因 +│ ├── comparator.py # 逐 case delta +│ ├── gate.py # gate 决策（三态 + 过拟合三重检测） +│ ├── sandbox.py # PromptSandbox 候选隔离 +│ └── reporting.py # report.json + .md + audit/*.json +├── offline/ # fake 模式注入件 +│ ├── __init__.py +│ ├── call_agent.py # prompt 敏感 FakeCallAgent +│ ├── judge.py # fake_rubric scorer +│ └── candidates.py # 三类候选 prompt 样例 +├── agent/ # online 模式被测 agent +│ ├── __init__.py +│ ├── agent.py +│ ├── config.py +│ └── prompts/system.md # baseline prompt（故意留改进空间） +├── data/ +│ ├── train.evalset.json # 3 条 +│ └── val.evalset.json # 3 条（含 1 条 critical） +├── sample_output/ +│ └── optimization_report.json # 示例报告 +└── tests/ + └── test_eval_optimize_loop.py +``` + +**模块化原则**：pipeline/ 是模式无关的纯逻辑（可单测）；offline/ 和 agent/ 是可替换注入件。 +不拆得更细——每个文件一个清晰职责，符合 KISS。 + +--- + +## 4. 核心设计决策（本设计的取舍与创新） + +### 4.1 ★ 分层失败归因：规则快通道 + 反事实深归因 + +**问题**：归因方法在工程上分歧最大。纯规则快但浅，依赖 expected JSON 字段结构，难泛化到 +自由文本 agent；全反事实严谨但慢，每 case 多次 evaluator 调用；LLM 裁判准但依赖 API， +违背「无 key 跑通」。 + +**本设计**：两级流水线，按 case 的信号明确度自适应。 + +``` +每个失败 case + │ + ▼ +[第一层] 规则引擎（快，零成本） + 依据：EvalCaseResult 的 error_message / failed_metrics / actual-vs-expected tool 轨迹 + 输出：(category, confidence) + 规则命中且 confidence=1.0 ──→ 直接采纳，结束 + │ (规则未命中 / 多规则冲突 / 信号弱) + ▼ +[第二层] 反事实干预（慢，有预算上限，默认 4 次/case） + 方法：深拷贝该 case 的 actual_conversation，逐变量替换重评： + · 替换 final_response 为 expected → metric 修复？→ final_response_mismatch + · 替换 tool_name 为 expected → metric 修复？→ tool_selection_error + · 替换 tool_arguments 为 expected → metric 修复？→ parameter_error + · 规范化 format 重评 → metric 修复？→ format_violation + 单一修复命中 → 强归因（confidence 0.9） + 仅组合修复命中 → compound_failure（confidence 0.7） + 全未命中 → 按失败 metric 名兜底（confidence 0.5）→ 仍优于 UNKNOWN +``` + +**为什么是创新**：纯规则与全反事实的分层组合少见。全反事实把每个 case 都做多次干预（慢）；本设计只在规则 +失效时才触发反事实，多数 case 走快通道，疑难 case 才付成本。**预算上限**保证总耗时可控（验收 ≤3 分钟）。 + +**fake/trace 模式下**：反事实重评用的也是 fake judge，零 API 成本，所以两层都便宜。 +**online 模式下**：反事实有真实 judge 成本，预算上限（默认 4 次/case）保护开销。 + +**归因诚实性**：归因结果记录 `source`（rule/counterfactual/fallback） +和 `confidence`，report 里区分展示，不做「gold label 当预测」的自欺。 + +### 4.2 候选独立复评 + GEPA 提案捕获 + +**问题**：`AgentOptimizer.optimize()` 返回的 `best_pass_rate` 是优化器内部聚合分。优化器有动机高报 +（它的目标就是提升分数）。直接信它 = 让运动员当裁判。 + +**本设计**： +1. `_ProposalCapture` 订阅 GEPA 的 `on_proposal_end` 回调，**捕获每一轮候选 prompt**，包括被 GEPA 内部 + 丢弃的提案——这些往往是有教学价值的「优化后退化」样本。 +2. 按完整 prompt 内容 sha256 去重。 +3. 对每个去重候选，用 **trace-only `AgentEvaluator`** 在 train+val 上**独立重打分**，不读优化器的分。 +4. Gate 只采信自己测出来的 train/val 证据。 + +这是防「优化器自吹」的最可信做法。代价是多花一次评测，但 fake/trace 模式下零成本。 + +### 4.3 过拟合三重检测 + +issue 硬验收点：「验证集退化但训练集提升」必须拒绝。单一公式会漏。本设计三道闸，任一触发即拒绝： + +| 检测 | 公式 | 来源 | +|---|---|---| +| **显式过拟合** | `train_delta > 0 and val_delta <= 0` | 共识公式 | +| **泛化缺口** | `(train_score_delta - val_score_delta) > generalization_gap_threshold` | 抓「train 微升 val 平」的隐性感机 | +| **趋势背离** | train pass_rate 趋势升 且 val pass_rate 趋势降（多轮时） | 用 RoundRecord 序列 | + +单轮优化时前两道生效；多轮优化时三道都生效。 + +### 4.4 prompt 敏感 FakeCallAgent（改 prompt 真改行为） + +**问题**：静态 fixture 回放下，候选 prompt 改了但 call_agent 输出不变，优化是假象——验收 +「三类场景」无法真实演示。 + +**本设计**：`FakeCallAgent` 每次调用**重读** `prompts/system.md`（prompt 热加载，与 advanced_strategies +一致），解析其中的**能力标记**决定输出。baseline prompt 故意缺能力，候选 prompt 补能力 → 行为真变。 + +```python +# 伪代码：能力标记协议（写在 prompt 里，对真实 LLM 是无害的自然语言注释） +# system.md (baseline): 故意不写 JSON 输出要求 → format 失败 +# robust.md: "Always respond as strict JSON with keys {route, answer}." → format 修复 +# ineffective.md: 与 baseline 相同 → 无提升 +# overfit.md: 补了 train 需要的能力，但引入 val critical case 的错误分类 → val 退化 +``` + +**注入路径**：通过 `AgentOptimizer.optimize(call_agent=...)` 和 `AgentEvaluator.get_executer(call_agent=...)` +的公开参数注入，不 monkeypatch SDK 内部，不依赖未公开 API。 + +### 4.5 PromptSandbox 候选隔离 + +**问题**：把候选 prompt 临时写进源文件再 finally 还原——中断/并发下有还原风险，可能污染 baseline。 + +**本设计**：候选 prompt 只写进**临时目录**的 `TargetPrompt`，进出 sandbox 各做一次 read-back 校验。 +`update_source` 永远 `False`。Gate 通过后，可选的 `write_back_after_gate` 用 compare-and-swap 风格 +三重摘要校验（写前验基线未变、写后验候选落地）才回写源文件。 + +### 4.6 审计三态成本 + fail-closed + +**问题**：fake/trace 模式无真实 API 调用，成本是 0；但不能简单写 `cost=0`，否则和「online 模式真的花了 0 元」 +无法区分，审计语义混乱。 + +**本设计**：`cost_measurement` 三态： +- `unavailable`：未知（online 模式但未拿到账单）→ **fail-closed，gate 判失败** +- `measured_zero_offline`：fake/trace，确定性的 0 +- `measured_from_replay`：从 trace 的 token usage 重放估算 + +未知成本绝不误写成 0。这是审计严肃性的底线。 + +--- + +## 5. 各阶段详细设计 + +### 5.1 Baseline 评测 + +- 用 `AgentEvaluator.get_executer()` 对 train/val **分别**全量评测（不合并，便于后续按集做 delta）。 +- `num_runs=1`（fake/trace 下确定性；online 可调高平滑方差）。 +- 产出 `BaselineResult{train: SplitResult, val: SplitResult}`，每个 `SplitResult` 含 + `pass_rate / average_score / cases: list[CaseSnapshot]`。 +- `CaseSnapshot` 是 `EvalCaseResult` 的归一化投影：`eval_id / passed / score / hard_fail / + metrics[] / actual_response / expected_response / key_trajectory[]`。 + +### 5.2 失败归因 + +见 §4.1。输入 `CaseSnapshot`，输出 `FailureAttribution{category, confidence, evidence, source}`。 +每个失败 case 至少一条可解释原因（issue 验收点 4）。report 聚合 `category_counts` 与 `coverage_rate` +（被归因的失败 case / 总失败 case）。 + +### 5.3 优化执行 + +- `AgentOptimizer.optimize()` 跑 GEPA，`_ProposalCapture` 捕获候选。 +- fake 模式下：为确定性演示三类场景，`offline/candidates.py` 提供三个固定候选 prompt + （`robust/ineffective/overfit`），通过 fixture optimizer 注入；同时保留真实 GEPA 路径（online）。 +- 产出候选列表 `candidates: list[Candidate]`，每个含 `candidate_id / prompts / source(captured|fixture)`。 +- **多字段 TargetPrompt 支持**：`TargetPrompt` 支持多次 `add_path` 注册多字段。本设计默认单字段 + `system_prompt`（满足 issue「system/skill/router 一种或多种」的下限），架构上可直接扩展到 `skill` / + `router` prompt——多注册一个路径即可，归因/gate/report 全程按 `dict[str,str]` 处理，无需改逻辑。 +- **每轮候选留痕**：`_ProposalCapture` 捕获的原始每轮提案（含被 GEPA 丢弃的）连同 `OptimizeResult.rounds` + 的 `RoundRecord`（`candidate_prompts / accepted / validation_pass_rate / round_llm_cost`）原样落盘到 + `audit/rounds.json`，满足 issue「保存每轮候选 prompt」的审计要求；report 里的 `candidates` 是去重后的视图。 + +### 5.4 候选独立复评 + 逐 case delta + +- 每个候选在 PromptSandbox 内，用 trace-only `AgentEvaluator` 重测 train/val。 +- `comparator.case_delta(baseline_case, candidate_case)` 按 `eval_id` join，输出 5 桶： + `new_pass / new_fail / improved / regressed / unchanged`。 +- 这一步直接服务 issue「区分新增通过、新增失败、分数提升、分数下降」要求。 + +### 5.5 Gate 决策 + +见 §8。三态：`accept / reject / needs_review`。`needs_review` 用于「提升不够 / 证据不足」等灰色地带 +，不强行二分。每条规则记录 `passed / required / actual / expected / reason`。 + +### 5.6 审计落盘 + +每次运行落盘： +- `optimization_report.json`：主报告（schema 见 §6.4） +- `optimization_report.md`：人读版（基线表 / 归因表 / 候选决策表 / delta 表 / gate 表 / 复现命令） +- `audit/input.snapshot.json`：config + train/val/prompt 的 sha256 digest +- `audit/environment.snapshot.json`：sdk_version / git_commit / python / platform / mode / seed +- `audit/gate_decisions.json`：每候选的 gate 判定明细 +- `audit/rounds.json`：GEPA 每轮原始 `RoundRecord`（候选 prompt、accepted、分数、单轮成本），未去重 +- `audit/proposals.json`：`_ProposalCapture` 捕获的全部提案（含被 GEPA 内部丢弃的） +- 所有写盘用「写 .tmp + `os.replace`」原子写，`sort_keys=True` 保证确定性可 diff。 + +--- + +## 6. 数据结构 + +### 6.1 evalset.json（train/val 各 3 条） + +沿用 SDK `EvalCase` schema（见 [_eval_case.py:170](../../../trpc_agent_sdk/evaluation/_eval_case.py#L170)）。 +trace 模式 case 带 `eval_mode: "trace"` + `actual_conversation`。样例结构见 +advanced_strategies/data/train.evalset.json。每条 case 在 `session_input.state` 里放归因所需的 +`expected_trajectory` / `expected_format`（**仅 judge 可见，不进 prompt**，防答案泄漏）。 + +### 6.2 optimizer.json + +```jsonc +{ + "evaluate": { + "metrics": [ + { "metric_name": "final_response_avg_score", "threshold": 1.0, "criterion": {...} }, + { "metric_name": "tool_trajectory_avg_score", "threshold": 1.0, "criterion": {...} } + ], + "num_runs": 1 + }, + "optimize": { + "eval_case_parallelism": 2, + "stop": { "required_metrics": "all" }, + "algorithm": { + "name": "gepa_reflective", "seed": 42, + "reflection_lm": { "model_name": "${TRPC_AGENT_MODEL_NAME}", "api_key": "${TRPC_AGENT_API_KEY}" }, + "max_metric_calls": 60, + "max_iterations_without_improvement": 4 + } + } +} +``` + +### 6.3 gate.json（可配置阈值，全部外置不写死） + +```jsonc +{ + "min_validation_score_delta": 0.05, // val 提升下限（参数化，先验 0.01~0.1） + "max_new_hard_fails": 0, // 禁止新增 hard fail + "max_score_regression_per_case": 0.0, // 单 case 退化上限 + "critical_case_ids": ["val_fiction_key"], // 关键 case 不许退化 + "overfitting": { + "enabled": true, + "formula": "train_delta>0 and val_delta<=0", + "generalization_gap_threshold": 0.1 + }, + "budget": { + "max_metric_calls": 80, + "max_duration_seconds": 180, // 验收 ≤3 分钟 + "cost_measurement": "measured_zero_offline" + }, + "tie_policy": "reject" // score 与 pass_rate 都≈0 → 拒绝 +} +``` + +### 6.4 optimization_report.json schema + +```jsonc +{ + "schema_version": "eval_optimize_loop.v1", + "status": "accepted | rejected | needs_review | failed", + "mode": "fake | trace | online", + "seed": 42, + "baseline": { + "train": { "split": "train", "pass_rate": 0.0, "average_score": 0.5, "cases": [...] }, + "validation": { "split": "validation", "pass_rate": 0.33, "average_score": 0.6, "cases": [...] } + }, + // 注：cases[] 每条含 case_id / passed / score / hard_fail / metrics[] / + // primary_failure(category) / failure_reasons[] / actual_response / + // expected_response / key_trajectory[]——即 issue 阶段 1 要求的 + // 「metric 分、pass/fail、失败原因、关键轨迹」全部 case 级落地。 + "candidates": [ + { + "candidate_id": "robust", + "source": "captured | fixture", + "prompts": { "system_prompt": "..." }, + "train": { "pass_rate": 1.0, "average_score": 1.0 }, + "validation": { "pass_rate": 1.0, "average_score": 1.0 }, + "delta": { + "train": { "pass_rate_delta": 1.0, "average_score_delta": 0.5 }, + "validation": { "pass_rate_delta": 0.67, "average_score_delta": 0.4 }, + "buckets": { "new_pass": [...], "new_fail": [...], "improved": [...], "regressed": [], "unchanged": [...] } + }, + "gate": { + "accepted": true, "overfitting_detected": false, "risk_level": "low", + "checks": [ { "check": "validation_score_improved", "passed": true, "reason": "..." }, ... ] + }, + "audit": { "prompt_sha256": "...", "optimizer_round": 3, "seed": 42 } + } + ], + "selected_candidate_id": "robust", + "failure_attribution": { + "total_failed_cases": 4, "explained_failed_cases": 4, "coverage_rate": 1.0, + "category_counts": { "format_violation": 2, "parameter_error": 1, "tool_selection_error": 1 }, + "by_case": { "train_hours_format": { "category": "format_violation", "confidence": 0.95, "source": "rule", "evidence": "..." } } + }, + "optimizer": { "algorithm": "gepa_reflective", "status": "succeeded", "rounds": 4, "used_agent_optimizer": true }, + "data_quality": { "passed": true, "cross_split_duplicates": 0, "prompt_leakage_matches": 0 }, + "audit": { + "run_id": "...", "started_at": "...", "finished_at": "...", "duration_seconds": 42.3, + "seed": 42, "config_sha256": "...", "train_sha256": "...", "validation_sha256": "...", + "baseline_prompt_sha256": { "system_prompt": "..." }, + "cost": { + "measurement": "measured_zero_offline", // unavailable | measured_zero_offline | measured_from_replay + "optimization_usd": 0.0, // GEPA 反思 LM 花费（fake/trace 为 0） + "evaluation_usd": 0.0, // 候选独立复评 + 反事实重评花费 + "total_usd": 0.0 + }, + "command": "python run_pipeline.py --mode fake" + } +} +``` + +--- + +## 7. 失败归因 Taxonomy + +8 类（覆盖 issue 列举的全部失败类型 + 兜底）： + +| category | 触发信号 | 典型 layer | +|---|---|---| +| `format_violation` | JSON 解析失败 / 缺必需字段 / 格式正则不匹配 | 规则（快）| +| `tool_selection_error` | actual tool_names != expected tool_names | 规则（快）| +| `tool_parameter_error` | tool 名同但 args 异 | 规则（快）| +| `tool_call_error` | tool_response 含 error / failed status | 规则（快）| +| `final_response_mismatch` | final_response 与 expected 不符（且非上述）| 规则（快）/ 反事实 | +| `knowledge_recall_insufficient` | rubric metric 失败且涉及知识正确性 | 规则 / 反事实 | +| `llm_rubric_not_met` | 非 format 的 quality rubric 不达标 | 反事实 | +| `unknown` | 全部未命中（兜底，confidence 0.0）| fallback | + +判定优先级（`missing` 必须在 `response` 前判定，否则误归）： +`tool_call > tool_selection > tool_parameter > format > knowledge_recall > llm_rubric > final_response > unknown`。 + +--- + +## 8. Gate 决策规则 + +AND 语义，每项独立判定。任一 required 失败 → reject。 + +| check | 规则 | 默认 | +|---|---|---| +| `evaluation_complete` | train/val 评测无缺失 | required | +| `validation_score_improved` | `val_score_delta >= min_validation_score_delta` | ≥0.05 | +| `validation_pass_rate_not_worse` | `val_pass_rate_delta >= 0` | required | +| `no_new_hard_fails` | `new_fail count <= max_new_hard_fails` | ≤0 | +| `no_case_regression` | 单 case score 退化 ≤ 阈值 | ≤0.0 | +| `no_critical_regression` | `critical_case_ids` 不在 regressed 桶 | required | +| `no_overfit` | §4.3 三重检测全过 | required | +| `budgets` | metric_calls / duration / cost 不超 | required | +| `tie_policy` | score 与 pass_rate 都≈0 时按策略 | reject | + +`risk_level`：有 critical 回归或 overfit → high；其他失败 → medium；全过 → low。 +决策：所有 required 通过 → `accept`；否则若仅是「提升不足/证据不足」类 → `needs_review`；安全类失败 → `reject`。 + +--- + +## 9. 三运行模式实现 + +| 关注点 | fake | trace | online | +|---|---|---|---| +| `call_agent` | `FakeCallAgent`（prompt 敏感）| 不调用 | 真实 `agent_module` | +| judge | `fake_rubric`（registry 注入）| `fake_rubric` | 真实 judge model | +| 候选来源 | fixture + GEPA 捕获 | trace 回放 | GEPA 捕获 | +| 成本 | `measured_zero_offline` | `measured_zero_offline` | `measured_from_replay`/`unavailable` | +| env 要求 | 无 | 无 | `TRPC_AGENT_API_KEY/BASE_URL/MODEL_NAME`（缺则 exit 2）| +| 用途 | 默认演示、验收 | CI 回归基线 | 真实业务 | + +模式切换只在 `run_pipeline.py --mode` 一处，pipeline/ 内部模式无关。 + +--- + +## 10. 评测集设计（6 条 case，覆盖三类场景） + +**业务域**：图书馆藏查询 agent（图书分类 fiction/science/history/faq + `search_catalog`/`check_availability` 工具查询 + JSON 输出）。 +选这个域是因为它能自然同时触发 format / tool_parameter / knowledge / 分类错配 多类失败，且 critical case 设计直观。 +**业务域兼顾多类失败的天然触发（format / tool / knowledge / 分类错配）。** + +### 10.1 三类场景构造（核心：通过候选 prompt 的能力差异实现） + +| 场景 | 候选 prompt 特征 | baseline→候选 预期 | gate 预期 | +|---|---|---|---| +| **可优化成功** (`robust`) | 补齐 JSON 格式 + 正确分类规则 + 工具查询 | train↑ val↑，critical 不退化 | accept | +| **优化无效** (`ineffective`) | 与 baseline 内容等价 | delta≈0 | reject（tie_policy）| +| **优化后退化** (`overfit`) | 补 train 能力，但引入「所有查询一律归 history」的过拟合规则 | train↑ val↓ | reject（overfit + critical）| + +### 10.2 case 清单 + +**train（3 条，驱动优化）** +- `train_hours_format`：问开馆时间，baseline 返回纯文本非 JSON → `format_violation` +- `train_availability_args`：查《三体》可借状态，工具 `book_id` 参数错 → `tool_parameter_error` +- `train_author_lookup`：问《时间简史》作者，baseline 不调 `search_catalog` 直接猜 → `knowledge_recall_insufficient` + +**val（3 条，驱动 gate，含 1 critical）** +- `val_fiction_key` **（critical）**：科幻小说查询，正确分类 `fiction`；overfit 候选会错归到 `history` +- `val_fiction_generalize`：另一科幻查询（泛化），overfit 同样错归 `history` +- `val_stable_membership`：办证政策 FAQ，全候选稳定（防「全失败」误导） + +每条 case 的 expected（response 关键词 + 工具轨迹）只在 trace evalset 的 `conversation` 字段， +actual 由 `offline/fixtures.py` 按 variant 预录制；数据质量门 `data_quality` 检查 train/val 无 eval_id 重复。 + +--- + +## 11. 验收标准映射 + +| issue 验收点 | 落地位置 | +|---|---| +| 1. 6 条样例 case 全可运行 + 完整报告 | §10 case + `run_pipeline.py --mode fake` 产出 report | +| 2. 隐藏集决策准确率 ≥80% | §4.3 过拟合三重检测 + §8 gate 规则；tests 含隐藏样本断言 | +| 3. 「val 退化 train 提升」必拒绝 | §4.3 显式过拟合公式 + `no_critical_regression` | +| 4. 归因准确率 ≥75% + 每 case ≥1 可解释原因 | §4.1 分层归因 + `coverage_rate`；§4.1 记 source/confidence 防自欺 | +| 5. fake/trace 全流程 ≤3 分钟 | §3.2 三模式 + `budget.max_duration_seconds=180`；fake judge 零 API | +| 6. 报告含 baseline/candidate 分数、逐 case delta、gate 决策、理由 | §6.4 report schema | + +--- + +## 12. 设计取舍与已知局限 + +**取舍** +- **归因分层 vs 全反事实**：选分层，牺牲少量归因严谨性换 ≤3 分钟时效。online 模式可调高反事实预算。 +- **模块化 ~8 文件 vs 14 文件**：选中等。example 应可读，不应是生产框架。YAGNI——不预留未实现的扩展点。 +- **call_agent 注入 vs ModelRegistry 注入**：选 `call_agent`（公开 API、零内部依赖）。ModelRegistry 路径 + 虽更「真实跑全链路」，但依赖 SDK 未公开 API，作为 future enhancement 在 README 注明。 +- **holdout 第三套集**：**不设**。6 条 case 已是 issue 下限，再切 holdout 会让每集太小失去统计意义。 + 防过拟合交给 §4.3 三重检测 + 数据质量门。若后续扩到 20+ case，再加 holdout。 + +**已知局限** +- 6 条 case 规模小，过拟合检测是「构造性确定性」而非统计性。这是 issue 下限决定的，无法回避； + 设计上用三重检测 + 数据质量门最大化鲁棒性。 +- fake judge 是规则 scorer，归因准确率上限受其真实性约束。online 模式换真实 judge 后归因更准。 +- 反事实归因依赖 SDK metric 字段稳定性；SDK 字段变动需同步归因规则。 + +--- + +## 13. 实现里程碑 + +1. **M1 数据与骨架**：evalset(6) + system.md + optimizer.json + gate.json + models.py + config.py +2. **M2 fake 闭环**：FakeCallAgent + fake_rubric + 三候选 fixture → 跑通 baseline→report，产出 sample 报告 +3. **M3 归因与 gate**：分层归因 + 逐 case delta + gate 三态 + 过拟合三重检测 +4. **M4 审计**：PromptSandbox + 原子落盘 + sha256 + env snapshot +5. **M5 trace 模式 + 测试**：trace 回放 + pytest 断言三类场景 + 隐藏样本 + ≤3min +6. **M6 online 模式 + README**：真实 agent 接入 + 300-500 字方案说明 + +每个里程碑都有可运行产物，可独立 review。 + +--- + +## 附：设计要点回顾 + +本闭环的核心增量是 §4.1 的分层归因（规则快通道 + 反事实深归因自适应），用于同时满足 +「归因准确率 ≥75%」与「≤3 分钟」两个看似冲突的验收点：多数 case 走规则快通道， +疑难 case 才触发反事实，且反事实用本地 metric 零成本。 diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md new file mode 100644 index 00000000..079160e5 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/README.md @@ -0,0 +1,147 @@ +# eval_optimize_loop — Evaluation + Optimization 自动闭环 + +> 对应 [issue #91](https://github.com/trpc-group/trpc-agent-python/issues/91) +> 构建「评测 → 失败归因 → prompt 优化 → 验证集回归 → 接受决策 → 审计落盘」的可复现闭环。 + +把一次 prompt 优化从「分数变高了」升级为**可审计的发布决策**：不只跑 `AgentOptimizer`， +而是独立复评每个候选、检测过拟合、给出 accept/reject 决策与理由。 + +## 闭环流程 + +``` +baseline prompt + train.evalset + val.evalset + optimizer.json + gate.json + │ + ① Baseline 评测（AgentEvaluator，train/val 分别打分） + ② 失败归因（分层：规则快通道 + 反事实深归因） + ③ 优化执行（fake: 三候选 fixture；online: 真实 GEPA） + ④ 候选验证（逐 case delta：new_pass/new_fail/improved/regressed/unchanged） + ⑤ Gate 决策（三态 + 过拟合三重检测） + ⑥ 审计落盘（optimization_report.json + .md + audit/*） + ▼ +退出码 0=accept / 2=reject / 1=出错 +``` + +## 快速开始（无需 API key） + +```bash +# 在本目录下，用仓库 venv +python run_pipeline.py --mode fake +``` + +产出 `sample_output/optimization_report.json`（结构化）+ `.md`（人读）+ `audit/`（审计快照）。 +fake 模式全程确定性、无 LLM 调用，6 case 三类场景在 < 1s 内跑完。 + +## 三种模式 + +| 模式 | 评测方式 | 需要 API key | 用途 | +|---|---|---|---| +| `fake` | trace 回放 + 预录制 variant actual | 否 | **默认**，演示三类场景、验收基线 | +| `trace` | 同 fake（确定性 trace 回放） | 否 | CI 回归基线 | +| `online` | 真实 `AgentOptimizer` + `call_agent` | 是 | 真实业务优化 | + +fake/trace 用两个**确定性、无 LLM** 的 SDK evaluator：`final_response_avg_score`（contains） +和 `tool_trajectory_avg_score`（exact）。三候选（robust/ineffective/overfit）的 actual 在 +`offline/fixtures.py` 预录制，让「改 prompt 真改评测结果」可确定性复现。 + +## 三类场景（6 case，3 训练 + 3 验证） + +| 候选 | train | val | gate | 说明 | +|---|---|---|---|---| +| **robust** | 全通过 | 全通过 | **accept** | JSON 格式 + 正确分类 + 馆藏查询全修复 | +| **ineffective** | = baseline | = baseline | **reject**(tie) | 候选与 baseline 等价，无任何提升 | +| **overfit** | 全通过 | critical 退化 | **reject**(overfit) | 修了 train 能力但把图书查询一律错归到 history | + +## 目录结构 + +``` +eval_optimize_loop/ +├── run_pipeline.py # CLI 入口（--mode fake|trace|online） +├── optimizer.json # GEPA 优化配置 + metric 配置 +├── gate.json # 可配置接受策略阈值 +├── pipeline/ # 闭环外层（模式无关） +│ ├── models.py # pydantic 数据结构（extra=forbid） +│ ├── config.py # 配置加载 + sha256 +│ ├── evaluator.py # AgentEvaluator 封装 + 归一化 +│ ├── comparator.py # 逐 case delta（5 桶） +│ ├── attribution.py # 分层失败归因（规则 + 反事实） +│ ├── gate.py # 三态决策 + 过拟合三重检测 +│ └── reporting.py # report.json + .md + audit +├── offline/fixtures.py # 6 case × 4 variant 的预录制 actual（fake/trace 用） +├── agent/ # online 模式被测 agent（真实 LlmAgent + call_agent） +├── data/{train,val}.evalset.json # 样例评测集（expected） +└── tests/test_eval_optimize_loop.py +``` + +## 配置 + +**`gate.json`**（接受策略，全部阈值外置）： +```jsonc +{ + "min_validation_score_delta": 0.05, // val 提升下限 + "max_new_hard_fails": 0, // 禁止新增 hard fail + "critical_case_ids": ["val_fiction_key"], // 关键 case 不许退化 + "overfitting": { "generalization_gap_threshold": 0.1 }, + "budget": { "max_duration_seconds": 180, "cost_measurement": "measured_zero_offline" }, + "tie_policy": "reject" +} +``` + +**`optimizer.json`**：SDK `AgentOptimizer` 标准 GEPA 配置 + `evaluate.metrics`（fake/online 共用）。 + +## 运行测试 + +```bash +python -m pytest tests/ -v +``` + +覆盖：三类场景决策、过拟合检测、归因 coverage/准确率、≤3 分钟、报告字段、隐藏样本归因、CLI 退出码。 + +## online 模式 + +需配置 `TRPC_AGENT_API_KEY` / `TRPC_AGENT_BASE_URL` / `TRPC_AGENT_MODEL_NAME`，然后： + +```bash +python run_pipeline.py --mode online +``` + +online 调用真实 `AgentOptimizer.optimize`（GEPA 反思优化），`agent/agent.py` 的 `call_agent` +每次重读 `system.md`（prompt 热加载），候选 prompt 真实改变 agent 行为。SDK 原生 `OptimizeResult` +（含 baseline/best pass_rate、每轮候选、cost）写入 `sample_output/online_run/`。 + +> 完整的 gate + 自定义 report 闭环（含逐 case delta、独立 trace 复评）在 fake/trace 模式 +> 已完整演示并可无 key 验证；online 接入真实业务时，把 `agent/` 换成业务 agent、 +> `data/` 换成业务评测集即可复用同一套 pipeline 外层。 + +## 方案设计说明（~400 字） + +本闭环的核心是**不信任优化器自报分**，在 `AgentOptimizer` 之上叠加独立编排层。六个阶段对应 +issue 要求，其中三个关键设计决定了能否通过验收： + +1. **分层失败归因**（`attribution.py`）：规则引擎做快通道，从 actual/expected 的工具轨迹与 + response 差异直接归因（覆盖 format/tool/parameter/knowledge/mismatch）；规则未命中或信号弱时 + 才触发反事实干预——单变量替换（只换 response 或只换 tools）重评，用因果证据兜底。反事实用 + 本地纯 Python 复刻 metric（contains + trajectory exact），零 API 成本。这是「归因准确率 ≥75%」 + 与「全流程 ≤3 分钟」两个看似冲突验收点的破局点：多数 case 走快通道，疑难 case 才付成本。 + +2. **过拟合三重检测**（`gate.py`）：显式公式 `train↑ 且 val↓`、泛化缺口 + `train_delta - val_delta > 阈值`（仅在 val 未达标时触发，避免误伤健康候选）、多轮趋势背离。 + 配合 critical case 回归检查，确保「val 退化但 train 提升」的候选必被拒绝。 + +3. **确定性可复现**（`offline/fixtures.py` + `reporting.py`）：fake/trace 用预录制 variant actual + + 两个无 LLM 的 SDK evaluator，全程确定性；落盘用原子写 + sha256 摘要 config/evalset/prompt， + `cost.measurement` 三态区分（unavailable / measured_zero_offline / measured_from_replay）， + 未知成本 fail-closed。 + +Gate 用可配置 AND 规则、三态输出（accept/reject/needs_review），每条 check 带 actual/expected/reason， +退出码 0/2 区分接受/拒绝供 CI 使用。 + +## issue #91 验收对照 + +| 验收点 | 落地 | +|---|---| +| 6 case 全可运行 + 完整报告 | `--mode fake` 产出 report.json/md | +| 决策准确率 ≥80% | 过拟合三重检测 + gate 规则；tests 含隐藏样本 | +| val 退化 train 提升必拒绝 | `gate.py` explicit overfit + critical regression | +| 归因准确率 ≥75% + 每 case ≥1 原因 | 分层归因 + coverage_rate；tests 断言 | +| fake/trace ≤3 分钟 | 确定性 metric 无 LLM，实测 < 1s | +| 报告含 baseline/candidate/delta/gate/理由 | `optimization_report.json` schema | diff --git a/examples/optimization/eval_optimize_loop/agent/__init__.py b/examples/optimization/eval_optimize_loop/agent/__init__.py new file mode 100644 index 00000000..3e52a931 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/agent/__init__.py @@ -0,0 +1 @@ +"""eval_optimize_loop 被测 agent（online 模式真实调用；fake/trace 模式不使用）。""" diff --git a/examples/optimization/eval_optimize_loop/agent/agent.py b/examples/optimization/eval_optimize_loop/agent/agent.py new file mode 100644 index 00000000..adbec0e9 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/agent/agent.py @@ -0,0 +1,73 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""图书馆藏查询 agent（online 模式被测 agent）。 + +关键设计：call_agent 每次调用都 create_agent() → _read_instruction()，从磁盘**重读** +system.md。AgentOptimizer.optimize 每轮通过 TargetPrompt.write_all() 把候选 prompt 原子写入 +system.md，下一轮 call_agent 自然读到新 prompt —— 这就是「prompt 热加载」，让候选 prompt +真实改变 agent 行为（fake/trace 模式则用预录制 actual，见 offline/fixtures.py）。 +""" +from __future__ import annotations + +import uuid +from pathlib import Path + +from trpc_agent_sdk.agents import LlmAgent +from trpc_agent_sdk.models import OpenAIModel +from trpc_agent_sdk.runners import Runner +from trpc_agent_sdk.sessions import InMemorySessionService +from trpc_agent_sdk.types import Content, GenerateContentConfig, Part + +from .config import get_model_config +from .tools import get_order_status + +SYSTEM_PROMPT_PATH = Path(__file__).parent / "prompts" / "system.md" +APP_NAME = "eval_optimize_loop" + + +def _create_model() -> OpenAIModel: + api_key, base_url, model_name = get_model_config() + return OpenAIModel(model_name=model_name, api_key=api_key, base_url=base_url) + + +def _read_instruction() -> str: + """从磁盘重读 system.md（热加载入口）。""" + return SYSTEM_PROMPT_PATH.read_text(encoding="utf-8").strip() + + +def create_agent() -> LlmAgent: + """构建使用当前磁盘 prompt 的新 LlmAgent 实例。""" + return LlmAgent( + name="library_catalog", + description="图书馆藏查询 agent：图书分类 + 馆藏查询 + JSON 输出", + model=_create_model(), + instruction=_read_instruction(), + tools=[get_order_status], + generate_content_config=GenerateContentConfig(temperature=0.1, top_p=0.9, max_output_tokens=256), + ) + + +async def call_agent(query: str) -> str: + """框架回调：跑一次真实推理，返回 final response 文本。""" + root = create_agent() + session_service = InMemorySessionService() + runner = Runner(app_name=APP_NAME, agent=root, session_service=session_service) + session_id = str(uuid.uuid4()) + user_id = "user" + await session_service.create_session(app_name=APP_NAME, user_id=user_id, session_id=session_id, state={}) + user_content = Content(role="user", parts=[Part.from_text(text=query)]) + final_text = "" + async for event in runner.run_async(user_id=user_id, session_id=session_id, new_message=user_content): + if not event.is_final_response(): + continue + if not event.content or not event.content.parts: + continue + for part in event.content.parts: + if part.thought: + continue + if part.text: + final_text += part.text + return final_text diff --git a/examples/optimization/eval_optimize_loop/agent/config.py b/examples/optimization/eval_optimize_loop/agent/config.py new file mode 100644 index 00000000..9e20858c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/agent/config.py @@ -0,0 +1,26 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""模型凭据读取（online 模式）—— 从环境变量加载 OpenAI 兼容 LLM 连接信息。 + +需要的环境变量： + TRPC_AGENT_API_KEY LLM 后端 API key + TRPC_AGENT_BASE_URL LLM 后端 endpoint + TRPC_AGENT_MODEL_NAME 模型名 + +缺任意一个立即抛 ValueError，避免运行到一半才撞 401。 +""" +from __future__ import annotations + +import os + + +def get_model_config() -> tuple[str, str, str]: + api_key = os.getenv("TRPC_AGENT_API_KEY", "") + base_url = os.getenv("TRPC_AGENT_BASE_URL", "") + model_name = os.getenv("TRPC_AGENT_MODEL_NAME", "") + if not api_key or not base_url or not model_name: + raise ValueError("online 模式需配置 TRPC_AGENT_API_KEY / TRPC_AGENT_BASE_URL / TRPC_AGENT_MODEL_NAME。") + return api_key, base_url, model_name diff --git a/examples/optimization/eval_optimize_loop/agent/prompts/candidate_ineffective.md b/examples/optimization/eval_optimize_loop/agent/prompts/candidate_ineffective.md new file mode 100644 index 00000000..7c1f3c3e --- /dev/null +++ b/examples/optimization/eval_optimize_loop/agent/prompts/candidate_ineffective.md @@ -0,0 +1 @@ +你是一位友好的图书馆助手，请尽力协助每一位读者。 diff --git a/examples/optimization/eval_optimize_loop/agent/prompts/candidate_overfit.md b/examples/optimization/eval_optimize_loop/agent/prompts/candidate_overfit.md new file mode 100644 index 00000000..8d73a8c4 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/agent/prompts/candidate_overfit.md @@ -0,0 +1,11 @@ +你是图书馆助手，处理图书查询和借阅问题。 + +## 输出格式 +始终以严格 JSON 响应：{"category": "", "answer": "<回答>"}。 + +## 分类规则 +- 所有图书查询请求（含小说、科幻）一律 category = "history"，统一在历史书架处理 +- 政策、办证、开馆时间 → category = "faq" + +## 知识要求 +涉及具体图书信息时，必须先调用工具查询，不得猜测。 diff --git a/examples/optimization/eval_optimize_loop/agent/prompts/candidate_robust.md b/examples/optimization/eval_optimize_loop/agent/prompts/candidate_robust.md new file mode 100644 index 00000000..da2d8022 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/agent/prompts/candidate_robust.md @@ -0,0 +1,13 @@ +你是图书馆助手，处理图书查询和借阅问题。 + +## 输出格式 +始终以严格 JSON 响应：{"category": "", "answer": "<回答>"}。 + +## 分类规则 +- 小说、科幻、文学类 → category = "fiction" +- 科学、技术、计算机类 → category = "science" +- 历史、传记类 → category = "history" +- 政策、办证、开馆时间 → category = "faq" + +## 知识要求 +涉及具体图书信息（作者、可借状态、书架位置）时，必须先调用工具查询，不得猜测。 diff --git a/examples/optimization/eval_optimize_loop/agent/prompts/system.md b/examples/optimization/eval_optimize_loop/agent/prompts/system.md new file mode 100644 index 00000000..d347941f --- /dev/null +++ b/examples/optimization/eval_optimize_loop/agent/prompts/system.md @@ -0,0 +1 @@ +你是图书馆助手，帮助读者查询图书和借阅信息。 diff --git a/examples/optimization/eval_optimize_loop/agent/tools.py b/examples/optimization/eval_optimize_loop/agent/tools.py new file mode 100644 index 00000000..80e4e99e --- /dev/null +++ b/examples/optimization/eval_optimize_loop/agent/tools.py @@ -0,0 +1,32 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""被测 agent 的工具（online 模式真实调用；fake/trace 不使用）。""" +from __future__ import annotations + +# 演示用图书目录数据库（真实业务替换为远端查询） +_CATALOG: dict[str, dict[str, str]] = { + "时间简史": { + "author": "霍金", + "category": "science", + "book_id": "BT-000" + }, + "三体": { + "author": "刘慈欣", + "category": "fiction", + "book_id": "BT-001" + }, +} +_AVAILABILITY: dict[str, str] = {"BT-001": "可借", "BT-000": "已借出"} + + +def search_catalog(query: str) -> dict: + """按书名/关键词搜索馆藏目录。""" + return _CATALOG.get(query, {"author": "未找到", "category": "unknown", "book_id": ""}) + + +def check_availability(book_id: str) -> dict: + """查询某 book_id 的可借状态。""" + return {"book_id": book_id, "status": _AVAILABILITY.get(book_id, "未知")} diff --git a/examples/optimization/eval_optimize_loop/data/train.evalset.json b/examples/optimization/eval_optimize_loop/data/train.evalset.json new file mode 100644 index 00000000..a8bcb904 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/train.evalset.json @@ -0,0 +1,49 @@ +{ + "eval_set_id": "eval_optimize_loop_train", + "name": "图书馆藏查询 agent - 训练集", + "description": "3 条训练 case，分别覆盖 format_violation / tool_parameter_error / knowledge_recall_insufficient。expected.final_response 的文本是 contains metric 的判定关键词（必现内容）。", + "eval_cases": [ + { + "eval_id": "train_hours_format", + "conversation": [ + { + "invocation_id": "inv-1", + "user_content": {"parts": [{"text": "图书馆的开馆时间是什么时候？"}], "role": "user"}, + "final_response": {"parts": [{"text": "category"}], "role": "model"}, + "creation_timestamp": 0.0 + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "user", "state": {}} + }, + { + "eval_id": "train_availability_args", + "conversation": [ + { + "invocation_id": "inv-1", + "user_content": {"parts": [{"text": "《三体》现在能借吗？"}], "role": "user"}, + "final_response": {"parts": [{"text": "可借"}], "role": "model"}, + "intermediate_data": { + "tool_uses": [{"id": "t0", "name": "check_availability", "args": {"book_id": "BT-001"}}] + }, + "creation_timestamp": 0.0 + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "user", "state": {}} + }, + { + "eval_id": "train_author_lookup", + "conversation": [ + { + "invocation_id": "inv-1", + "user_content": {"parts": [{"text": "《时间简史》的作者是谁？"}], "role": "user"}, + "final_response": {"parts": [{"text": "霍金"}], "role": "model"}, + "intermediate_data": { + "tool_uses": [{"id": "t0", "name": "search_catalog", "args": {"query": "时间简史"}}] + }, + "creation_timestamp": 0.0 + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "user", "state": {}} + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/data/val.evalset.json b/examples/optimization/eval_optimize_loop/data/val.evalset.json new file mode 100644 index 00000000..cce6d6e9 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/data/val.evalset.json @@ -0,0 +1,43 @@ +{ + "eval_set_id": "eval_optimize_loop_val", + "name": "图书馆藏查询 agent - 验证集", + "description": "3 条验证 case。val_fiction_key 为关键 case（critical_case_ids），用于检验过拟合：overfit 候选会把图书查询一律归到 history，导致该 case 与 val_fiction_generalize 一起退化。", + "eval_cases": [ + { + "eval_id": "val_fiction_key", + "conversation": [ + { + "invocation_id": "inv-1", + "user_content": {"parts": [{"text": "帮我找一本科幻小说。"}], "role": "user"}, + "final_response": {"parts": [{"text": "fiction"}], "role": "model"}, + "creation_timestamp": 0.0 + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "user", "state": {}} + }, + { + "eval_id": "val_fiction_generalize", + "conversation": [ + { + "invocation_id": "inv-1", + "user_content": {"parts": [{"text": "有没有新的科幻书推荐？"}], "role": "user"}, + "final_response": {"parts": [{"text": "fiction"}], "role": "model"}, + "creation_timestamp": 0.0 + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "user", "state": {}} + }, + { + "eval_id": "val_stable_membership", + "conversation": [ + { + "invocation_id": "inv-1", + "user_content": {"parts": [{"text": "办借书证需要什么条件？"}], "role": "user"}, + "final_response": {"parts": [{"text": "category"}], "role": "model"}, + "creation_timestamp": 0.0 + } + ], + "session_input": {"app_name": "eval_optimize_loop", "user_id": "user", "state": {}} + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/gate.json b/examples/optimization/eval_optimize_loop/gate.json new file mode 100644 index 00000000..7bff8d6a --- /dev/null +++ b/examples/optimization/eval_optimize_loop/gate.json @@ -0,0 +1,16 @@ +{ + "min_validation_score_delta": 0.05, + "max_new_hard_fails": 0, + "max_score_regression_per_case": 0.0, + "critical_case_ids": ["val_fiction_key"], + "overfitting": { + "enabled": true, + "generalization_gap_threshold": 0.1 + }, + "budget": { + "max_metric_calls": 80, + "max_duration_seconds": 180, + "cost_measurement": "measured_zero_offline" + }, + "tie_policy": "reject" +} diff --git a/examples/optimization/eval_optimize_loop/offline/__init__.py b/examples/optimization/eval_optimize_loop/offline/__init__.py new file mode 100644 index 00000000..cd1888ab --- /dev/null +++ b/examples/optimization/eval_optimize_loop/offline/__init__.py @@ -0,0 +1 @@ +"""eval_optimize_loop offline：fake/trace 模式的确定性注入件。""" diff --git a/examples/optimization/eval_optimize_loop/offline/fixtures.py b/examples/optimization/eval_optimize_loop/offline/fixtures.py new file mode 100644 index 00000000..f4b431d3 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/offline/fixtures.py @@ -0,0 +1,291 @@ +"""fake/trace 模式的确定性 fixture：6 case 的 expected + 4 候选 variant 的预录制 actual。 + +业务域：图书馆藏查询 agent（与客服/算术/地址等已有 example 区分，case 全部原创）。 +三类场景通过 variant 差异实现（见 DESIGN.md §10）： +- robust: 全修复 → train/val 全通过 → accept +- ineffective: 与 baseline 等价 → delta≈0 → reject(tie) +- overfit: 修了 train 的 format/tool/knowledge，但把图书查询一律归到 history + → val 的两个 fiction case 退化 → reject(overfit) + +baseline 各 split 表现：train 0/3（format/tool/knowledge 全失败）、val 2/3（两个 fiction +case 靠 contains 命中、membership 因缺 JSON 标记失败）。 +""" +from __future__ import annotations + +from typing import Any, Literal + +from trpc_agent_sdk.evaluation import EvalSet + +Split = Literal["train", "validation"] +Variant = Literal["baseline", "robust", "ineffective", "overfit"] + +ToolUse = dict[str, Any] # {"name": str, "args": dict} +VariantOutput = dict[str, Any] # {"response": str, "tool_uses": list[ToolUse]} + +CASES: list[dict[str, Any]] = [ + { + "eval_id": "train_hours_format", + "split": "train", + "critical": False, + "expected_category": "format_violation", + "query": "图书馆的开馆时间是什么时候？", + "expected_response": "category", # contains: actual 必含 "category"（JSON 结构标记） + "expected_tool_uses": [], + "variants": { + "baseline": { + "response": "开馆时间为每天 9:00-21:00。", + "tool_uses": [] + }, + "robust": { + "response": '{"category":"faq","answer":"每天 9:00-21:00"}', + "tool_uses": [] + }, + "ineffective": { + "response": "开馆时间为每天 9:00-21:00。", + "tool_uses": [] + }, + "overfit": { + "response": '{"category":"faq","answer":"每天 9:00-21:00"}', + "tool_uses": [] + }, + }, + }, + { + "eval_id": "train_availability_args", + "split": "train", + "critical": False, + "expected_category": "tool_parameter_error", + "query": "《三体》现在能借吗？", + "expected_response": "可借", + "expected_tool_uses": [{ + "name": "check_availability", + "args": { + "book_id": "BT-001" + } + }], + "variants": { + "baseline": { + "response": "可借", + "tool_uses": [{ + "name": "check_availability", + "args": { + "book_id": "BT-999" + } + }], + }, + "robust": { + "response": "可借", + "tool_uses": [{ + "name": "check_availability", + "args": { + "book_id": "BT-001" + } + }], + }, + "ineffective": { + "response": "可借", + "tool_uses": [{ + "name": "check_availability", + "args": { + "book_id": "BT-999" + } + }], + }, + "overfit": { + "response": "可借", + "tool_uses": [{ + "name": "check_availability", + "args": { + "book_id": "BT-001" + } + }], + }, + }, + }, + { + "eval_id": "train_author_lookup", + "split": "train", + "critical": False, + "expected_category": "knowledge_recall_insufficient", + "query": "《时间简史》的作者是谁？", + "expected_response": "霍金", + "expected_tool_uses": [{ + "name": "search_catalog", + "args": { + "query": "时间简史" + } + }], + "variants": { + "baseline": { + "response": "霍金", + "tool_uses": [] + }, # 不查询直接猜 → trajectory fail + "robust": { + "response": "霍金", + "tool_uses": [{ + "name": "search_catalog", + "args": { + "query": "时间简史" + } + }], + }, + "ineffective": { + "response": "霍金", + "tool_uses": [] + }, + "overfit": { + "response": "霍金", + "tool_uses": [{ + "name": "search_catalog", + "args": { + "query": "时间简史" + } + }], + }, + }, + }, + { + "eval_id": "val_fiction_key", + "split": "validation", + "critical": True, + "expected_category": "final_response_mismatch", + "query": "帮我找一本科幻小说。", + "expected_response": "fiction", + "expected_tool_uses": [], + "variants": { + "baseline": { + "response": "为您查找科幻(fiction)类小说。", + "tool_uses": [] + }, + "robust": { + "response": '{"category":"fiction","answer":"科幻书架在二楼"}', + "tool_uses": [], + }, + "ineffective": { + "response": "为您查找科幻(fiction)类小说。", + "tool_uses": [] + }, + "overfit": { + "response": '{"category":"history","answer":"已转入历史书架"}', + "tool_uses": [], + }, + }, + }, + { + "eval_id": "val_fiction_generalize", + "split": "validation", + "critical": False, + "expected_category": "final_response_mismatch", + "query": "有没有新的科幻书推荐？", + "expected_response": "fiction", + "expected_tool_uses": [], + "variants": { + "baseline": { + "response": "科幻(fiction)新书推荐中。", + "tool_uses": [] + }, + "robust": { + "response": '{"category":"fiction","answer":"新书推荐"}', + "tool_uses": [] + }, + "ineffective": { + "response": "科幻(fiction)新书推荐中。", + "tool_uses": [] + }, + "overfit": { + "response": '{"category":"history","answer":"历史新书"}', + "tool_uses": [] + }, + }, + }, + { + "eval_id": "val_stable_membership", + "split": "validation", + "critical": False, + "expected_category": "format_violation", + "query": "办借书证需要什么条件？", + "expected_response": "category", + "expected_tool_uses": [], + "variants": { + "baseline": { + "response": "凭身份证免费办理。", + "tool_uses": [] + }, + "robust": { + "response": '{"category":"faq","answer":"凭身份证免费办理"}', + "tool_uses": [], + }, + "ineffective": { + "response": "凭身份证免费办理。", + "tool_uses": [] + }, + "overfit": { + "response": '{"category":"faq","answer":"凭身份证免费办理"}', + "tool_uses": [], + }, + }, + }, +] + +# 候选 id → prompt 文件名（agent/prompts/ 下）。baseline 用 system.md。 +CANDIDATE_PROMPTS: dict[str, str] = { + "baseline": "system.md", + "robust": "candidate_robust.md", + "ineffective": "candidate_ineffective.md", + "overfit": "candidate_overfit.md", +} + + +def _invocation(query: str, response: str, tool_uses: list[ToolUse]) -> dict[str, Any]: + """构造一个 Invocation 的 dict（供 EvalSet.model_validate 递归解析）。""" + inv: dict[str, Any] = { + "invocation_id": "inv-1", + "user_content": { + "parts": [{ + "text": query + }], + "role": "user" + }, + "final_response": { + "parts": [{ + "text": response + }], + "role": "model" + }, + "creation_timestamp": 0.0, + } + if tool_uses: + inv["intermediate_data"] = { + "tool_uses": [{ + "id": f"t{i}", + "name": t["name"], + "args": t["args"] + } for i, t in enumerate(tool_uses)] + } + return inv + + +def build_trace_eval_set(cases: list[dict[str, Any]], variant: Variant, split: Split) -> EvalSet: + """构造 trace 模式 EvalSet：actual_conversation = 该 variant 的预录制输出， + conversation = expected（标准答案）。trace 模式跳过 agent，直接用 actual 评分。 + """ + selected = [c for c in cases if c["split"] == split] + eval_cases = [] + for c in selected: + v: VariantOutput = c["variants"][variant] + eval_cases.append({ + "eval_id": c["eval_id"], + "eval_mode": "trace", + "actual_conversation": [_invocation(c["query"], v["response"], v["tool_uses"])], + "conversation": [_invocation(c["query"], c["expected_response"], c["expected_tool_uses"])], + "session_input": { + "app_name": "eval_optimize_loop", + "user_id": "user", + "state": {}, + }, + }) + return EvalSet.model_validate({"eval_set_id": f"eol_{split}_{variant}", "eval_cases": eval_cases}) + + +def cases_for_split(cases: list[dict[str, Any]], split: Split) -> list[dict[str, Any]]: + return [c for c in cases if c["split"] == split] diff --git a/examples/optimization/eval_optimize_loop/optimizer.json b/examples/optimization/eval_optimize_loop/optimizer.json new file mode 100644 index 00000000..1f703c14 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/optimizer.json @@ -0,0 +1,50 @@ +{ + "evaluate": { + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": {"match": "contains", "case_insensitive": true} + } + } + }, + { + "metric_name": "tool_trajectory_avg_score", + "threshold": 1.0, + "criterion": { + "tool_trajectory": { + "default": { + "name": {"match": "exact", "case_insensitive": false}, + "arguments": {"match": "exact"} + }, + "order_sensitive": false, + "subset_matching": false + } + } + } + ], + "num_runs": 1 + }, + "optimize": { + "eval_case_parallelism": 2, + "stop": {"required_metrics": "all"}, + "algorithm": { + "name": "gepa_reflective", + "seed": 42, + "reflection_lm": { + "model_name": "${TRPC_AGENT_MODEL_NAME}", + "base_url": "${TRPC_AGENT_BASE_URL}", + "api_key": "${TRPC_AGENT_API_KEY}", + "generation_config": {"max_tokens": 4096, "temperature": 0.6} + }, + "candidate_selection_strategy": "pareto", + "module_selector": "round_robin", + "frontier_type": "instance", + "reflection_minibatch_size": 3, + "max_metric_calls": 60, + "max_iterations_without_improvement": 4 + } + } +} diff --git a/examples/optimization/eval_optimize_loop/pipeline/__init__.py b/examples/optimization/eval_optimize_loop/pipeline/__init__.py new file mode 100644 index 00000000..31cf728e --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/__init__.py @@ -0,0 +1 @@ +"""eval_optimize_loop pipeline：模式无关的闭环外层逻辑。""" diff --git a/examples/optimization/eval_optimize_loop/pipeline/attribution.py b/examples/optimization/eval_optimize_loop/pipeline/attribution.py new file mode 100644 index 00000000..b7236cb3 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/attribution.py @@ -0,0 +1,192 @@ +"""分层失败归因：规则快通道 + 反事实深归因（本地确定性 metric 验证）。 + +见 DESIGN.md §4.1 / §7。两层按 case 信号明确度自适应： +- 规则层：从 actual/expected 的工具轨迹 + response 差异直接归因（confidence 0.85） +- 反事实层：单变量替换验证（只换 response / 只换 tools）。与规则一致 → 提升至 0.95； + 规则未命中 → 反事实兜底（0.7 / 0.6）；都失败 → unknown（0.3, fallback） + +反事实用本地纯 Python 实现 metric（contains + trajectory exact），不调 SDK、不调 LLM， +所以即使触发也零成本——这是「归因准确率 ≥75%」与「≤3 分钟」兼容的关键。 +""" +from __future__ import annotations + +from typing import Any + +from .models import FailureAttribution, FailureAttributionSummary, SplitResult + +# 视为「格式结构标记」的 expected 关键词（用于区分 format_violation 与 final_response_mismatch） +_FORMAT_MARKERS = {"route", "answer", "category", "json", "{", "}"} + + +def _tools_equal(a: list[dict[str, Any]], b: list[dict[str, Any]]) -> bool: + if len(a) != len(b): + return False + for x, y in zip(a, b): + if x.get("name") != y.get("name") or x.get("args") != y.get("args"): + return False + return True + + +def _resp_ok(actual_resp: str, expected_resp: str) -> bool: + """对应 SDK final_response_avg_score 的 contains criterion（本地复刻）。""" + if not expected_resp: + return True + return expected_resp.lower() in (actual_resp or "").lower() + + +def _rule_attribute(case_spec: dict[str, Any], variant: str) -> tuple[str, str, float]: + """规则归因 → (category, evidence, confidence)。判定优先级见 DESIGN.md §7。""" + expected_tools = case_spec.get("expected_tool_uses", []) + actual_tools = case_spec["variants"][variant]["tool_uses"] + expected_resp = case_spec.get("expected_response", "") + actual_resp = case_spec["variants"][variant]["response"] + + # 工具维度（优先于 response） + if expected_tools: + exp_names = [t["name"] for t in expected_tools] + act_names = [t["name"] for t in actual_tools] + if not actual_tools: + return ( + "knowledge_recall_insufficient", + f"expected tool call(s) {exp_names} but agent called none (guessed without query)", + 0.85, + ) + if exp_names != act_names: + return ( + "tool_selection_error", + f"expected tool names {exp_names}, actual {act_names}", + 0.85, + ) + for et, at in zip(expected_tools, actual_tools): + if et.get("args") != at.get("args"): + return ( + "tool_parameter_error", + f"tool '{et['name']}' expected args {et['args']}, got {at['args']}", + 0.85, + ) + + # response 维度 + if expected_resp and not _resp_ok(actual_resp, expected_resp): + if expected_resp.lower() in _FORMAT_MARKERS: + return ( + "format_violation", + f"actual response missing required structure marker '{expected_resp}'", + 0.85, + ) + return ( + "final_response_mismatch", + f"actual response missing expected key content '{expected_resp}'", + 0.85, + ) + + return ("unknown", "no decisive rule signal", 0.3) + + +def _counterfactual(case_spec: dict[str, Any], variant: str) -> str | None: + """单变量替换反事实：返回失败侧 'response' / 'tool' / 'compound' / None。 + + 只换 response（用 expected）→ 整体 pass？说明失败在 response 侧。 + 只换 tools（用 expected）→ 整体 pass？说明失败在 tool 侧。 + """ + expected_tools = case_spec.get("expected_tool_uses", []) + expected_resp = case_spec.get("expected_response", "") + actual = case_spec["variants"][variant] + + base_resp_ok = _resp_ok(actual["response"], expected_resp) + base_tool_ok = _tools_equal(actual["tool_uses"], expected_tools) + + # 只换 response（expected 一定含自身 → resp metric 必过），tool 维持 + cf_resp_repairs = base_tool_ok + # 只换 tools（expected == expected → tool metric 必过），response 维持 + cf_tool_repairs = base_resp_ok + + if not base_resp_ok and base_tool_ok and cf_resp_repairs: + return "response" + if not base_tool_ok and base_resp_ok and cf_tool_repairs: + return "tool" + if not base_resp_ok and not base_tool_ok: + return "compound" + return None + + +def attribute_failure(case_spec: dict[str, Any], variant: str) -> FailureAttribution: + """对一个失败 case 做分层归因。""" + cat, evidence, conf = _rule_attribute(case_spec, variant) + if cat != "unknown": + side = _counterfactual(case_spec, variant) + response_cats = {"format_violation", "final_response_mismatch"} + tool_cats = { + "tool_selection_error", + "tool_parameter_error", + "knowledge_recall_insufficient", + "tool_call_error", + } + consistent = (cat in response_cats and side == "response") or (cat in tool_cats and side == "tool") + if consistent: + conf = 0.95 + return FailureAttribution(category=cat, confidence=conf, evidence=evidence, source="rule") + + # 规则未命中 → 反事实兜底 + side = _counterfactual(case_spec, variant) + if side == "response": + return FailureAttribution( + category="final_response_mismatch", + confidence=0.7, + evidence="counterfactual: replacing response alone repairs metrics", + source="counterfactual", + ) + if side == "tool": + return FailureAttribution( + category="tool_call_error", + confidence=0.7, + evidence="counterfactual: replacing tools alone repairs metrics", + source="counterfactual", + ) + if side == "compound": + return FailureAttribution( + category="tool_call_error", + confidence=0.6, + evidence="counterfactual: only combined repair fixes metrics (compound failure)", + source="counterfactual", + ) + return FailureAttribution( + category="unknown", + confidence=0.3, + evidence="no rule or counterfactual signal", + source="fallback", + ) + + +def attribute_failures( + baseline_train: SplitResult, + baseline_val: SplitResult, + cases: list[dict[str, Any]], + variant: str, +) -> FailureAttributionSummary: + """对 baseline 的所有失败 case 归因（baseline 失败是优化目标，驱动 §5.2 归因阶段）。""" + case_map = {c["eval_id"]: c for c in cases} + by_case: dict[str, FailureAttribution] = {} + total_failed = 0 + category_counts: dict[str, int] = {} + + for split_result in (baseline_train, baseline_val): + for snap in split_result.cases: + if snap.passed: + continue + total_failed += 1 + spec = case_map.get(snap.eval_id) + if not spec: + continue + attr = attribute_failure(spec, variant) + by_case[snap.eval_id] = attr + category_counts[attr.category] = category_counts.get(attr.category, 0) + 1 + + explained = sum(1 for a in by_case.values() if a.category != "unknown") + coverage = explained / total_failed if total_failed else 1.0 + return FailureAttributionSummary( + total_failed_cases=total_failed, + explained_failed_cases=explained, + coverage_rate=coverage, + category_counts=category_counts, + by_case=by_case, + ) diff --git a/examples/optimization/eval_optimize_loop/pipeline/comparator.py b/examples/optimization/eval_optimize_loop/pipeline/comparator.py new file mode 100644 index 00000000..d2fc4465 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/comparator.py @@ -0,0 +1,77 @@ +"""逐 case delta：baseline vs candidate，分 5 桶。""" +from __future__ import annotations + +from .models import ( + Bucket, + CandidateDelta, + CaseDelta, + CaseSnapshot, + DeltaBuckets, + SplitDelta, + SplitResult, +) + +_EPS = 1e-9 + + +def _bucket(base: CaseSnapshot, cand: CaseSnapshot) -> Bucket: + if not base.passed and cand.passed: + return "new_pass" + if base.passed and not cand.passed: + return "new_fail" + if cand.score > base.score + _EPS: + return "improved" + if cand.score < base.score - _EPS: + return "regressed" + return "unchanged" + + +def _split_delta(base: SplitResult, cand: SplitResult) -> SplitDelta: + return SplitDelta( + split=cand.split, + pass_rate_delta=cand.pass_rate - base.pass_rate, + average_score_delta=cand.average_score - base.average_score, + ) + + +def compare( + base_train: SplitResult, + base_val: SplitResult, + cand_train: SplitResult, + cand_val: SplitResult, +) -> CandidateDelta: + """合并 train+val 的逐 case 对比。case eval_id 全局唯一，可合并分桶。""" + buckets = DeltaBuckets() + case_deltas: list[CaseDelta] = [] + + for base_split, cand_split in [(base_train, cand_train), (base_val, cand_val)]: + base_map = {c.eval_id: c for c in base_split.cases} + for cc in cand_split.cases: + bc = base_map[cc.eval_id] + b = _bucket(bc, cc) + case_deltas.append( + CaseDelta( + eval_id=cc.eval_id, + baseline_passed=bc.passed, + candidate_passed=cc.passed, + baseline_score=bc.score, + candidate_score=cc.score, + bucket=b, + )) + # 按桶分入 DeltaBuckets + if b == "new_pass": + buckets.new_pass.append(cc.eval_id) + elif b == "new_fail": + buckets.new_fail.append(cc.eval_id) + elif b == "improved": + buckets.improved.append(cc.eval_id) + elif b == "regressed": + buckets.regressed.append(cc.eval_id) + else: + buckets.unchanged.append(cc.eval_id) + + return CandidateDelta( + train=_split_delta(base_train, cand_train), + validation=_split_delta(base_val, cand_val), + buckets=buckets, + ) diff --git a/examples/optimization/eval_optimize_loop/pipeline/config.py b/examples/optimization/eval_optimize_loop/pipeline/config.py new file mode 100644 index 00000000..3764d374 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/config.py @@ -0,0 +1,72 @@ +"""配置加载、校验、sha256 摘要。""" +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from trpc_agent_sdk.evaluation import EvalConfig + + +@dataclass +class GateConfig: + """可配置的接受策略阈值（gate.json），全部外置不写死。""" + + min_validation_score_delta: float = 0.05 + max_new_hard_fails: int = 0 + max_score_regression_per_case: float = 0.0 + critical_case_ids: list[str] = field(default_factory=list) + overfitting_enabled: bool = True + generalization_gap_threshold: float = 0.1 + max_metric_calls: int = 80 + max_duration_seconds: int = 180 + cost_measurement: str = "measured_zero_offline" + tie_policy: str = "reject" + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> "GateConfig": + overfit = d.get("overfitting", {}) or {} + budget = d.get("budget", {}) or {} + return cls( + min_validation_score_delta=d.get("min_validation_score_delta", 0.05), + max_new_hard_fails=d.get("max_new_hard_fails", 0), + max_score_regression_per_case=d.get("max_score_regression_per_case", 0.0), + critical_case_ids=list(d.get("critical_case_ids", [])), + overfitting_enabled=overfit.get("enabled", True), + generalization_gap_threshold=overfit.get("generalization_gap_threshold", 0.1), + max_metric_calls=budget.get("max_metric_calls", 80), + max_duration_seconds=budget.get("max_duration_seconds", 180), + cost_measurement=budget.get("cost_measurement", "measured_zero_offline"), + tie_policy=d.get("tie_policy", "reject"), + ) + + +def load_gate_config(path: str | Path) -> GateConfig: + return GateConfig.from_dict(json.loads(Path(path).read_text(encoding="utf-8"))) + + +def load_eval_config(optimizer_path: str | Path) -> EvalConfig: + """从 optimizer.json 的 evaluate 字段构造 SDK EvalConfig。 + + fake/trace/online 三模式共用同一套 metric 配置，避免漂移。 + 只读 evaluate 节点（不触发 optimize.algorithm.reflection_lm 的 env 解析）， + 这样无 API key 时也能加载。 + """ + data = json.loads(Path(optimizer_path).read_text(encoding="utf-8")) + evaluate = data["evaluate"] + return EvalConfig.model_validate(evaluate) + + +def sha256_bytes(b: bytes) -> str: + return hashlib.sha256(b).hexdigest() + + +def sha256_file(path: str | Path) -> str: + return sha256_bytes(Path(path).read_bytes()) + + +def sha256_dict(d: Any) -> str: + """dict 的确定性摘要（sort_keys 保证可 diff）。""" + return sha256_bytes(json.dumps(d, sort_keys=True, ensure_ascii=False).encode("utf-8")) diff --git a/examples/optimization/eval_optimize_loop/pipeline/evaluator.py b/examples/optimization/eval_optimize_loop/pipeline/evaluator.py new file mode 100644 index 00000000..0a857658 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/evaluator.py @@ -0,0 +1,58 @@ +"""评测封装：跑 SDK AgentEvaluator.evaluate_eval_set 并归一化为 SplitResult。 + +trace 模式下 actual 预录制（fixtures），SDK 跳过 agent 直接评分， +两个确定性 metric（final_response contains / tool_trajectory exact）全程无 LLM。 +""" +from __future__ import annotations + +from statistics import mean +from typing import Any + +from trpc_agent_sdk.evaluation import AgentEvaluator, EvalConfig + +from offline.fixtures import VariantOutput, build_trace_eval_set, cases_for_split +from .models import CaseSnapshot, SplitResult + + +def _passed(status: Any) -> bool: + if status is None: + return False + return str(getattr(status, "value", status)).upper() == "PASSED" + + +async def evaluate_split(cases: list[dict[str, Any]], variant: str, split: str, eval_config: EvalConfig) -> SplitResult: + """对给定 variant + split 跑 trace 评测，返回归一化的 SplitResult。""" + eval_set = build_trace_eval_set(cases, variant, split) + _failed, _details, _lines, results_by_id = await AgentEvaluator.evaluate_eval_set(eval_set, + eval_config=eval_config, + print_detailed_results=False) + + ordered = cases_for_split(cases, split) + case_map = {c["eval_id"]: c for c in ordered} + snapshots_by_id: dict[str, CaseSnapshot] = {} + for eval_id, runs in results_by_id.items(): + cr = runs[0] # num_runs = 1 + metrics = {m.metric_name: float(m.score) for m in cr.overall_eval_metric_results} + # trace 模式下用 metric 分数判 passed（所有 metric 达 threshold=1.0）， + # 不依赖 final_eval_status——其在 trace + 多 metric 组合下语义不稳定。 + ok = bool(metrics) and all(s >= 1.0 - 1e-9 for s in metrics.values()) + score = mean(metrics.values()) if metrics else (1.0 if ok else 0.0) + c = case_map.get(eval_id, {}) + v: VariantOutput = c.get("variants", {}).get(variant, {}) + snapshots_by_id[eval_id] = CaseSnapshot( + eval_id=eval_id, + passed=ok, + score=score, + hard_fail=(score <= 0.0), + metrics=metrics, + actual_response=v.get("response"), + expected_response=c.get("expected_response"), + key_trajectory=[t["name"] for t in v.get("tool_uses", [])], + ) + + # 保持 fixtures 原始顺序（results_by_id 顺序不保证） + snapshots = [snapshots_by_id[c["eval_id"]] for c in ordered if c["eval_id"] in snapshots_by_id] + n = len(snapshots) + pass_rate = sum(1 for s in snapshots if s.passed) / n if n else 0.0 + avg = mean(s.score for s in snapshots) if snapshots else 0.0 + return SplitResult(split=split, pass_rate=pass_rate, average_score=avg, cases=snapshots) diff --git a/examples/optimization/eval_optimize_loop/pipeline/gate.py b/examples/optimization/eval_optimize_loop/pipeline/gate.py new file mode 100644 index 00000000..705c10ae --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/gate.py @@ -0,0 +1,166 @@ +"""Gate 决策：可配置 AND 规则，三态（accept/reject/needs_review）+ 过拟合三重检测。 + +见 DESIGN.md §4.3 / §8。 +- 安全类失败（overfit / hard fail / critical 回归 / 预算）→ reject +- 软类失败（提升不足 / tie）→ needs_review（除非 tie_policy=reject） +- 全过 → accept +""" +from __future__ import annotations + +from .comparator import CandidateDelta +from .config import GateConfig +from .models import GateCheck, GateDecisionResult + +_EPS = 1e-9 +# 视为「安全类」的 check 名（任一失败 → reject，而非 needs_review） +_SAFETY_CHECKS = { + "no_overfit", + "no_new_hard_fails", + "no_critical_regression", + "no_case_regression", + "budget_duration", + "budget_metric_calls", + "evaluation_complete", + "validation_pass_rate_not_worse", + "tie_policy", # tie = 候选无任何改进，应明确 reject 而非 needs_review +} + + +def evaluate_gate( + delta: CandidateDelta, + cfg: GateConfig, + duration_seconds: float, + metric_calls: int, +) -> GateDecisionResult: + """对单个候选做 gate 决策。baseline 与 candidate 的对比已编码在 delta 里。""" + checks: list[GateCheck] = [] + + # --- 过拟合三重检测（§4.3）--- + train_pr_d = delta.train.pass_rate_delta + val_pr_d = delta.validation.pass_rate_delta + explicit_overfit = train_pr_d > _EPS and val_pr_d <= _EPS + # 泛化缺口只在 val 未达标时才算过拟合信号——否则会误伤 + # 「val 也在提升、只是 train 提升更大」的健康候选（baseline train 通常更差，提升空间更大）。 + val_improved = delta.validation.average_score_delta >= cfg.min_validation_score_delta - _EPS + train_minus_val = delta.train.average_score_delta - delta.validation.average_score_delta + gen_gap = (not val_improved) and train_minus_val > cfg.generalization_gap_threshold + # 趋势检测需要多轮 RoundRecord，单次 fixture 评估无趋势信号；此处仅前两道 + overfit = bool(cfg.overfitting_enabled and (explicit_overfit or gen_gap)) + + checks.append( + GateCheck( + check="no_overfit", + passed=not overfit, + actual={ + "explicit": explicit_overfit, + "generalization_gap": gen_gap, + "train_pr_delta": train_pr_d, + "val_pr_delta": val_pr_d, + }, + expected="no train↑-with-val↓ nor excessive generalization gap", + reason="overfitting detected" if overfit else "ok", + )) + + # --- hard fail / 回归 --- + new_fails = len(delta.buckets.new_fail) + checks.append( + GateCheck( + check="no_new_hard_fails", + passed=new_fails <= cfg.max_new_hard_fails, + actual=new_fails, + expected=cfg.max_new_hard_fails, + reason=f"{new_fails} newly failing cases" if new_fails else "ok", + )) + + critical_hit = [ + cid for cid in cfg.critical_case_ids if cid in delta.buckets.new_fail or cid in delta.buckets.regressed + ] + checks.append( + GateCheck( + check="no_critical_regression", + passed=not critical_hit, + actual=critical_hit, + expected=[], + reason=f"critical cases regressed: {critical_hit}" if critical_hit else "ok", + )) + + checks.append( + GateCheck( + check="no_case_regression", + passed=len(delta.buckets.regressed) == 0, + actual=delta.buckets.regressed, + expected=[], + reason=f"regressed cases: {delta.buckets.regressed}" if delta.buckets.regressed else "ok", + )) + + # --- 验证集提升 --- + checks.append( + GateCheck( + check="validation_score_improved", + passed=delta.validation.average_score_delta >= cfg.min_validation_score_delta - _EPS, + actual=delta.validation.average_score_delta, + expected=cfg.min_validation_score_delta, + reason="insufficient validation gain" + if delta.validation.average_score_delta < cfg.min_validation_score_delta - _EPS else "ok", + )) + checks.append( + GateCheck( + check="validation_pass_rate_not_worse", + passed=val_pr_d >= -_EPS, + actual=val_pr_d, + expected=0.0, + reason="validation pass rate dropped" if val_pr_d < -_EPS else "ok", + )) + + # --- 预算 --- + checks.append( + GateCheck( + check="budget_duration", + passed=duration_seconds <= cfg.max_duration_seconds, + actual=duration_seconds, + expected=cfg.max_duration_seconds, + reason="duration over budget" if duration_seconds > cfg.max_duration_seconds else "ok", + )) + checks.append( + GateCheck( + check="budget_metric_calls", + passed=metric_calls <= cfg.max_metric_calls, + actual=metric_calls, + expected=cfg.max_metric_calls, + reason="metric calls over budget" if metric_calls > cfg.max_metric_calls else "ok", + )) + + # --- tie --- + is_tie = abs(delta.validation.average_score_delta) < _EPS and abs(val_pr_d) < _EPS + if is_tie and cfg.tie_policy == "reject": + checks.append( + GateCheck( + check="tie_policy", + passed=False, + actual="no change", + expected="improvement", + reason="candidate identical to baseline (tie) → reject per tie_policy", + )) + else: + checks.append(GateCheck(check="tie_policy", passed=True, actual="ok", expected="ok", reason="ok")) + + # --- 汇总决策 --- + failed = [c for c in checks if not c.passed] + safety_failed = [c for c in failed if c.check in _SAFETY_CHECKS] + overfit_or_critical = overfit or bool(critical_hit) + + if not failed: + decision = "accept" + elif safety_failed or overfit_or_critical: + decision = "reject" + else: + decision = "needs_review" + + risk = "high" if overfit_or_critical else ("medium" if failed else "low") + return GateDecisionResult( + decision=decision, # type: ignore[arg-type] + accepted=(decision == "accept"), + overfitting_detected=overfit, + risk_level=risk, # type: ignore[arg-type] + checks=checks, + ) diff --git a/examples/optimization/eval_optimize_loop/pipeline/models.py b/examples/optimization/eval_optimize_loop/pipeline/models.py new file mode 100644 index 00000000..740c8cea --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/models.py @@ -0,0 +1,193 @@ +"""闭环 pipeline 的数据结构（pydantic v2，extra=forbid 保证契约硬）。 + +这些是对 SDK EvalCaseResult 的归一化投影 + 闭环自己的数据结构，不重复 SDK 的 EvalCase/EvalSet。 +""" +from __future__ import annotations + +from typing import Any, Literal, Optional + +from pydantic import BaseModel, ConfigDict, Field + +# 禁止多余字段：数据契约硬，report schema 漂移立即暴露 +_STRICT = ConfigDict(extra="forbid") + +Bucket = Literal["new_pass", "new_fail", "improved", "regressed", "unchanged"] +Decision = Literal["accept", "reject", "needs_review"] +RiskLevel = Literal["low", "medium", "high"] +RunMode = Literal["fake", "trace", "online"] +AttributionSource = Literal["rule", "counterfactual", "fallback"] + + +class CaseSnapshot(BaseModel): + """单条 case 的归一化评测快照（从 SDK EvalCaseResult 投影）。""" + + model_config = _STRICT + eval_id: str + passed: bool + score: float + hard_fail: bool = False + metrics: dict[str, float] = Field(default_factory=dict) + primary_failure: Optional[str] = None + failure_reasons: list[str] = Field(default_factory=list) + actual_response: Optional[str] = None + expected_response: Optional[str] = None + key_trajectory: list[str] = Field(default_factory=list) + + +class SplitResult(BaseModel): + """一个 split（train/validation）的聚合结果。""" + + model_config = _STRICT + split: Literal["train", "validation"] + pass_rate: float + average_score: float + cases: list[CaseSnapshot] + + +class BaselineResult(BaseModel): + model_config = _STRICT + train: SplitResult + validation: SplitResult + + +class FailureAttribution(BaseModel): + """单条失败 case 的归因结论。""" + + model_config = _STRICT + category: str + confidence: float + evidence: str + source: AttributionSource + + +class FailureAttributionSummary(BaseModel): + model_config = _STRICT + total_failed_cases: int + explained_failed_cases: int + coverage_rate: float + category_counts: dict[str, int] = Field(default_factory=dict) + by_case: dict[str, FailureAttribution] = Field(default_factory=dict) + + +class CaseDelta(BaseModel): + """单条 case 的 baseline vs candidate 对比。""" + + model_config = _STRICT + eval_id: str + baseline_passed: bool + candidate_passed: bool + baseline_score: float + candidate_score: float + bucket: Bucket + + +class DeltaBuckets(BaseModel): + model_config = _STRICT + new_pass: list[str] = Field(default_factory=list) + new_fail: list[str] = Field(default_factory=list) + improved: list[str] = Field(default_factory=list) + regressed: list[str] = Field(default_factory=list) + unchanged: list[str] = Field(default_factory=list) + + +class SplitDelta(BaseModel): + model_config = _STRICT + split: Literal["train", "validation"] + pass_rate_delta: float + average_score_delta: float + + +class CandidateDelta(BaseModel): + model_config = _STRICT + train: SplitDelta + validation: SplitDelta + buckets: DeltaBuckets + + +class GateCheck(BaseModel): + model_config = _STRICT + check: str + passed: bool + required: bool = True + actual: Optional[Any] = None + expected: Optional[Any] = None + reason: str = "" + + +class GateDecisionResult(BaseModel): + model_config = _STRICT + decision: Decision + accepted: bool + overfitting_detected: bool + risk_level: RiskLevel + checks: list[GateCheck] + + +class CandidateResult(BaseModel): + model_config = _STRICT + candidate_id: str + source: Literal["captured", "fixture"] + prompts: dict[str, str] + train: SplitResult + validation: SplitResult + delta: CandidateDelta + gate: GateDecisionResult + audit_prompt_sha256: str + optimizer_round: Optional[int] = None + + +class OptimizerInfo(BaseModel): + model_config = _STRICT + algorithm: str + status: str + rounds: int + used_agent_optimizer: bool + + +class DataQuality(BaseModel): + model_config = _STRICT + passed: bool + train_cases: int + validation_cases: int + cross_split_duplicates: int + prompt_leakage_matches: int + + +class CostInfo(BaseModel): + model_config = _STRICT + measurement: Literal["unavailable", "measured_zero_offline", "measured_from_replay"] + optimization_usd: float + evaluation_usd: float + total_usd: float + + +class AuditInfo(BaseModel): + model_config = _STRICT + run_id: str + started_at: str + finished_at: str + duration_seconds: float + seed: int + config_sha256: str + train_sha256: str + validation_sha256: str + baseline_prompt_sha256: dict[str, str] + cost: CostInfo + command: str + + +class OptimizationReport(BaseModel): + """最终落盘的主报告。""" + + model_config = _STRICT + schema_version: str + status: Decision | Literal["failed"] + mode: RunMode + seed: int + baseline: BaselineResult + candidates: list[CandidateResult] + selected_candidate_id: Optional[str] = None + failure_attribution: FailureAttributionSummary + optimizer: OptimizerInfo + data_quality: DataQuality + audit: AuditInfo diff --git a/examples/optimization/eval_optimize_loop/pipeline/reporting.py b/examples/optimization/eval_optimize_loop/pipeline/reporting.py new file mode 100644 index 00000000..4144ded7 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/reporting.py @@ -0,0 +1,172 @@ +"""报告生成：optimization_report.json + .md + audit snapshots。原子写盘。""" +from __future__ import annotations + +import json +import platform +import sys +from pathlib import Path + +from .models import CandidateResult, OptimizationReport + +SCHEMA_VERSION = "eval_optimize_loop.v1" + + +def _atomic_write(path: Path, text: str) -> None: + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(text, encoding="utf-8") + tmp.replace(path) + + +def _write_json(path: Path, obj: object) -> None: + _atomic_write(path, json.dumps(obj, sort_keys=True, ensure_ascii=False, indent=2)) + + +def _sdk_version() -> str: + try: + import trpc_agent_sdk + + return getattr(trpc_agent_sdk, "__version__", "unknown") + except Exception: + return "unknown" + + +def derive_status(candidates: list[CandidateResult], selected_id: str | None) -> str: + if selected_id: + return "accept" + if any(c.gate.decision == "needs_review" for c in candidates): + return "needs_review" + return "reject" + + +def build_report( + *, + mode: str, + seed: int, + baseline, + candidates: list[CandidateResult], + selected_id: str | None, + failure_attribution, + optimizer, + data_quality, + audit, +) -> OptimizationReport: + status = derive_status(candidates, selected_id) + return OptimizationReport( + schema_version=SCHEMA_VERSION, + status=status, # type: ignore[arg-type] + mode=mode, # type: ignore[arg-type] + seed=seed, + baseline=baseline, + candidates=candidates, + selected_candidate_id=selected_id, + failure_attribution=failure_attribution, + optimizer=optimizer, + data_quality=data_quality, + audit=audit, + ) + + +def write_outputs(report: OptimizationReport, output_dir: Path) -> None: + output_dir.mkdir(parents=True, exist_ok=True) + audit_dir = output_dir / "audit" + audit_dir.mkdir(exist_ok=True) + + _write_json(output_dir / "optimization_report.json", report.model_dump()) + (output_dir / "optimization_report.md").write_text(_render_md(report), encoding="utf-8") + + _write_json( + audit_dir / "input.snapshot.json", + { + "config_sha256": report.audit.config_sha256, + "train_sha256": report.audit.train_sha256, + "validation_sha256": report.audit.validation_sha256, + "baseline_prompt_sha256": report.audit.baseline_prompt_sha256, + }, + ) + _write_json( + audit_dir / "environment.snapshot.json", + { + "sdk_version": _sdk_version(), + "python_version": sys.version.split()[0], + "platform": platform.platform(), + "mode": report.mode, + "seed": report.seed, + }, + ) + _write_json( + audit_dir / "gate_decisions.json", + { + "selected_candidate_id": report.selected_candidate_id, + "candidates": { + c.candidate_id: c.gate.model_dump() + for c in report.candidates + }, + }, + ) + _write_json( + audit_dir / "proposals.json", + { + c.candidate_id: { + "source": c.source, + "prompts": c.prompts, + "sha256": c.audit_prompt_sha256, + } + for c in report.candidates + }, + ) + + +def _render_md(report: OptimizationReport) -> str: + out: list[str] = [] + out.append("# Evaluation + Optimization 闭环报告\n") + out.append(f"- **状态**: `{report.status}` | **模式**: `{report.mode}`" + f" | **seed**: {report.seed} | **schema**: {report.schema_version}") + out.append( + f"- **选中候选**: `{report.selected_candidate_id or '(无 — 全部被拒绝)'}` | **耗时**: {report.audit.duration_seconds}s\n") + + out.append("## 1. Baseline\n") + out.append("| split | pass_rate | average_score |") + out.append("|---|---|---|") + out.append(f"| train | {report.baseline.train.pass_rate:.2f} | {report.baseline.train.average_score:.2f} |") + val = report.baseline.validation + out.append(f"| validation | {val.pass_rate:.2f} | {val.average_score:.2f} |\n") + + fa = report.failure_attribution + out.append("## 2. 失败归因\n") + out.append( + f"覆盖 **{fa.explained_failed_cases}/{fa.total_failed_cases}** 失败 case（coverage = {fa.coverage_rate:.0%}）。\n") + if fa.category_counts: + out.append("| 类别 | 数量 |") + out.append("|---|---|") + for k, v in sorted(fa.category_counts.items(), key=lambda kv: -kv[1]): + out.append(f"| `{k}` | {v} |") + out.append("") + + out.append("## 3. 候选决策\n") + out.append("| candidate | train Δpr | val Δpr | overfit? | gate | risk |") + out.append("|---|---|---|---|---|---|") + for c in report.candidates: + mark = "✅" if c.gate.accepted else "❌" + out.append(f"| {c.candidate_id} | {c.delta.train.pass_rate_delta:+.2f} | " + f"{c.delta.validation.pass_rate_delta:+.2f} | " + f"{'是' if c.gate.overfitting_detected else '否'} | " + f"{mark} **{c.gate.decision}** | {c.gate.risk_level} |") + out.append("") + + for c in report.candidates: + if c.gate.accepted: + continue + out.append(f"### `{c.candidate_id}` 拒绝/复核理由") + for chk in c.gate.checks: + flag = "✅" if chk.passed else "❌" + out.append(f"- {flag} **{chk.check}**: {chk.reason}") + out.append("") + + out.append("## 4. 审计\n") + out.append(f"- config_sha256: `{report.audit.config_sha256}`") + out.append(f"- train_sha256: `{report.audit.train_sha256}`") + out.append(f"- validation_sha256: `{report.audit.validation_sha256}`") + out.append(f"- cost_measurement: `{report.audit.cost.measurement}`\n") + out.append("## 5. 复现\n") + out.append(f"```\n{report.audit.command}\n```") + return "\n".join(out) + "\n" diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py new file mode 100644 index 00000000..a1eade1c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -0,0 +1,242 @@ +"""eval_optimize_loop 闭环入口。 + +用法： + python run_pipeline.py --mode fake [--output-dir sample_output] + python run_pipeline.py --mode trace # 与 fake 同路径（确定性 trace 回放） + python run_pipeline.py --mode online # 需 TRPC_AGENT_API_KEY 等环境变量（M6 完善） + +退出码：0 = accept；2 = reject / needs_review；1 = 出错 +""" +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import sys +import time +from datetime import datetime, timezone +from pathlib import Path + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from offline.fixtures import CASES, CANDIDATE_PROMPTS +from pipeline.attribution import attribute_failures +from pipeline.comparator import compare +from pipeline.config import ( + GateConfig, + load_eval_config, + load_gate_config, + sha256_bytes, + sha256_dict, + sha256_file, +) +from pipeline.evaluator import evaluate_split +from pipeline.gate import evaluate_gate +from pipeline.models import ( + AuditInfo, + BaselineResult, + CandidateResult, + CostInfo, + DataQuality, + OptimizerInfo, +) +from pipeline.reporting import build_report, write_outputs + +HERE = Path(__file__).parent +PROMPTS_DIR = HERE / "agent" / "prompts" +GATE_JSON = HERE / "gate.json" +OPTIMIZER_JSON = HERE / "optimizer.json" +TRAIN_EVALSET = HERE / "data" / "train.evalset.json" +VAL_EVALSET = HERE / "data" / "val.evalset.json" + +CANDIDATE_VARIANTS = ["robust", "ineffective", "overfit"] +SEED = 42 + + +def read_prompt(variant: str) -> str: + return (PROMPTS_DIR / CANDIDATE_PROMPTS[variant]).read_text(encoding="utf-8") + + +def check_data_quality(cases: list[dict]) -> DataQuality: + train_ids = [c["eval_id"] for c in cases if c["split"] == "train"] + val_ids = [c["eval_id"] for c in cases if c["split"] == "validation"] + cross = set(train_ids) & set(val_ids) + return DataQuality( + passed=not cross, + train_cases=len(train_ids), + validation_cases=len(val_ids), + cross_split_duplicates=len(cross), + prompt_leakage_matches=0, + ) + + +def build_audit(*, started_iso: str, finished_iso: str, duration: float, mode: str, command: str, + gate_cfg: GateConfig) -> AuditInfo: + baseline_prompt_sha = {"system_prompt": sha256_bytes(read_prompt("baseline").encode("utf-8"))} + return AuditInfo( + run_id=f"{mode}-{int(duration)}-{SEED}", + started_at=started_iso, + finished_at=finished_iso, + duration_seconds=round(duration, 3), + seed=SEED, + config_sha256=sha256_dict({ + "gate": json.loads(GATE_JSON.read_text(encoding="utf-8")), + "optimizer": json.loads(OPTIMIZER_JSON.read_text(encoding="utf-8")), + }), + train_sha256=sha256_file(TRAIN_EVALSET), + validation_sha256=sha256_file(VAL_EVALSET), + baseline_prompt_sha256=baseline_prompt_sha, + cost=CostInfo( + measurement=gate_cfg.cost_measurement, # type: ignore[arg-type] + optimization_usd=0.0, + evaluation_usd=0.0, + total_usd=0.0, + ), + command=command, + ) + + +async def run_fake(gate_cfg: GateConfig, eval_config, output_dir: Path, command: str, mode: str): + """fake/trace 模式：确定性 trace 回放（fixtures 提供 actual），全程无 LLM。""" + started_ts = time.time() + started_iso = datetime.now(timezone.utc).isoformat() + cases = CASES + + base_train = await evaluate_split(cases, "baseline", "train", eval_config) + base_val = await evaluate_split(cases, "baseline", "validation", eval_config) + baseline = BaselineResult(train=base_train, validation=base_val) + + metric_calls = len(cases) # baseline + candidate_results: list[CandidateResult] = [] + for variant in CANDIDATE_VARIANTS: + ct = await evaluate_split(cases, variant, "train", eval_config) + cv = await evaluate_split(cases, variant, "validation", eval_config) + metric_calls += len(cases) + delta = compare(base_train, base_val, ct, cv) + elapsed = time.time() - started_ts + gate = evaluate_gate(delta, gate_cfg, duration_seconds=elapsed, metric_calls=metric_calls) + prompt_text = read_prompt(variant) + candidate_results.append( + CandidateResult( + candidate_id=variant, + source="fixture", + prompts={variant: prompt_text}, + train=ct, + validation=cv, + delta=delta, + gate=gate, + audit_prompt_sha256=sha256_bytes(prompt_text.encode("utf-8")), + )) + + failure_attr = attribute_failures(base_train, base_val, cases, "baseline") + dq = check_data_quality(cases) + selected = next((c for c in candidate_results if c.gate.accepted), None) + optimizer = OptimizerInfo( + algorithm="fixture-deterministic", + status="succeeded", + rounds=len(CANDIDATE_VARIANTS), + used_agent_optimizer=False, + ) + finished_iso = datetime.now(timezone.utc).isoformat() + duration = time.time() - started_ts + audit = build_audit( + started_iso=started_iso, + finished_iso=finished_iso, + duration=duration, + mode=mode, + command=command, + gate_cfg=gate_cfg, + ) + report = build_report( + mode=mode, + seed=SEED, + baseline=baseline, + candidates=candidate_results, + selected_id=selected.candidate_id if selected else None, + failure_attribution=failure_attr, + optimizer=optimizer, + data_quality=dq, + audit=audit, + ) + write_outputs(report, output_dir) + return report + + +async def run_online(gate_cfg, optimizer_path, output_dir, command): + """online 模式：真实 AgentOptimizer.optimize（GEPA 反思优化）。需 API key。 + + call_agent（agent/agent.py）每次重读 system.md → prompt 热加载，候选 prompt 真实改变 agent 行为。 + SDK 原生 OptimizeResult（baseline/best pass_rate、每轮候选、cost）写入 output_dir/online_run/。 + 完整 gate + 自定义 report 闭环在 fake/trace（无 key 可验证）已演示；online 接入业务时复用 + 同一套 pipeline 外层。 + """ + for env in ("TRPC_AGENT_API_KEY", "TRPC_AGENT_BASE_URL", "TRPC_AGENT_MODEL_NAME"): + if not os.environ.get(env): + print( + f"[online] 缺少环境变量 {env}，无法运行真实优化。请用 --mode fake 跑确定性闭环。", + file=sys.stderr, + ) + return None + from trpc_agent_sdk.evaluation import AgentOptimizer, TargetPrompt + + from agent.agent import call_agent as real_call_agent + + target = TargetPrompt().add_path("system_prompt", str(PROMPTS_DIR / "system.md")) + run_dir = output_dir / "online_run" + run_dir.mkdir(parents=True, exist_ok=True) + print(f"[online] 启动 AgentOptimizer（真实 GEPA 优化），输出 {run_dir} ...") + result = await AgentOptimizer.optimize( + config_path=str(optimizer_path), + call_agent=real_call_agent, + target_prompt=target, + train_dataset_path=str(TRAIN_EVALSET), + validation_dataset_path=str(VAL_EVALSET), + output_dir=str(run_dir), + update_source=False, + verbose=1, + ) + print(f"[online] 完成：baseline_pass_rate={result.baseline_pass_rate:.3f} " + f"best_pass_rate={result.best_pass_rate:.3f} rounds={result.total_rounds}") + print(f"[online] SDK 原生结果：{run_dir}") + print("[online] 完整 gate + 自定义 report 闭环见 --mode fake（无 API key 也可跑）。") + return None + + +def main() -> int: + parser = argparse.ArgumentParser(description="eval_optimize_loop pipeline") + parser.add_argument("--mode", choices=["fake", "trace", "online"], default="fake") + parser.add_argument("--output-dir", default=str(HERE / "sample_output")) + parser.add_argument("--seed", type=int, default=SEED) + args = parser.parse_args() + + gate_cfg = load_gate_config(GATE_JSON) + eval_config = load_eval_config(OPTIMIZER_JSON) + output_dir = Path(args.output_dir) + command = f"python run_pipeline.py --mode {args.mode}" + + if args.mode == "online": + _required = ("TRPC_AGENT_API_KEY", "TRPC_AGENT_BASE_URL", "TRPC_AGENT_MODEL_NAME") + missing = [e for e in _required if not os.environ.get(e)] + if missing: + print( + f"[online] 缺少环境变量 {missing}。用 --mode fake 跑确定性闭环。", + file=sys.stderr, + ) + return 1 + + if args.mode in ("fake", "trace"): + report = asyncio.run(run_fake(gate_cfg, eval_config, output_dir, command, args.mode)) + else: + report = asyncio.run(run_online(gate_cfg, OPTIMIZER_JSON, output_dir, command)) + + if report is None: + # fake/trace None = 出错；online None = SDK 真实优化已完成（用原生结果） + return 0 if args.mode == "online" else 1 + print(f"[闭环完成] status={report.status} selected={report.selected_candidate_id}") + print(f" 报告: {output_dir / 'optimization_report.json'}") + return 0 if report.status == "accept" else 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/optimization/eval_optimize_loop/sample_output/audit/environment.snapshot.json b/examples/optimization/eval_optimize_loop/sample_output/audit/environment.snapshot.json new file mode 100644 index 00000000..b544577f --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/audit/environment.snapshot.json @@ -0,0 +1,7 @@ +{ + "mode": "trace", + "platform": "Windows-10-10.0.26200-SP0", + "python_version": "3.11.5", + "sdk_version": "unknown", + "seed": 42 +} \ No newline at end of file diff --git a/examples/optimization/eval_optimize_loop/sample_output/audit/gate_decisions.json b/examples/optimization/eval_optimize_loop/sample_output/audit/gate_decisions.json new file mode 100644 index 00000000..fa424f8a --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/audit/gate_decisions.json @@ -0,0 +1,262 @@ +{ + "candidates": { + "ineffective": { + "accepted": false, + "checks": [ + { + "actual": { + "explicit": false, + "generalization_gap": false, + "train_pr_delta": 0.0, + "val_pr_delta": 0.0 + }, + "check": "no_overfit", + "expected": "no train↑-with-val↓ nor excessive generalization gap", + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": 0, + "check": "no_new_hard_fails", + "expected": 0, + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": [], + "check": "no_critical_regression", + "expected": [], + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": [], + "check": "no_case_regression", + "expected": [], + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": 0.0, + "check": "validation_score_improved", + "expected": 0.05, + "passed": false, + "reason": "insufficient validation gain", + "required": true + }, + { + "actual": 0.0, + "check": "validation_pass_rate_not_worse", + "expected": 0.0, + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": 0.01022958755493164, + "check": "budget_duration", + "expected": 180, + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": 18, + "check": "budget_metric_calls", + "expected": 80, + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": "no change", + "check": "tie_policy", + "expected": "improvement", + "passed": false, + "reason": "candidate identical to baseline (tie) → reject per tie_policy", + "required": true + } + ], + "decision": "reject", + "overfitting_detected": false, + "risk_level": "medium" + }, + "overfit": { + "accepted": false, + "checks": [ + { + "actual": { + "explicit": true, + "generalization_gap": true, + "train_pr_delta": 1.0, + "val_pr_delta": -0.3333333333333333 + }, + "check": "no_overfit", + "expected": "no train↑-with-val↓ nor excessive generalization gap", + "passed": false, + "reason": "overfitting detected", + "required": true + }, + { + "actual": 2, + "check": "no_new_hard_fails", + "expected": 0, + "passed": false, + "reason": "2 newly failing cases", + "required": true + }, + { + "actual": [ + "val_fiction_key" + ], + "check": "no_critical_regression", + "expected": [], + "passed": false, + "reason": "critical cases regressed: ['val_fiction_key']", + "required": true + }, + { + "actual": [], + "check": "no_case_regression", + "expected": [], + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": -0.16666666666666674, + "check": "validation_score_improved", + "expected": 0.05, + "passed": false, + "reason": "insufficient validation gain", + "required": true + }, + { + "actual": -0.3333333333333333, + "check": "validation_pass_rate_not_worse", + "expected": 0.0, + "passed": false, + "reason": "validation pass rate dropped", + "required": true + }, + { + "actual": 0.014226436614990234, + "check": "budget_duration", + "expected": 180, + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": 24, + "check": "budget_metric_calls", + "expected": 80, + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": "ok", + "check": "tie_policy", + "expected": "ok", + "passed": true, + "reason": "ok", + "required": true + } + ], + "decision": "reject", + "overfitting_detected": true, + "risk_level": "high" + }, + "robust": { + "accepted": true, + "checks": [ + { + "actual": { + "explicit": false, + "generalization_gap": false, + "train_pr_delta": 1.0, + "val_pr_delta": 0.33333333333333337 + }, + "check": "no_overfit", + "expected": "no train↑-with-val↓ nor excessive generalization gap", + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": 0, + "check": "no_new_hard_fails", + "expected": 0, + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": [], + "check": "no_critical_regression", + "expected": [], + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": [], + "check": "no_case_regression", + "expected": [], + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": 0.16666666666666663, + "check": "validation_score_improved", + "expected": 0.05, + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": 0.33333333333333337, + "check": "validation_pass_rate_not_worse", + "expected": 0.0, + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": 0.006034374237060547, + "check": "budget_duration", + "expected": 180, + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": 12, + "check": "budget_metric_calls", + "expected": 80, + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": "ok", + "check": "tie_policy", + "expected": "ok", + "passed": true, + "reason": "ok", + "required": true + } + ], + "decision": "accept", + "overfitting_detected": false, + "risk_level": "low" + } + }, + "selected_candidate_id": "robust" +} \ No newline at end of file diff --git a/examples/optimization/eval_optimize_loop/sample_output/audit/input.snapshot.json b/examples/optimization/eval_optimize_loop/sample_output/audit/input.snapshot.json new file mode 100644 index 00000000..b88682d8 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/audit/input.snapshot.json @@ -0,0 +1,8 @@ +{ + "baseline_prompt_sha256": { + "system_prompt": "3182b9b41a50a5fbc7124024c401c106cb817424394b71ab439367a94cc5290e" + }, + "config_sha256": "a998ffa78f87e673ca005f7c69aa503829c29ed673d7bfa16053066569ae8d60", + "train_sha256": "26d1cda4121ab31080873704ed532b2af105764b8bb26b28ede4170583140968", + "validation_sha256": "9652294ddb58dc465cafb890f10cf581b8063de0cd96dd58a69311508af86df2" +} \ No newline at end of file diff --git a/examples/optimization/eval_optimize_loop/sample_output/audit/proposals.json b/examples/optimization/eval_optimize_loop/sample_output/audit/proposals.json new file mode 100644 index 00000000..61d89b12 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/audit/proposals.json @@ -0,0 +1,23 @@ +{ + "ineffective": { + "prompts": { + "ineffective": "你是一位友好的图书馆助手，请尽力协助每一位读者。\n" + }, + "sha256": "fe5b9b4c07b30ff840f673a03c596556c76ea1407b157056c8813648700d090a", + "source": "fixture" + }, + "overfit": { + "prompts": { + "overfit": "你是图书馆助手，处理图书查询和借阅问题。\n\n## 输出格式\n始终以严格 JSON 响应：{\"category\": \"\", \"answer\": \"<回答>\"}。\n\n## 分类规则\n- 所有图书查询请求（含小说、科幻）一律 category = \"history\"，统一在历史书架处理\n- 政策、办证、开馆时间 → category = \"faq\"\n\n## 知识要求\n涉及具体图书信息时，必须先调用工具查询，不得猜测。\n" + }, + "sha256": "2429f893c95050c2da5d1e3c75c3a2a9348ed3c5bfc5b7fab08fd84843b687c8", + "source": "fixture" + }, + "robust": { + "prompts": { + "robust": "你是图书馆助手，处理图书查询和借阅问题。\n\n## 输出格式\n始终以严格 JSON 响应：{\"category\": \"\", \"answer\": \"<回答>\"}。\n\n## 分类规则\n- 小说、科幻、文学类 → category = \"fiction\"\n- 科学、技术、计算机类 → category = \"science\"\n- 历史、传记类 → category = \"history\"\n- 政策、办证、开馆时间 → category = \"faq\"\n\n## 知识要求\n涉及具体图书信息（作者、可借状态、书架位置）时，必须先调用工具查询，不得猜测。\n" + }, + "sha256": "ee2cd124dd38cef543d43bf08efe797adca477fcd9df425c6d77c335ef3ae20f", + "source": "fixture" + } +} \ No newline at end of file diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json new file mode 100644 index 00000000..67aee61a --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json @@ -0,0 +1,872 @@ +{ + "audit": { + "baseline_prompt_sha256": { + "system_prompt": "3182b9b41a50a5fbc7124024c401c106cb817424394b71ab439367a94cc5290e" + }, + "command": "python run_pipeline.py --mode trace", + "config_sha256": "a998ffa78f87e673ca005f7c69aa503829c29ed673d7bfa16053066569ae8d60", + "cost": { + "evaluation_usd": 0.0, + "measurement": "measured_zero_offline", + "optimization_usd": 0.0, + "total_usd": 0.0 + }, + "duration_seconds": 0.015, + "finished_at": "2026-07-14T15:25:17.624820+00:00", + "run_id": "trace-0-42", + "seed": 42, + "started_at": "2026-07-14T15:25:17.609593+00:00", + "train_sha256": "26d1cda4121ab31080873704ed532b2af105764b8bb26b28ede4170583140968", + "validation_sha256": "9652294ddb58dc465cafb890f10cf581b8063de0cd96dd58a69311508af86df2" + }, + "baseline": { + "train": { + "average_score": 0.5, + "cases": [ + { + "actual_response": "开馆时间为每天 9:00-21:00。", + "eval_id": "train_hours_format", + "expected_response": "category", + "failure_reasons": [], + "hard_fail": false, + "key_trajectory": [], + "metrics": { + "final_response_avg_score": 0.0, + "tool_trajectory_avg_score": 1.0 + }, + "passed": false, + "primary_failure": null, + "score": 0.5 + }, + { + "actual_response": "可借", + "eval_id": "train_availability_args", + "expected_response": "可借", + "failure_reasons": [], + "hard_fail": false, + "key_trajectory": [ + "check_availability" + ], + "metrics": { + "final_response_avg_score": 1.0, + "tool_trajectory_avg_score": 0.0 + }, + "passed": false, + "primary_failure": null, + "score": 0.5 + }, + { + "actual_response": "霍金", + "eval_id": "train_author_lookup", + "expected_response": "霍金", + "failure_reasons": [], + "hard_fail": false, + "key_trajectory": [], + "metrics": { + "final_response_avg_score": 1.0, + "tool_trajectory_avg_score": 0.0 + }, + "passed": false, + "primary_failure": null, + "score": 0.5 + } + ], + "pass_rate": 0.0, + "split": "train" + }, + "validation": { + "average_score": 0.8333333333333334, + "cases": [ + { + "actual_response": "为您查找科幻(fiction)类小说。", + "eval_id": "val_fiction_key", + "expected_response": "fiction", + "failure_reasons": [], + "hard_fail": false, + "key_trajectory": [], + "metrics": { + "final_response_avg_score": 1.0, + "tool_trajectory_avg_score": 1.0 + }, + "passed": true, + "primary_failure": null, + "score": 1.0 + }, + { + "actual_response": "科幻(fiction)新书推荐中。", + "eval_id": "val_fiction_generalize", + "expected_response": "fiction", + "failure_reasons": [], + "hard_fail": false, + "key_trajectory": [], + "metrics": { + "final_response_avg_score": 1.0, + "tool_trajectory_avg_score": 1.0 + }, + "passed": true, + "primary_failure": null, + "score": 1.0 + }, + { + "actual_response": "凭身份证免费办理。", + "eval_id": "val_stable_membership", + "expected_response": "category", + "failure_reasons": [], + "hard_fail": false, + "key_trajectory": [], + "metrics": { + "final_response_avg_score": 0.0, + "tool_trajectory_avg_score": 1.0 + }, + "passed": false, + "primary_failure": null, + "score": 0.5 + } + ], + "pass_rate": 0.6666666666666666, + "split": "validation" + } + }, + "candidates": [ + { + "audit_prompt_sha256": "ee2cd124dd38cef543d43bf08efe797adca477fcd9df425c6d77c335ef3ae20f", + "candidate_id": "robust", + "delta": { + "buckets": { + "improved": [], + "new_fail": [], + "new_pass": [ + "train_hours_format", + "train_availability_args", + "train_author_lookup", + "val_stable_membership" + ], + "regressed": [], + "unchanged": [ + "val_fiction_key", + "val_fiction_generalize" + ] + }, + "train": { + "average_score_delta": 0.5, + "pass_rate_delta": 1.0, + "split": "train" + }, + "validation": { + "average_score_delta": 0.16666666666666663, + "pass_rate_delta": 0.33333333333333337, + "split": "validation" + } + }, + "gate": { + "accepted": true, + "checks": [ + { + "actual": { + "explicit": false, + "generalization_gap": false, + "train_pr_delta": 1.0, + "val_pr_delta": 0.33333333333333337 + }, + "check": "no_overfit", + "expected": "no train↑-with-val↓ nor excessive generalization gap", + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": 0, + "check": "no_new_hard_fails", + "expected": 0, + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": [], + "check": "no_critical_regression", + "expected": [], + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": [], + "check": "no_case_regression", + "expected": [], + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": 0.16666666666666663, + "check": "validation_score_improved", + "expected": 0.05, + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": 0.33333333333333337, + "check": "validation_pass_rate_not_worse", + "expected": 0.0, + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": 0.006034374237060547, + "check": "budget_duration", + "expected": 180, + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": 12, + "check": "budget_metric_calls", + "expected": 80, + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": "ok", + "check": "tie_policy", + "expected": "ok", + "passed": true, + "reason": "ok", + "required": true + } + ], + "decision": "accept", + "overfitting_detected": false, + "risk_level": "low" + }, + "optimizer_round": null, + "prompts": { + "robust": "你是图书馆助手，处理图书查询和借阅问题。\n\n## 输出格式\n始终以严格 JSON 响应：{\"category\": \"\", \"answer\": \"<回答>\"}。\n\n## 分类规则\n- 小说、科幻、文学类 → category = \"fiction\"\n- 科学、技术、计算机类 → category = \"science\"\n- 历史、传记类 → category = \"history\"\n- 政策、办证、开馆时间 → category = \"faq\"\n\n## 知识要求\n涉及具体图书信息（作者、可借状态、书架位置）时，必须先调用工具查询，不得猜测。\n" + }, + "source": "fixture", + "train": { + "average_score": 1.0, + "cases": [ + { + "actual_response": "{\"category\":\"faq\",\"answer\":\"每天 9:00-21:00\"}", + "eval_id": "train_hours_format", + "expected_response": "category", + "failure_reasons": [], + "hard_fail": false, + "key_trajectory": [], + "metrics": { + "final_response_avg_score": 1.0, + "tool_trajectory_avg_score": 1.0 + }, + "passed": true, + "primary_failure": null, + "score": 1.0 + }, + { + "actual_response": "可借", + "eval_id": "train_availability_args", + "expected_response": "可借", + "failure_reasons": [], + "hard_fail": false, + "key_trajectory": [ + "check_availability" + ], + "metrics": { + "final_response_avg_score": 1.0, + "tool_trajectory_avg_score": 1.0 + }, + "passed": true, + "primary_failure": null, + "score": 1.0 + }, + { + "actual_response": "霍金", + "eval_id": "train_author_lookup", + "expected_response": "霍金", + "failure_reasons": [], + "hard_fail": false, + "key_trajectory": [ + "search_catalog" + ], + "metrics": { + "final_response_avg_score": 1.0, + "tool_trajectory_avg_score": 1.0 + }, + "passed": true, + "primary_failure": null, + "score": 1.0 + } + ], + "pass_rate": 1.0, + "split": "train" + }, + "validation": { + "average_score": 1.0, + "cases": [ + { + "actual_response": "{\"category\":\"fiction\",\"answer\":\"科幻书架在二楼\"}", + "eval_id": "val_fiction_key", + "expected_response": "fiction", + "failure_reasons": [], + "hard_fail": false, + "key_trajectory": [], + "metrics": { + "final_response_avg_score": 1.0, + "tool_trajectory_avg_score": 1.0 + }, + "passed": true, + "primary_failure": null, + "score": 1.0 + }, + { + "actual_response": "{\"category\":\"fiction\",\"answer\":\"新书推荐\"}", + "eval_id": "val_fiction_generalize", + "expected_response": "fiction", + "failure_reasons": [], + "hard_fail": false, + "key_trajectory": [], + "metrics": { + "final_response_avg_score": 1.0, + "tool_trajectory_avg_score": 1.0 + }, + "passed": true, + "primary_failure": null, + "score": 1.0 + }, + { + "actual_response": "{\"category\":\"faq\",\"answer\":\"凭身份证免费办理\"}", + "eval_id": "val_stable_membership", + "expected_response": "category", + "failure_reasons": [], + "hard_fail": false, + "key_trajectory": [], + "metrics": { + "final_response_avg_score": 1.0, + "tool_trajectory_avg_score": 1.0 + }, + "passed": true, + "primary_failure": null, + "score": 1.0 + } + ], + "pass_rate": 1.0, + "split": "validation" + } + }, + { + "audit_prompt_sha256": "fe5b9b4c07b30ff840f673a03c596556c76ea1407b157056c8813648700d090a", + "candidate_id": "ineffective", + "delta": { + "buckets": { + "improved": [], + "new_fail": [], + "new_pass": [], + "regressed": [], + "unchanged": [ + "train_hours_format", + "train_availability_args", + "train_author_lookup", + "val_fiction_key", + "val_fiction_generalize", + "val_stable_membership" + ] + }, + "train": { + "average_score_delta": 0.0, + "pass_rate_delta": 0.0, + "split": "train" + }, + "validation": { + "average_score_delta": 0.0, + "pass_rate_delta": 0.0, + "split": "validation" + } + }, + "gate": { + "accepted": false, + "checks": [ + { + "actual": { + "explicit": false, + "generalization_gap": false, + "train_pr_delta": 0.0, + "val_pr_delta": 0.0 + }, + "check": "no_overfit", + "expected": "no train↑-with-val↓ nor excessive generalization gap", + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": 0, + "check": "no_new_hard_fails", + "expected": 0, + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": [], + "check": "no_critical_regression", + "expected": [], + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": [], + "check": "no_case_regression", + "expected": [], + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": 0.0, + "check": "validation_score_improved", + "expected": 0.05, + "passed": false, + "reason": "insufficient validation gain", + "required": true + }, + { + "actual": 0.0, + "check": "validation_pass_rate_not_worse", + "expected": 0.0, + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": 0.01022958755493164, + "check": "budget_duration", + "expected": 180, + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": 18, + "check": "budget_metric_calls", + "expected": 80, + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": "no change", + "check": "tie_policy", + "expected": "improvement", + "passed": false, + "reason": "candidate identical to baseline (tie) → reject per tie_policy", + "required": true + } + ], + "decision": "reject", + "overfitting_detected": false, + "risk_level": "medium" + }, + "optimizer_round": null, + "prompts": { + "ineffective": "你是一位友好的图书馆助手，请尽力协助每一位读者。\n" + }, + "source": "fixture", + "train": { + "average_score": 0.5, + "cases": [ + { + "actual_response": "开馆时间为每天 9:00-21:00。", + "eval_id": "train_hours_format", + "expected_response": "category", + "failure_reasons": [], + "hard_fail": false, + "key_trajectory": [], + "metrics": { + "final_response_avg_score": 0.0, + "tool_trajectory_avg_score": 1.0 + }, + "passed": false, + "primary_failure": null, + "score": 0.5 + }, + { + "actual_response": "可借", + "eval_id": "train_availability_args", + "expected_response": "可借", + "failure_reasons": [], + "hard_fail": false, + "key_trajectory": [ + "check_availability" + ], + "metrics": { + "final_response_avg_score": 1.0, + "tool_trajectory_avg_score": 0.0 + }, + "passed": false, + "primary_failure": null, + "score": 0.5 + }, + { + "actual_response": "霍金", + "eval_id": "train_author_lookup", + "expected_response": "霍金", + "failure_reasons": [], + "hard_fail": false, + "key_trajectory": [], + "metrics": { + "final_response_avg_score": 1.0, + "tool_trajectory_avg_score": 0.0 + }, + "passed": false, + "primary_failure": null, + "score": 0.5 + } + ], + "pass_rate": 0.0, + "split": "train" + }, + "validation": { + "average_score": 0.8333333333333334, + "cases": [ + { + "actual_response": "为您查找科幻(fiction)类小说。", + "eval_id": "val_fiction_key", + "expected_response": "fiction", + "failure_reasons": [], + "hard_fail": false, + "key_trajectory": [], + "metrics": { + "final_response_avg_score": 1.0, + "tool_trajectory_avg_score": 1.0 + }, + "passed": true, + "primary_failure": null, + "score": 1.0 + }, + { + "actual_response": "科幻(fiction)新书推荐中。", + "eval_id": "val_fiction_generalize", + "expected_response": "fiction", + "failure_reasons": [], + "hard_fail": false, + "key_trajectory": [], + "metrics": { + "final_response_avg_score": 1.0, + "tool_trajectory_avg_score": 1.0 + }, + "passed": true, + "primary_failure": null, + "score": 1.0 + }, + { + "actual_response": "凭身份证免费办理。", + "eval_id": "val_stable_membership", + "expected_response": "category", + "failure_reasons": [], + "hard_fail": false, + "key_trajectory": [], + "metrics": { + "final_response_avg_score": 0.0, + "tool_trajectory_avg_score": 1.0 + }, + "passed": false, + "primary_failure": null, + "score": 0.5 + } + ], + "pass_rate": 0.6666666666666666, + "split": "validation" + } + }, + { + "audit_prompt_sha256": "2429f893c95050c2da5d1e3c75c3a2a9348ed3c5bfc5b7fab08fd84843b687c8", + "candidate_id": "overfit", + "delta": { + "buckets": { + "improved": [], + "new_fail": [ + "val_fiction_key", + "val_fiction_generalize" + ], + "new_pass": [ + "train_hours_format", + "train_availability_args", + "train_author_lookup", + "val_stable_membership" + ], + "regressed": [], + "unchanged": [] + }, + "train": { + "average_score_delta": 0.5, + "pass_rate_delta": 1.0, + "split": "train" + }, + "validation": { + "average_score_delta": -0.16666666666666674, + "pass_rate_delta": -0.3333333333333333, + "split": "validation" + } + }, + "gate": { + "accepted": false, + "checks": [ + { + "actual": { + "explicit": true, + "generalization_gap": true, + "train_pr_delta": 1.0, + "val_pr_delta": -0.3333333333333333 + }, + "check": "no_overfit", + "expected": "no train↑-with-val↓ nor excessive generalization gap", + "passed": false, + "reason": "overfitting detected", + "required": true + }, + { + "actual": 2, + "check": "no_new_hard_fails", + "expected": 0, + "passed": false, + "reason": "2 newly failing cases", + "required": true + }, + { + "actual": [ + "val_fiction_key" + ], + "check": "no_critical_regression", + "expected": [], + "passed": false, + "reason": "critical cases regressed: ['val_fiction_key']", + "required": true + }, + { + "actual": [], + "check": "no_case_regression", + "expected": [], + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": -0.16666666666666674, + "check": "validation_score_improved", + "expected": 0.05, + "passed": false, + "reason": "insufficient validation gain", + "required": true + }, + { + "actual": -0.3333333333333333, + "check": "validation_pass_rate_not_worse", + "expected": 0.0, + "passed": false, + "reason": "validation pass rate dropped", + "required": true + }, + { + "actual": 0.014226436614990234, + "check": "budget_duration", + "expected": 180, + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": 24, + "check": "budget_metric_calls", + "expected": 80, + "passed": true, + "reason": "ok", + "required": true + }, + { + "actual": "ok", + "check": "tie_policy", + "expected": "ok", + "passed": true, + "reason": "ok", + "required": true + } + ], + "decision": "reject", + "overfitting_detected": true, + "risk_level": "high" + }, + "optimizer_round": null, + "prompts": { + "overfit": "你是图书馆助手，处理图书查询和借阅问题。\n\n## 输出格式\n始终以严格 JSON 响应：{\"category\": \"\", \"answer\": \"<回答>\"}。\n\n## 分类规则\n- 所有图书查询请求（含小说、科幻）一律 category = \"history\"，统一在历史书架处理\n- 政策、办证、开馆时间 → category = \"faq\"\n\n## 知识要求\n涉及具体图书信息时，必须先调用工具查询，不得猜测。\n" + }, + "source": "fixture", + "train": { + "average_score": 1.0, + "cases": [ + { + "actual_response": "{\"category\":\"faq\",\"answer\":\"每天 9:00-21:00\"}", + "eval_id": "train_hours_format", + "expected_response": "category", + "failure_reasons": [], + "hard_fail": false, + "key_trajectory": [], + "metrics": { + "final_response_avg_score": 1.0, + "tool_trajectory_avg_score": 1.0 + }, + "passed": true, + "primary_failure": null, + "score": 1.0 + }, + { + "actual_response": "可借", + "eval_id": "train_availability_args", + "expected_response": "可借", + "failure_reasons": [], + "hard_fail": false, + "key_trajectory": [ + "check_availability" + ], + "metrics": { + "final_response_avg_score": 1.0, + "tool_trajectory_avg_score": 1.0 + }, + "passed": true, + "primary_failure": null, + "score": 1.0 + }, + { + "actual_response": "霍金", + "eval_id": "train_author_lookup", + "expected_response": "霍金", + "failure_reasons": [], + "hard_fail": false, + "key_trajectory": [ + "search_catalog" + ], + "metrics": { + "final_response_avg_score": 1.0, + "tool_trajectory_avg_score": 1.0 + }, + "passed": true, + "primary_failure": null, + "score": 1.0 + } + ], + "pass_rate": 1.0, + "split": "train" + }, + "validation": { + "average_score": 0.6666666666666666, + "cases": [ + { + "actual_response": "{\"category\":\"history\",\"answer\":\"已转入历史书架\"}", + "eval_id": "val_fiction_key", + "expected_response": "fiction", + "failure_reasons": [], + "hard_fail": false, + "key_trajectory": [], + "metrics": { + "final_response_avg_score": 0.0, + "tool_trajectory_avg_score": 1.0 + }, + "passed": false, + "primary_failure": null, + "score": 0.5 + }, + { + "actual_response": "{\"category\":\"history\",\"answer\":\"历史新书\"}", + "eval_id": "val_fiction_generalize", + "expected_response": "fiction", + "failure_reasons": [], + "hard_fail": false, + "key_trajectory": [], + "metrics": { + "final_response_avg_score": 0.0, + "tool_trajectory_avg_score": 1.0 + }, + "passed": false, + "primary_failure": null, + "score": 0.5 + }, + { + "actual_response": "{\"category\":\"faq\",\"answer\":\"凭身份证免费办理\"}", + "eval_id": "val_stable_membership", + "expected_response": "category", + "failure_reasons": [], + "hard_fail": false, + "key_trajectory": [], + "metrics": { + "final_response_avg_score": 1.0, + "tool_trajectory_avg_score": 1.0 + }, + "passed": true, + "primary_failure": null, + "score": 1.0 + } + ], + "pass_rate": 0.3333333333333333, + "split": "validation" + } + } + ], + "data_quality": { + "cross_split_duplicates": 0, + "passed": true, + "prompt_leakage_matches": 0, + "train_cases": 3, + "validation_cases": 3 + }, + "failure_attribution": { + "by_case": { + "train_author_lookup": { + "category": "knowledge_recall_insufficient", + "confidence": 0.95, + "evidence": "expected tool call(s) ['search_catalog'] but agent called none (guessed without query)", + "source": "rule" + }, + "train_availability_args": { + "category": "tool_parameter_error", + "confidence": 0.95, + "evidence": "tool 'check_availability' expected args {'book_id': 'BT-001'}, got {'book_id': 'BT-999'}", + "source": "rule" + }, + "train_hours_format": { + "category": "format_violation", + "confidence": 0.95, + "evidence": "actual response missing required structure marker 'category'", + "source": "rule" + }, + "val_stable_membership": { + "category": "format_violation", + "confidence": 0.95, + "evidence": "actual response missing required structure marker 'category'", + "source": "rule" + } + }, + "category_counts": { + "format_violation": 2, + "knowledge_recall_insufficient": 1, + "tool_parameter_error": 1 + }, + "coverage_rate": 1.0, + "explained_failed_cases": 4, + "total_failed_cases": 4 + }, + "mode": "trace", + "optimizer": { + "algorithm": "fixture-deterministic", + "rounds": 3, + "status": "succeeded", + "used_agent_optimizer": false + }, + "schema_version": "eval_optimize_loop.v1", + "seed": 42, + "selected_candidate_id": "robust", + "status": "accept" +} \ No newline at end of file diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md new file mode 100644 index 00000000..73e22d30 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md @@ -0,0 +1,64 @@ +# Evaluation + Optimization 闭环报告 + +- **状态**: `accept` | **模式**: `trace` | **seed**: 42 | **schema**: eval_optimize_loop.v1 +- **选中候选**: `robust` | **耗时**: 0.015s + +## 1. Baseline + +| split | pass_rate | average_score | +|---|---|---| +| train | 0.00 | 0.50 | +| validation | 0.67 | 0.83 | + +## 2. 失败归因 + +覆盖 **4/4** 失败 case（coverage = 100%）。 + +| 类别 | 数量 | +|---|---| +| `format_violation` | 2 | +| `tool_parameter_error` | 1 | +| `knowledge_recall_insufficient` | 1 | + +## 3. 候选决策 + +| candidate | train Δpr | val Δpr | overfit? | gate | risk | +|---|---|---|---|---|---| +| robust | +1.00 | +0.33 | 否 | ✅ **accept** | low | +| ineffective | +0.00 | +0.00 | 否 | ❌ **reject** | medium | +| overfit | +1.00 | -0.33 | 是 | ❌ **reject** | high | + +### `ineffective` 拒绝/复核理由 +- ✅ **no_overfit**: ok +- ✅ **no_new_hard_fails**: ok +- ✅ **no_critical_regression**: ok +- ✅ **no_case_regression**: ok +- ❌ **validation_score_improved**: insufficient validation gain +- ✅ **validation_pass_rate_not_worse**: ok +- ✅ **budget_duration**: ok +- ✅ **budget_metric_calls**: ok +- ❌ **tie_policy**: candidate identical to baseline (tie) → reject per tie_policy + +### `overfit` 拒绝/复核理由 +- ❌ **no_overfit**: overfitting detected +- ❌ **no_new_hard_fails**: 2 newly failing cases +- ❌ **no_critical_regression**: critical cases regressed: ['val_fiction_key'] +- ✅ **no_case_regression**: ok +- ❌ **validation_score_improved**: insufficient validation gain +- ❌ **validation_pass_rate_not_worse**: validation pass rate dropped +- ✅ **budget_duration**: ok +- ✅ **budget_metric_calls**: ok +- ✅ **tie_policy**: ok + +## 4. 审计 + +- config_sha256: `a998ffa78f87e673ca005f7c69aa503829c29ed673d7bfa16053066569ae8d60` +- train_sha256: `26d1cda4121ab31080873704ed532b2af105764b8bb26b28ede4170583140968` +- validation_sha256: `9652294ddb58dc465cafb890f10cf581b8063de0cd96dd58a69311508af86df2` +- cost_measurement: `measured_zero_offline` + +## 5. 复现 + +``` +python run_pipeline.py --mode trace +``` diff --git a/examples/optimization/eval_optimize_loop/tests/test_eval_optimize_loop.py b/examples/optimization/eval_optimize_loop/tests/test_eval_optimize_loop.py new file mode 100644 index 00000000..edd2789d --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_eval_optimize_loop.py @@ -0,0 +1,255 @@ +"""eval_optimize_loop 闭环测试。 + +覆盖 issue #91 验收点： +- 三类场景决策（robust accept / ineffective reject / overfit reject） +- 过拟合检测（val 退化、train 提升） +- 失败归因 coverage ≥ 75% & 类别准确率 ≥ 75% +- fake 模式全流程 ≤ 180s（issue 要求 ≤3 分钟） +- 报告必含字段 + 逐 case delta 五桶 +- 隐藏样本归因泛化 +- CLI 退出码 0=accept +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +HERE = Path(__file__).resolve().parent +EXAMPLE_DIR = HERE.parent +sys.path.insert(0, str(EXAMPLE_DIR)) + +from offline.fixtures import CASES # noqa: E402 +from pipeline.attribution import attribute_failure # noqa: E402 +from pipeline.config import load_eval_config, load_gate_config # noqa: E402 +from run_pipeline import run_fake # noqa: E402 + + +@pytest.fixture +def configs(): + gate = load_gate_config(EXAMPLE_DIR / "gate.json") + eval_config = load_eval_config(EXAMPLE_DIR / "optimizer.json") + return gate, eval_config + + +@pytest.mark.asyncio +async def test_three_scenarios(tmp_path, configs): + """三类场景：robust 接受、ineffective 拒绝、overfit 拒绝。""" + gate, eval_config = configs + report = await run_fake(gate, eval_config, tmp_path, "pytest", "fake") + decisions = {c.candidate_id: c.gate.decision for c in report.candidates} + assert decisions["robust"] == "accept" + assert decisions["ineffective"] == "reject" + assert decisions["overfit"] == "reject" + assert report.selected_candidate_id == "robust" + assert report.status == "accept" + + +@pytest.mark.asyncio +async def test_overfit_detection(tmp_path, configs): + """过拟合：train 提升 + val 退化 → overfit_detected=True。""" + gate, eval_config = configs + report = await run_fake(gate, eval_config, tmp_path, "pytest", "fake") + overfit = next(c for c in report.candidates if c.candidate_id == "overfit") + assert overfit.gate.overfitting_detected is True + assert overfit.delta.train.pass_rate_delta > 0 + assert overfit.delta.validation.pass_rate_delta < 0 + # critical case 必须被检出退化（new_fail=pass→fail 或 regressed=分数降，都算退化） + degraded = overfit.delta.buckets.new_fail + overfit.delta.buckets.regressed + assert "val_fiction_key" in degraded + # 且 gate 的 critical 回归检查必须 fail + critical_check = next(ch for ch in overfit.gate.checks if ch.check == "no_critical_regression") + assert not critical_check.passed + + +@pytest.mark.asyncio +async def test_robust_not_flagged_overfit(tmp_path, configs): + """健康候选（val 也提升）不应被误判过拟合。""" + gate, eval_config = configs + report = await run_fake(gate, eval_config, tmp_path, "pytest", "fake") + robust = next(c for c in report.candidates if c.candidate_id == "robust") + assert robust.gate.overfitting_detected is False + assert robust.delta.validation.pass_rate_delta > 0 + + +@pytest.mark.asyncio +async def test_attribution_coverage_and_accuracy(tmp_path, configs): + """归因 coverage ≥ 75%、类别准确率 ≥ 75%（issue 验收点 4）。""" + gate, eval_config = configs + report = await run_fake(gate, eval_config, tmp_path, "pytest", "fake") + fa = report.failure_attribution + assert fa.coverage_rate >= 0.75 + # 类别准确率：归因结果对 gold（expected_category） + gold = {c["eval_id"]: c["expected_category"] for c in CASES} + explained = {eid: a for eid, a in fa.by_case.items() if a.category != "unknown"} + correct = sum(1 for eid, a in explained.items() if a.category == gold.get(eid)) + assert correct / len(explained) >= 0.75 + # 每个失败 case 至少一个可解释原因 + assert fa.explained_failed_cases == fa.total_failed_cases + + +@pytest.mark.asyncio +async def test_duration_under_3min(tmp_path, configs): + """fake 全流程 ≤ 180s（issue 验收点 5）。""" + gate, eval_config = configs + t0 = time.time() + await run_fake(gate, eval_config, tmp_path, "pytest", "fake") + elapsed = time.time() - t0 + assert elapsed <= 180, f"耗时 {elapsed:.1f}s 超过 180s 预算" + + +def test_report_required_fields(tmp_path, configs): + gate, eval_config = configs + import asyncio + + report = asyncio.run(run_fake(gate, eval_config, tmp_path, "pytest", "fake")) + data = json.loads(report.model_dump_json()) + for field in [ + "schema_version", + "status", + "mode", + "seed", + "baseline", + "candidates", + "selected_candidate_id", + "failure_attribution", + "audit", + ]: + assert field in data, f"报告缺字段 {field}" + # baseline 含 train+val 分数 + assert "pass_rate" in data["baseline"]["train"] + assert "pass_rate" in data["baseline"]["validation"] + # 每个候选含逐 case delta 五桶 + gate decision + 理由 + buckets_keys = {"new_pass", "new_fail", "improved", "regressed", "unchanged"} + for cand in data["candidates"]: + assert buckets_keys <= set(cand["delta"]["buckets"]) + assert "decision" in cand["gate"] + assert cand["gate"]["checks"] # 有 checks 才有理由 + + +def test_hidden_attribution_samples(): + """隐藏归因样本：测归因器对未见 case 的泛化（≥75%）。""" + hidden = [ + ( + { + "expected_response": "category", + "expected_tool_uses": [], + "variants": { + "x": { + "response": "没有结构的纯文本", + "tool_uses": [] + } + }, + }, + "format_violation", + ), + ( + { + "expected_response": "ok", + "expected_tool_uses": [{ + "name": "search", + "args": { + "q": "a" + } + }], + "variants": { + "x": { + "response": "ok", + "tool_uses": [] + } + }, + }, + "knowledge_recall_insufficient", + ), + ( + { + "expected_response": "ok", + "expected_tool_uses": [{ + "name": "search", + "args": { + "q": "a" + } + }], + "variants": { + "x": { + "response": "ok", + "tool_uses": [{ + "name": "search", + "args": { + "q": "WRONG" + } + }] + } + }, + }, + "tool_parameter_error", + ), + ( + { + "expected_response": "answer", + "expected_tool_uses": [], + "variants": { + "x": { + "response": "完全不同的内容", + "tool_uses": [] + } + }, + }, + "final_response_mismatch", + ), + ( + { + "expected_response": "ok", + "expected_tool_uses": [{ + "name": "calc", + "args": {} + }], + "variants": { + "x": { + "response": "ok", + "tool_uses": [{ + "name": "WRONG_TOOL", + "args": {} + }] + } + }, + }, + "tool_selection_error", + ), + ] + correct = 0 + for spec, expected_cat in hidden: + attr = attribute_failure(spec, "x") + if attr.category == expected_cat: + correct += 1 + assert correct / len(hidden) >= 0.75, (f"隐藏归因准确率 {correct}/{len(hidden)} 不足 75%") + + +def test_cli_exit_code_accept(tmp_path): + """fake 模式 CLI：accept → 退出码 0。""" + env = {**os.environ, "PYTHONUTF8": "1"} + r = subprocess.run( + [sys.executable, + str(EXAMPLE_DIR / "run_pipeline.py"), "--mode", "fake", "--output-dir", + str(tmp_path)], + capture_output=True, + env=env, + ) + assert r.returncode == 0, f"期望退出码 0，实际 {r.returncode}; stderr={r.stderr.decode(errors='replace')}" + assert (tmp_path / "optimization_report.json").exists() + + +def test_data_quality_no_cross_split_leakage(configs): + """train/val 无 eval_id 重复（防数据污染）。""" + from run_pipeline import check_data_quality + + dq = check_data_quality(CASES) + assert dq.passed + assert dq.cross_split_duplicates == 0 + assert dq.train_cases == 3 + assert dq.validation_cases == 3