diff --git a/.gitignore b/.gitignore index 233248dd..ece565cd 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,12 @@ *.lock *.log examples/*.log +runs/ +examples/**/runs/ +**/online_eval_metrics.json +**/offline_metrics.json +**/trace_metrics.json +**/trace_evalset.json trpc-agent-py.egg-info diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..e89f9891 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,30 @@ +# Project Agent Guidelines + +These rules apply to the whole repository unless a deeper `AGENTS.md` overrides them. + +## Engineering Defaults + +- Prefer the smallest change that satisfies the requested behavior. +- Match existing file layout, naming, and test style before adding new patterns. +- Do not refactor adjacent code or rewrite unrelated docs while implementing an issue. +- Treat prompt, evalset, optimizer config, and report files as auditable artifacts. + +## Evaluation And Optimization Work + +- Default to offline, deterministic flows for examples and tests. +- Online model calls must be opt-in and gated by explicit environment variables. +- Keep train and validation evalsets physically separate. +- Do not expose validation gold data to optimizer logic beyond final gate scoring. +- Write generated reports only under `runs/` or a caller-provided output directory. + +## Reports + +- Produce machine-readable JSON first; human-readable Markdown may summarize it. +- Avoid new Markdown files unless the issue asks for them or they are standard example docs. +- Reports must state baseline score, candidate score, case deltas, gate decision, and rejection or acceptance reasons. + +## Testing + +- Add focused tests for new behavior before broad regression runs. +- Fake or trace modes must run without API keys. +- Online-mode tests should validate configuration and wiring without consuming real API calls. diff --git a/docs/superpowers/plans/2026-07-10-eval-optimize-loop-hardening.md b/docs/superpowers/plans/2026-07-10-eval-optimize-loop-hardening.md new file mode 100644 index 00000000..f5e23be5 --- /dev/null +++ b/docs/superpowers/plans/2026-07-10-eval-optimize-loop-hardening.md @@ -0,0 +1,1433 @@ +# Eval Optimize Loop Hidden-Test Hardening Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Close the remaining issue-compliance and hidden-test gaps in the evaluation/optimization example while preserving the verified fake, trace, and real-API behavior. + +**Architecture:** Keep the issue-facing pipeline in `run_pipeline.py`, but make its three decision boundaries explicit: normalized evaluator evidence, total gate decisions, and a strict report contract. Reuse `AgentEvaluator`, `AgentOptimizer`, `TargetPrompt`, and native optimizer round records; do not create a second optimization engine or expose final-validation gold to optimization. + +**Tech Stack:** Python 3.10+, asyncio, tRPC-Agent `AgentEvaluator`/`AgentOptimizer`, Pydantic-backed SDK result models, JSON Schema Draft 2020-12, pytest, fake fixtures, trace replay, OpenAI-compatible real API. + +## Global Constraints + +- Fake and trace modes require no API variables and must finish in less than 180 seconds. +- Online mode remains opt-in through `TRPC_AGENT_API_KEY`, `TRPC_AGENT_BASE_URL`, and `TRPC_AGENT_MODEL_NAME`. +- `train.evalset.json`, `optimizer_dev.evalset.json`, and `val.evalset.json` must be path-distinct and case-id/content-disjoint where their roles require isolation. +- Final-validation gold may be used only for baseline scoring, final candidate scoring, and the product gate. +- Generated outputs stay under `runs/` or a caller-provided output directory. +- Reports remain JSON-first and must not contain API keys, authorization headers, model thoughts, or unredacted provider URLs. +- Keep the existing source prompts unchanged; candidate prompts are written only as run artifacts unless a caller explicitly opts into source updates. +- Do not silence repository-wide warnings or skip unrelated tests merely to make this issue appear green. +- Every task ends with focused verification and one scoped commit. + +--- + +## Verified Starting Point + +The following evidence was reproduced on 2026-07-10 before writing this plan: + +- Focused example suite: 28 passed, 1 opt-in online test skipped. +- Entire `tests/evaluation` suite: exit code 0, with the same opt-in online skip. +- Fake run: accepted `candidate_local_patch`, duration 4.58 seconds. +- Trace run: accepted `candidate_local_patch`, duration 4.58 seconds, no API required. +- Real API weak-baseline run: baseline validation 0.666667, candidate validation 1.0, accepted, 176.3446 seconds, 37 model calls. +- Real API default-prompt run: baseline and candidate validation 1.0, rejected for no improvement, 95.69541 seconds, 36 model calls. +- Full repository `pytest -q`: non-zero on Windows because of existing POSIX path, permission, shell-command, symlink, dependency-version, and one LangGraph `Interrupt(id=...)` incompatibility; therefore “all tests pass” is not currently a true repository-wide statement. + +The issue-specific implementation is functional, but the reproduced hidden-style failures are: + +- malformed `tool` values can crash failure attribution; +- `arguments: null` and `arguments: []` are mislabeled as `knowledge_gap`; +- a candidate can omit validation cases and still be accepted; +- an unexpected candidate case raises `KeyError`; +- non-finite scores can be accepted; +- equality at an exact configured delta boundary is accepted even though the issue specifies improvement strictly greater than the threshold; +- case deltas lack explicit new-pass/new-fail/improved/regressed classification; +- the schema accepts empty case results, empty deltas, empty case-delta objects, and out-of-range attribution coverage; +- candidate-level cost/duration/reproducibility fields are absent; +- the Design Notes section is 976 non-whitespace characters, outside the requested 300-500-character range; +- warning suppression and a global DeepSeek log-level downgrade hide symptoms instead of fixing or recording them; +- the checked-in sample has a trailing blank line that fails `git diff --check`. + +--- + +### Task 1: Make Failure Attribution Total and Type-Safe + +**Files:** +- Modify: `examples/optimization/eval_optimize_loop/run_pipeline.py:420` +- Modify: `tests/evaluation/test_eval_optimize_loop_example.py:494` + +**Interfaces:** +- Consumes: final response strings and per-case metric dictionaries. +- Produces: `attribute_failure_case(...) -> {"root_cause": str, "reasons": list[str]}` for every input without raising. + +- [ ] **Step 1: Add failing malformed-structure tests** + +```python +@pytest.mark.parametrize( + ("actual_text", "expected_root"), + [ + ( + '{"route":"faq","tool":"none","reason":"bad shape"}', + "tool_call_error", + ), + ( + '{"route":"faq","tool":{"name":"none","arguments":null},"reason":"bad args"}', + "parameter_error", + ), + ( + '{"route":"faq","tool":{"name":"none","arguments":[]},"reason":"bad args"}', + "parameter_error", + ), + ( + '{"route":"faq","tool":{"name":"none"},"reason":"missing args"}', + "parameter_error", + ), + ], +) +def test_failure_attribution_is_total_for_malformed_tool_shapes( + actual_text: str, + expected_root: str, +): + module = load_pipeline_module() + result = module.attribute_failure_case( + actual_text=actual_text, + expected_text=( + '{"route":"faq","tool":{"name":"none","arguments":{}},' + '"reason":"expected"}' + ), + error_message=None, + metrics={ROUTE_TOOL_ARGS_METRIC: {"passed": False}}, + ) + assert result["root_cause"] == expected_root + assert result["reasons"] +``` + +- [ ] **Step 2: Run the new test and verify the current crash/misclassification** + +Run: + +```powershell +python -m pytest tests/evaluation/test_eval_optimize_loop_example.py::test_failure_attribution_is_total_for_malformed_tool_shapes -q +``` + +Expected: FAIL with either `AttributeError: 'str' object has no attribute 'get'` or `knowledge_gap != parameter_error`. + +- [ ] **Step 3: Replace truthiness-based tool/argument parsing with type checks** + +```python +def _failed_metric_names(metrics: dict[str, dict[str, Any]]) -> list[str]: + return sorted(name for name, metric in metrics.items() if _metric_failed(metric)) + + +def _metric_failure_root(failed_metric_names: list[str]) -> tuple[str, str]: + rubric = [name for name in failed_metric_names if "rubric" in name or name.startswith("llm_")] + if rubric: + return "rubric_failed", "rubric metric failed: " + ", ".join(rubric) + knowledge = [ + name + for name in failed_metric_names + if any(token in name.lower() for token in ("knowledge", "retrieval", "recall", "ground")) + ] + if knowledge: + return "knowledge_gap", "knowledge metric failed: " + ", ".join(knowledge) + return "metric_failed", "content metric failed: " + ", ".join(failed_metric_names) +``` + +In `attribute_failure_case`, replace the current `actual_tool = actual.get("tool") or {}` block with: + +```python + actual_tool = actual.get("tool") + expected_tool = expected.get("tool") + if not isinstance(expected_tool, dict): + return { + "root_cause": "runtime_error", + "reasons": ["expected final response has a non-object tool field"], + } + if str(actual.get("route", "")) != str(expected.get("route", "")): + return { + "root_cause": "final_response_mismatch", + "reasons": [ + f"actual route {actual.get('route')!r} did not match " + f"expected route {expected.get('route')!r}" + ], + } + if not isinstance(actual_tool, dict): + return { + "root_cause": "tool_call_error", + "reasons": ["actual tool must be a JSON object"], + } + if str(actual_tool.get("name", "")) != str(expected_tool.get("name", "")): + return { + "root_cause": "tool_call_error", + "reasons": [ + f"actual tool {actual_tool.get('name')!r} did not match " + f"expected tool {expected_tool.get('name')!r}" + ], + } + actual_arguments = actual_tool.get("arguments", _MISSING) + expected_arguments = expected_tool.get("arguments", _MISSING) + if not isinstance(expected_arguments, dict): + return { + "root_cause": "runtime_error", + "reasons": ["expected tool arguments must be a JSON object"], + } + if not isinstance(actual_arguments, dict): + return { + "root_cause": "parameter_error", + "reasons": ["actual tool arguments must be a JSON object"], + } + if actual_arguments != expected_arguments: + return { + "root_cause": "parameter_error", + "reasons": ["tool arguments did not match expected arguments"], + } + + failed_metric_names = _failed_metric_names(metrics) + if failed_metric_names: + root_cause, reason = _metric_failure_root(failed_metric_names) + return {"root_cause": root_cause, "reasons": [reason]} + return { + "root_cause": "metric_failed", + "reasons": ["case failed without a reported failed metric"], + } +``` + +Add `"metric_failed"` to `TAXONOMY`. + +- [ ] **Step 4: Run canonical and adversarial attribution tests** + +Run: + +```powershell +python -m pytest tests/evaluation/test_eval_optimize_loop_example.py -k "failure_attribution or route_tool_argument" -q +``` + +Expected: PASS; every failed case has one taxonomy value and at least one non-empty reason. + +- [ ] **Step 5: Commit the isolated attribution fix** + +```powershell +git add examples/optimization/eval_optimize_loop/run_pipeline.py tests/evaluation/test_eval_optimize_loop_example.py +git commit -m "fix: harden optimization failure attribution" +``` + +--- + +### Task 2: Make Gate Decisions Total, Strict, and Fail-Closed + +**Files:** +- Modify: `examples/optimization/eval_optimize_loop/run_pipeline.py:907` +- Modify: `tests/evaluation/test_eval_optimize_loop_example.py:400` + +**Interfaces:** +- Consumes: baseline and candidate evaluation summaries. +- Produces: a gate result that never accepts malformed evidence and never raises for case-set mismatch. + +- [ ] **Step 1: Add a table-driven gate adversarial suite** + +```python +def _gate_summary( + score: float, + cases: list[dict[str, Any]], + *, + metric_passed: bool = True, +) -> dict[str, Any]: + return { + "score": score, + "metrics": {ROUTE_TOOL_ARGS_METRIC: {"passed": metric_passed}}, + "case_results": cases, + } + + +def test_gate_fails_closed_for_boundary_and_invalid_evidence(): + module = load_pipeline_module() + baseline = _gate_summary( + 0.25, + [ + {"case_id": "a", "score": 0.0, "passed": False, "tags": []}, + {"case_id": "b", "score": 0.5, "passed": True, "tags": ["critical"]}, + ], + ) + valid_candidate = _gate_summary( + 0.75, + [ + {"case_id": "a", "score": 1.0, "passed": True, "tags": []}, + {"case_id": "b", "score": 0.5, "passed": True, "tags": ["critical"]}, + ], + ) + + exact_boundary = module.apply_gate( + candidate_id="boundary", + baseline_val=baseline, + candidate_val=valid_candidate, + gate_config={ + "min_validation_delta": 0.5, + "required_metrics": [ROUTE_TOOL_ARGS_METRIC], + }, + duration_seconds=1.0, + cost_usd=0.0, + ) + assert exact_boundary["accepted"] is False + + missing_case = copy.deepcopy(valid_candidate) + missing_case["case_results"] = missing_case["case_results"][:1] + missing = module.apply_gate( + candidate_id="missing", + baseline_val=baseline, + candidate_val=missing_case, + gate_config={"required_metrics": [ROUTE_TOOL_ARGS_METRIC]}, + duration_seconds=1.0, + cost_usd=0.0, + ) + assert missing["accepted"] is False + assert missing["missing_case_ids"] == ["b"] + + extra_case = copy.deepcopy(valid_candidate) + extra_case["case_results"].append( + {"case_id": "c", "score": 1.0, "passed": True, "tags": []} + ) + extra = module.apply_gate( + candidate_id="extra", + baseline_val=baseline, + candidate_val=extra_case, + gate_config={"required_metrics": [ROUTE_TOOL_ARGS_METRIC]}, + duration_seconds=1.0, + cost_usd=0.0, + ) + assert extra["accepted"] is False + assert extra["unexpected_case_ids"] == ["c"] + + non_finite = copy.deepcopy(valid_candidate) + non_finite["score"] = float("nan") + invalid = module.apply_gate( + candidate_id="nan", + baseline_val=baseline, + candidate_val=non_finite, + gate_config={"required_metrics": [ROUTE_TOOL_ARGS_METRIC]}, + duration_seconds=1.0, + cost_usd=0.0, + ) + assert invalid["accepted"] is False + assert "finite" in " ".join(invalid["reasons"]) +``` + +- [ ] **Step 2: Verify the suite fails on all reproduced boundary defects** + +Run: + +```powershell +python -m pytest tests/evaluation/test_eval_optimize_loop_example.py::test_gate_fails_closed_for_boundary_and_invalid_evidence -q +``` + +Expected: FAIL because the current gate accepts a missing case and NaN, raises on an extra case, and accepts an exact 0.5 boundary. + +- [ ] **Step 3: Add finite-number and case-index helpers** + +```python +import math + + +def _finite_float(value: Any) -> float | None: + try: + parsed = float(value) + except (TypeError, ValueError): + return None + return parsed if math.isfinite(parsed) else None + + +def _index_gate_cases( + evaluation: dict[str, Any], +) -> tuple[dict[str, dict[str, Any]], list[str]]: + cases = evaluation.get("case_results") + if not isinstance(cases, list): + return {}, ["case_results must be an array"] + indexed: dict[str, dict[str, Any]] = {} + issues: list[str] = [] + for position, case in enumerate(cases): + if not isinstance(case, dict) or not str(case.get("case_id", "")).strip(): + issues.append(f"case_results[{position}] has no case_id") + continue + case_id = str(case["case_id"]) + if case_id in indexed: + issues.append(f"duplicate case_id: {case_id}") + continue + indexed[case_id] = case + return indexed, issues +``` + +- [ ] **Step 4: Replace gate preconditions and threshold comparison** + +At the start of `apply_gate`, build both indexes, append all validation issues to `reasons`, and set `accepted = False` when any issue exists. Compute: + +```python + baseline_score = _finite_float(baseline_val.get("score")) + candidate_score = _finite_float(candidate_val.get("score")) + validation_delta = ( + None + if baseline_score is None or candidate_score is None + else candidate_score - baseline_score + ) + if validation_delta is None: + accepted = False + reasons.append("baseline and candidate validation scores must be finite numbers") + else: + min_delta = float(gate_config.get("min_validation_delta", 0.0)) + if validation_delta <= min_delta: + accepted = False + reasons.append( + f"validation score improvement {validation_delta:.4f} " + f"must be greater than required {min_delta:.4f}" + ) + + baseline_ids = set(baseline_by_id) + candidate_ids = set(candidate_by_id) + missing_case_ids = sorted(baseline_ids - candidate_ids) + unexpected_case_ids = sorted(candidate_ids - baseline_ids) + if missing_case_ids: + accepted = False + reasons.append("candidate omitted validation case(s): " + ", ".join(missing_case_ids)) + if unexpected_case_ids: + accepted = False + reasons.append("candidate introduced unknown validation case(s): " + ", ".join(unexpected_case_ids)) +``` + +Only compute hard-fail and critical-regression checks over `sorted(baseline_ids & candidate_ids)`. Normalize tags with `{str(tag).lower() for tag in candidate_case.get("tags", [])}`. + +Return `missing_case_ids`, `unexpected_case_ids`, and nullable `validation_delta` in the gate result. + +- [ ] **Step 5: Run the complete gate test set** + +Run: + +```powershell +python -m pytest tests/evaluation/test_eval_optimize_loop_example.py -k "gate or regression or cost_budget" -q +``` + +Expected: PASS for improvement, no-op, aggregate regression, critical regression, hard fail, required metric, cost, duration, exact threshold, malformed case set, and non-finite score cases. + +- [ ] **Step 6: Commit the gate hardening** + +```powershell +git add examples/optimization/eval_optimize_loop/run_pipeline.py tests/evaluation/test_eval_optimize_loop_example.py +git commit -m "fix: make optimization gates fail closed" +``` + +--- + +### Task 3: Add Explicit Case-Delta Classes and Key Traces + +**Files:** +- Modify: `examples/optimization/eval_optimize_loop/run_pipeline.py:573` +- Modify: `examples/optimization/eval_optimize_loop/run_pipeline.py:907` +- Modify: `tests/evaluation/test_eval_optimize_loop_example.py:225` + +**Interfaces:** +- Produces: each evaluation case with `expected_text` and `key_trace`. +- Produces: each case delta with pass-state transitions and one stable `change_type`. + +- [ ] **Step 1: Add failing tests for all required delta classes** + +```python +def test_case_deltas_classify_pass_fail_and_score_transitions(): + module = load_pipeline_module() + baseline = { + "case_results": [ + {"case_id": "new_pass", "score": 0.0, "passed": False, "actual_text": "b1"}, + {"case_id": "new_fail", "score": 1.0, "passed": True, "actual_text": "b2"}, + {"case_id": "up", "score": 0.4, "passed": True, "actual_text": "b3"}, + {"case_id": "down", "score": 0.8, "passed": True, "actual_text": "b4"}, + {"case_id": "same", "score": 1.0, "passed": True, "actual_text": "b5"}, + ] + } + candidate = { + "case_results": [ + {"case_id": "new_pass", "score": 1.0, "passed": True, "actual_text": "c1", "root_cause": "", "reasons": []}, + {"case_id": "new_fail", "score": 0.0, "passed": False, "actual_text": "c2", "root_cause": "format_error", "reasons": ["bad"]}, + {"case_id": "up", "score": 0.6, "passed": True, "actual_text": "c3", "root_cause": "", "reasons": []}, + {"case_id": "down", "score": 0.6, "passed": True, "actual_text": "c4", "root_cause": "", "reasons": []}, + {"case_id": "same", "score": 1.0, "passed": True, "actual_text": "c5", "root_cause": "", "reasons": []}, + ] + } + by_id = { + item["case_id"]: item + for item in module.build_case_deltas(baseline, candidate) + } + assert by_id["new_pass"]["change_type"] == "new_pass" + assert by_id["new_fail"]["change_type"] == "new_fail" + assert by_id["up"]["change_type"] == "score_improved" + assert by_id["down"]["change_type"] == "score_regressed" + assert by_id["same"]["change_type"] == "unchanged" + assert by_id["new_fail"]["baseline_passed"] is True + assert by_id["new_fail"]["candidate_passed"] is False +``` + +Extend the fake report test with: + +```python + first_case = report["baseline"]["validation"]["case_results"][0] + assert first_case["expected_text"] + assert first_case["key_trace"]["invocation_id"] + assert first_case["key_trace"]["actual_final_response"] == first_case["actual_text"] + assert first_case["key_trace"]["expected_final_response"] == first_case["expected_text"] +``` + +- [ ] **Step 2: Run the tests and verify missing fields** + +Run: + +```powershell +python -m pytest tests/evaluation/test_eval_optimize_loop_example.py -k "case_deltas_classify or generates_complete_report" -q +``` + +Expected: FAIL on missing `change_type`, pass-state fields, `expected_text`, and `key_trace`. + +- [ ] **Step 3: Add a deterministic classifier** + +```python +def classify_case_delta(before: dict[str, Any], after: dict[str, Any]) -> str: + if not bool(before.get("passed")) and bool(after.get("passed")): + return "new_pass" + if bool(before.get("passed")) and not bool(after.get("passed")): + return "new_fail" + delta = float(after.get("score", 0.0)) - float(before.get("score", 0.0)) + if delta > 0: + return "score_improved" + if delta < 0: + return "score_regressed" + return "unchanged" +``` + +Replace `build_case_deltas` with a union-based implementation so a rejected +case-set mismatch is still reportable: + +```python +def build_case_deltas( + baseline_val: dict[str, Any], + candidate_val: dict[str, Any], +) -> list[dict[str, Any]]: + baseline_by_id = { + str(case["case_id"]): case + for case in baseline_val.get("case_results", []) + if isinstance(case, dict) and case.get("case_id") + } + candidate_by_id = { + str(case["case_id"]): case + for case in candidate_val.get("case_results", []) + if isinstance(case, dict) and case.get("case_id") + } + deltas: list[dict[str, Any]] = [] + for case_id in sorted(set(baseline_by_id) | set(candidate_by_id)): + before = baseline_by_id.get(case_id) + case = candidate_by_id.get(case_id) + if before is None: + deltas.append({ + "case_id": case_id, + "baseline_score": None, + "candidate_score": case.get("score"), + "baseline_passed": None, + "candidate_passed": bool(case.get("passed")), + "delta": None, + "change_type": "unexpected_candidate", + "baseline_actual_text": "", + "candidate_actual_text": case.get("actual_text", ""), + "root_cause": "runtime_error", + "reasons": ["candidate introduced an unknown validation case"], + }) + continue + if case is None: + deltas.append({ + "case_id": case_id, + "baseline_score": before.get("score"), + "candidate_score": None, + "baseline_passed": bool(before.get("passed")), + "candidate_passed": None, + "delta": None, + "change_type": "missing_candidate", + "baseline_actual_text": before.get("actual_text", ""), + "candidate_actual_text": "", + "root_cause": "runtime_error", + "reasons": ["candidate omitted a baseline validation case"], + }) + continue + delta = round(float(case["score"]) - float(before["score"]), 6) + deltas.append({ + "case_id": case_id, + "baseline_score": before["score"], + "candidate_score": case["score"], + "baseline_passed": bool(before["passed"]), + "candidate_passed": bool(case["passed"]), + "delta": delta, + "change_type": classify_case_delta(before, case), + "baseline_actual_text": before.get("actual_text", ""), + "candidate_actual_text": case.get("actual_text", ""), + "root_cause": case.get("root_cause", ""), + "reasons": case.get("reasons", []), + }) + return deltas +``` + +- [ ] **Step 4: Add the key trace at evaluator-summary time** + +When constructing each case result in `summarize_evaluate_result`, include: + +```python + "expected_text": expected_text, + "key_trace": { + "invocation_id": str( + case_by_id[eval_id]["conversation"][0].get("invocation_id", "") + ), + "actual_final_response": actual_text, + "expected_final_response": expected_text, + "error_message": error_message, + }, +``` + +For the no-run branch, use the same shape with an empty actual response and `"AgentEvaluator returned no run for case"` as the error. Do not include chain-of-thought or provider headers. + +- [ ] **Step 5: Verify report and Markdown expose the same classifications** + +Update `render_markdown` so every winner case line appends `change_type`. Then run: + +```powershell +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake --output-dir runs/plan_verify --run-id delta_trace +python -m pytest tests/evaluation/test_eval_optimize_loop_example.py -k "case_delta or key_trace or fake_mode" -q +``` + +Expected: PASS; JSON and Markdown both distinguish new pass, new fail, score improvement, score regression, and unchanged. + +- [ ] **Step 6: Commit the delta and trace contract** + +```powershell +git add examples/optimization/eval_optimize_loop/run_pipeline.py tests/evaluation/test_eval_optimize_loop_example.py +git commit -m "feat: classify optimization case deltas" +``` + +--- + +### Task 4: Complete Candidate and Optimizer-Round Audit Data + +**Files:** +- Modify: `examples/optimization/eval_optimize_loop/run_pipeline.py:1026` +- Modify: `examples/optimization/eval_optimize_loop/run_pipeline.py:1219` +- Modify: `examples/optimization/eval_optimize_loop/run_pipeline.py:1683` +- Modify: `tests/evaluation/test_eval_optimize_loop_example.py:638` + +**Interfaces:** +- Produces: `candidate.audit` with seed, duration, cost status, and config digest. +- Produces: `optimization_rounds` containing native round prompt artifacts, metrics, decision, cost, and duration. + +- [ ] **Step 1: Add failing candidate-audit and artifact-existence tests** + +```python +def _assert_candidate_audit(candidate: dict[str, Any], seed: int) -> None: + audit = candidate["audit"] + assert audit["seed"] == seed + assert audit["duration_seconds"] >= 0 + assert audit["cost"]["currency"] == "USD" + assert audit["config_sha256"] + assert len(audit["config_sha256"]) == 64 + + +@pytest.mark.asyncio +async def test_fake_mode_audits_each_candidate_independently(tmp_path: Path): + module = load_pipeline_module() + run_dir = await module.run_fake_or_trace( + mode="fake", + seed=7, + output_dir=tmp_path, + run_id="candidate_audit", + ) + report = load_report(run_dir / "optimization_report.json") + for candidate in report["candidates"]: + _assert_candidate_audit(candidate, 7) + assert Path(candidate["artifacts"]["prompt_dir"]).is_dir() + assert Path(candidate["artifacts"]["prompt_patch"]).is_file() +``` + +In the mocked online test, assert: + +```python + assert report["optimization_rounds"] == [] + for name, value in report["artifacts"].items(): + if name.startswith("native_") and value: + assert Path(value).exists(), name +``` + +- [ ] **Step 2: Verify candidate audit is currently absent** + +Run: + +```powershell +python -m pytest tests/evaluation/test_eval_optimize_loop_example.py -k "audits_each_candidate or construct_optimizer_call" -q +``` + +Expected: FAIL on missing `candidate.audit` and on the nonexistent `native_rounds_dir` when zero rounds ran. + +- [ ] **Step 3: Add reproducibility digest helpers** + +```python +def sha256_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def build_candidate_audit( + *, + seed: int, + duration_seconds: float, + cost_usd: float | None, + optimizer_config: Path, +) -> dict[str, Any]: + return { + "seed": seed, + "duration_seconds": round(duration_seconds, 6), + "cost": { + "currency": "USD", + "estimated": cost_usd, + "known": cost_usd is not None, + }, + "config_path": str(optimizer_config), + "config_sha256": sha256_file(optimizer_config), + } +``` + +Extend `build_candidate_report` with required `seed` and `optimizer_config` parameters and add `"audit": build_candidate_audit(...)`. + +- [ ] **Step 4: Measure offline candidates independently** + +Set `candidate_started = time.perf_counter()` immediately before each candidate's train evaluation. Pass `time.perf_counter() - candidate_started` to `build_candidate_report`; do not use cumulative pipeline duration as candidate duration. + +- [ ] **Step 5: Normalize native optimizer rounds** + +```python +def write_optimizer_round_artifacts( + *, + run_dir: Path, + rounds: list[Any], +) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + for round_record in rounds: + round_id = int(round_record.round) + round_dir = run_dir / "prompts" / f"optimizer_round_{round_id:03d}" + round_dir.mkdir(parents=True, exist_ok=True) + prompt_paths: dict[str, str] = {} + prompt_hashes: dict[str, str] = {} + for name, content in sorted(round_record.candidate_prompts.items()): + prompt_path = round_dir / f"{name}.md" + prompt_path.write_text(content, encoding="utf-8") + prompt_paths[name] = str(prompt_path) + prompt_hashes[name] = sha256_text(content) + records.append({ + "round": round_id, + "optimized_field_names": list(round_record.optimized_field_names), + "prompt_paths": prompt_paths, + "prompt_sha256": prompt_hashes, + "validation_pass_rate": float(round_record.validation_pass_rate), + "metric_breakdown": dict(round_record.metric_breakdown), + "accepted": bool(round_record.accepted), + "decision_reason": ( + round_record.acceptance_reason + or round_record.skip_reason + or round_record.error_message + or "optimizer reported no reason" + ), + "failed_case_ids": list(round_record.failed_case_ids), + "cost_usd": float(round_record.round_llm_cost), + "token_usage": dict(round_record.round_token_usage), + "duration_seconds": float(round_record.duration_seconds), + }) + return records +``` + +Add `optimization_rounds` to online reports and `[]` to fake/trace reports. + +- [ ] **Step 6: List only native artifacts that exist** + +Build the online artifact dictionary with a helper: + +```python +def existing_artifact(path: Path) -> str: + return str(path) if path.exists() else "" +``` + +Use it for `native_result_json`, `native_summary_txt`, `native_rounds_dir`, `native_baseline_prompts_dir`, `native_best_prompts_dir`, and `native_config_snapshot_json`. Preserve the report JSON/Markdown destination paths because they are created by `write_report` immediately after validation. + +- [ ] **Step 7: Preserve an auditable report when native optimization returns FAILED** + +Merge source prompts before final evaluation: + +```python + source_prompt_texts = {name: text for name, (_, text) in source_prompts.items()} + best_prompt_texts = { + **source_prompt_texts, + **dict(getattr(result, "best_prompts", {}) or {}), + } +``` + +Use `best_prompt_texts` for the candidate evaluator and prompt artifacts. If `result.status != "SUCCEEDED"`, reject the candidate and include `result.error_message` in the gate reasons and `online_result`; still write the complete report. + +- [ ] **Step 8: Run audit tests and commit** + +Run: + +```powershell +python -m pytest tests/evaluation/test_eval_optimize_loop_example.py -k "audit or prompt_artifacts or online_mode" -q +``` + +Expected: PASS, including zero-round and failed-optimizer cases. + +```powershell +git add examples/optimization/eval_optimize_loop/run_pipeline.py tests/evaluation/test_eval_optimize_loop_example.py +git commit -m "feat: complete optimization candidate audit" +``` + +--- + +### Task 5: Turn the JSON Schema into an Enforced Report Contract + +**Files:** +- Modify: `examples/optimization/eval_optimize_loop/optimization_report.schema.json` +- Modify: `tests/evaluation/test_eval_optimize_loop_example.py:201` + +**Interfaces:** +- Consumes: fake, trace, and online report dictionaries. +- Produces: rejection of structurally incomplete or numerically invalid reports before any report file is written. + +- [ ] **Step 1: Add mutation tests for every currently permissive core object** + +```python +@pytest.mark.parametrize( + ("mutation_path", "replacement"), + [ + (("candidates", 0, "delta"), {}), + (("baseline", "validation", "case_results"), [{}]), + (("candidates", 0, "case_deltas"), [{}]), + (("failure_attribution", "coverage"), 9.0), + (("candidates", 0, "audit"), {}), + ], +) +def test_report_schema_rejects_incomplete_core_objects( + mutation_path: tuple[Any, ...], + replacement: Any, +): + module = load_pipeline_module() + report = load_report( + EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json" + ) + target: Any = report + for key in mutation_path[:-1]: + target = target[key] + target[mutation_path[-1]] = replacement + with pytest.raises(ValidationError): + module.validate_report_schema(report) +``` + +- [ ] **Step 2: Verify all five mutations are currently accepted** + +Run: + +```powershell +python -m pytest tests/evaluation/test_eval_optimize_loop_example.py::test_report_schema_rejects_incomplete_core_objects -q +``` + +Expected: FAIL because the current schema accepts the malformed replacements. + +- [ ] **Step 3: Define strict case and delta objects** + +Add these definitions and reference them from evaluation summaries and candidates: + +```json +"keyTrace": { + "type": "object", + "additionalProperties": false, + "required": [ + "invocation_id", "actual_final_response", + "expected_final_response", "error_message" + ], + "properties": { + "invocation_id": {"type": "string"}, + "actual_final_response": {"type": "string"}, + "expected_final_response": {"type": "string"}, + "error_message": {"type": ["string", "null"]} + } +}, +"evaluationCase": { + "type": "object", + "additionalProperties": false, + "required": [ + "case_id", "tags", "user", "score", "passed", "metrics", + "actual_text", "expected_text", "key_trace", "root_cause", "reasons" + ], + "properties": { + "case_id": {"type": "string", "minLength": 1}, + "tags": {"type": "array", "items": {"type": "string"}}, + "user": {"type": "string"}, + "score": {"type": "number", "minimum": 0, "maximum": 1}, + "passed": {"type": "boolean"}, + "metrics": { + "type": "object", + "minProperties": 1, + "additionalProperties": {"$ref": "#/$defs/metricSummary"} + }, + "actual_text": {"type": "string"}, + "expected_text": {"type": "string"}, + "key_trace": {"$ref": "#/$defs/keyTrace"}, + "root_cause": { + "enum": [ + "", "final_response_mismatch", "tool_call_error", "parameter_error", + "rubric_failed", "knowledge_gap", "format_error", "runtime_error", + "metric_failed" + ] + }, + "reasons": {"type": "array", "items": {"type": "string", "minLength": 1}} + } +}, +"caseDelta": { + "type": "object", + "additionalProperties": false, + "required": [ + "case_id", "baseline_score", "candidate_score", "baseline_passed", + "candidate_passed", "delta", "change_type", "baseline_actual_text", + "candidate_actual_text", "root_cause", "reasons" + ], + "properties": { + "case_id": {"type": "string", "minLength": 1}, + "baseline_score": {"type": ["number", "null"], "minimum": 0, "maximum": 1}, + "candidate_score": {"type": ["number", "null"], "minimum": 0, "maximum": 1}, + "baseline_passed": {"type": ["boolean", "null"]}, + "candidate_passed": {"type": ["boolean", "null"]}, + "delta": {"type": ["number", "null"], "minimum": -1, "maximum": 1}, + "change_type": { + "enum": [ + "new_pass", "new_fail", "score_improved", "score_regressed", + "unchanged", "missing_candidate", "unexpected_candidate" + ] + }, + "baseline_actual_text": {"type": "string"}, + "candidate_actual_text": {"type": "string"}, + "root_cause": {"type": "string"}, + "reasons": {"type": "array", "items": {"type": "string"}} + } +} +``` + +Require `case_results` with `minItems: 1` in every evaluation summary and `case_deltas` with `minItems: 1` in every candidate. + +- [ ] **Step 4: Tighten numeric, gate, attribution, and audit constraints** + +Require all three delta fields, bound scores/pass rates/coverage to `[0, 1]`, require non-negative taxonomy counts, and define: + +```json +"candidateAudit": { + "type": "object", + "additionalProperties": false, + "required": ["seed", "duration_seconds", "cost", "config_path", "config_sha256"], + "properties": { + "seed": {"type": "integer"}, + "duration_seconds": {"type": "number", "minimum": 0}, + "cost": { + "type": "object", + "additionalProperties": false, + "required": ["currency", "estimated", "known"], + "properties": { + "currency": {"const": "USD"}, + "estimated": {"type": ["number", "null"], "minimum": 0}, + "known": {"type": "boolean"} + } + }, + "config_path": {"type": "string", "minLength": 1}, + "config_sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"} + } +}, +"optimizationRound": { + "type": "object", + "additionalProperties": false, + "required": [ + "round", "optimized_field_names", "prompt_paths", "prompt_sha256", + "validation_pass_rate", "metric_breakdown", "accepted", + "decision_reason", "failed_case_ids", "cost_usd", "token_usage", + "duration_seconds" + ], + "properties": { + "round": {"type": "integer", "minimum": 1}, + "optimized_field_names": { + "type": "array", + "items": {"type": "string"} + }, + "prompt_paths": { + "type": "object", + "additionalProperties": {"type": "string", "minLength": 1} + }, + "prompt_sha256": { + "type": "object", + "additionalProperties": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "validation_pass_rate": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "metric_breakdown": { + "type": "object", + "additionalProperties": {"type": "number"} + }, + "accepted": {"type": "boolean"}, + "decision_reason": {"type": "string", "minLength": 1}, + "failed_case_ids": { + "type": "array", + "items": {"type": "string"} + }, + "cost_usd": {"type": "number", "minimum": 0}, + "token_usage": { + "type": "object", + "additionalProperties": false, + "required": ["prompt", "completion", "total"], + "properties": { + "prompt": {"type": "integer", "minimum": 0}, + "completion": {"type": "integer", "minimum": 0}, + "total": {"type": "integer", "minimum": 0} + } + }, + "duration_seconds": {"type": "number", "minimum": 0} + } +} +``` + +Require `candidate.audit`. Require `gate.validation_delta`, `new_hard_fail_ids`, `critical_regression_ids`, `missing_case_ids`, and `unexpected_case_ids`; allow `validation_delta` to be null only for malformed evidence that was rejected. + +Add this top-level property and add `optimization_rounds` to the root +`required` array: + +```json +"optimization_rounds": { + "type": "array", + "items": {"$ref": "#/$defs/optimizationRound"} +} +``` + +Fake and trace reports use an empty array. + +- [ ] **Step 5: Validate all three modes against the schema** + +Run: + +```powershell +python -m pytest tests/evaluation/test_eval_optimize_loop_example.py -k "schema or fake_mode or trace_mode or online_mode_can_construct" -q +``` + +Expected: PASS, and every malformed mutation raises `jsonschema.ValidationError`. + +- [ ] **Step 6: Commit the report contract** + +```powershell +git add examples/optimization/eval_optimize_loop/optimization_report.schema.json tests/evaluation/test_eval_optimize_loop_example.py +git commit -m "feat: enforce optimization report schema" +``` + +--- + +### Task 6: Make the Public Fixtures Prove Every Required Scenario + +**Files:** +- Modify: `examples/optimization/eval_optimize_loop/fixtures/fake_outputs.json` +- Modify: `examples/optimization/eval_optimize_loop/README.md` +- Modify: `tests/evaluation/test_eval_optimize_loop_example.py:171` +- Regenerate: `examples/optimization/eval_optimize_loop/fixtures/optimization_report.sample.json` + +**Interfaces:** +- Produces: one accepted improvement, one rejected no-op, and one rejected aggregate validation regression with train improvement. +- Produces: a 300-500-character design explanation. + +- [ ] **Step 1: Strengthen the public overfit assertion before changing fixtures** + +```python +def test_public_candidates_cover_success_noop_and_aggregate_regression(tmp_path: Path): + module = load_pipeline_module() + report = module.make_report( + mode="fake", + run_id="public_scenarios", + run_dir=tmp_path, + seed=7, + started=module.time.perf_counter(), + ) + candidates = {item["id"]: item for item in report["candidates"]} + assert candidates["candidate_local_patch"]["gate"]["accepted"] is True + assert candidates["candidate_noop"]["delta"]["validation_score"] == 0 + assert candidates["candidate_noop"]["gate"]["accepted"] is False + overfit = candidates["candidate_overfit"] + assert overfit["delta"]["train_score"] > 0 + assert overfit["delta"]["validation_score"] < 0 + assert overfit["gate"]["accepted"] is False +``` + +- [ ] **Step 2: Verify the current overfit fixture has only net-zero validation change** + +Run: + +```powershell +python -m pytest tests/evaluation/test_eval_optimize_loop_example.py::test_public_candidates_cover_success_noop_and_aggregate_regression -q +``` + +Expected: FAIL because the current overfit candidate fixes one validation case and breaks one, producing a zero aggregate delta. + +- [ ] **Step 3: Make the overfit fixture regress aggregate validation** + +In `candidate_overfit.outputs`, keep all three train outputs correct, keep `val_address_change_102` correct, keep `val_shipping_delay_103` incorrectly escalated, and set `val_refund_window_101` to the baseline FAQ output: + +```json +"val_refund_window_101": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund window policy question can be answered by FAQ.\"}" +``` + +This changes validation from baseline `2/3` to overfit `1/3` while train rises from `1/3` to `3/3`. + +- [ ] **Step 4: Replace Design Notes with a 300-500-character explanation** + +Use this text: + +```markdown +## Design Notes + +本示例把评测、归因、候选生成、验证回归和产品 gate 组织成一个可复现闭环。fake 与 trace 模式只替换 agent 输出来源，分数、逐 case pass/fail 和 metric 明细仍由 AgentEvaluator 生成，因此 CI 不依赖 API，也不会用 fixture 直接冒充分数。online 模式调用 AgentOptimizer 和 TargetPrompt，optimizer_dev 只服务优化器，val 仅参与 baseline 与最终候选复评，避免验证集答案进入 prompt 搜索。 + +报告先写 JSON，再渲染 Markdown。每个候选保存 prompt 摘要与哈希、训练和验证结果、逐 case 变化、失败原因、gate 检查、成本和耗时。gate 要求验证分数严格超过阈值，且不得新增 hard fail、关键 case 退化、必需 metric 失败或预算越界；成本未知且配置了成本上限时按失败处理。候选只写入运行目录，原始 prompt 不会被覆盖，随机种子、配置哈希和环境快照用于复现实验。 +``` + +Add: + +```python +def test_design_notes_length_is_within_issue_limit(): + readme = (EXAMPLE_DIR / "README.md").read_text(encoding="utf-8") + section = readme.split("## Design Notes", 1)[1].split("## Verification", 1)[0] + non_whitespace = len("".join(section.split())) + assert 300 <= non_whitespace <= 500 +``` + +- [ ] **Step 5: Regenerate the sample from the hardened fake pipeline** + +Run: + +```powershell +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake --output-dir runs --run-id sample +Copy-Item -LiteralPath runs/sample/optimization_report.json -Destination examples/optimization/eval_optimize_loop/fixtures/optimization_report.sample.json -Force +``` + +Normalize the sample deterministically: + +```powershell +$code = @' +import json +import os +from pathlib import Path + +repo = Path.cwd().resolve() +sample = Path("examples/optimization/eval_optimize_loop/fixtures/optimization_report.sample.json") +report = json.loads(sample.read_text(encoding="utf-8")) + +def normalize(value): + if isinstance(value, dict): + return {key: normalize(item) for key, item in value.items()} + if isinstance(value, list): + return [normalize(item) for item in value] + if isinstance(value, str): + prefix = str(repo) + os.sep + return value.replace(prefix, "").replace("\\", "/") + return value + +report = normalize(report) +report["duration_seconds"] = 0.0 +report["environment_snapshot"].update({ + "git_commit": "sample", + "git_dirty": False, + "python_version": "3.x", + "sdk_version": "sample", + "command": ( + "python examples/optimization/eval_optimize_loop/run_pipeline.py " + "--mode fake --output-dir runs --run-id sample" + ), +}) +sample.write_text( + json.dumps(report, indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", +) +'@ +python -c $code +``` + +Do not remove required fields or hand-edit computed scores. + +- [ ] **Step 6: Verify public scenarios, sample schema, and whitespace** + +Run: + +```powershell +python -m pytest tests/evaluation/test_eval_optimize_loop_example.py -k "public_candidates or design_notes or sample_report" -q +git diff --check +``` + +Expected: PASS; the sample has exactly one newline at EOF and no trailing blank line. + +- [ ] **Step 7: Commit fixtures and documentation** + +```powershell +git add examples/optimization/eval_optimize_loop/fixtures/fake_outputs.json examples/optimization/eval_optimize_loop/fixtures/optimization_report.sample.json examples/optimization/eval_optimize_loop/README.md tests/evaluation/test_eval_optimize_loop_example.py +git commit -m "docs: align optimization example with issue scenarios" +``` + +--- + +### Task 7: Restore Online Resource and Warning Observability + +**Files:** +- Modify: `examples/optimization/eval_optimize_loop/run_pipeline.py:188` +- Modify: `examples/optimization/eval_optimize_loop/run_pipeline.py:1469` +- Modify: `trpc_agent_sdk/models/openai_adapter/_deepseek.py:53` +- Modify: `tests/evaluation/test_eval_optimize_loop_example.py:971` + +**Interfaces:** +- Guarantees: every per-call `Runner` is closed in a `finally` block. +- Guarantees: provider limitations remain visible and are not hidden by repository-wide log-level changes. + +- [ ] **Step 1: Add a unit test that Runner closes on success and failure** + +```python +@pytest.mark.asyncio +@pytest.mark.parametrize("raise_during_run", [False, True]) +async def test_online_call_agent_closes_runner( + monkeypatch: pytest.MonkeyPatch, + raise_during_run: bool, +): + module = load_pipeline_module() + closed = [] + + class FakeRunner: + def __init__(self, **kwargs): + pass + + async def run_async(self, **kwargs): + if raise_during_run: + raise RuntimeError("model stream failed") + if False: + yield None + + async def close(self): + closed.append(True) + + import trpc_agent_sdk.runners as runners + + monkeypatch.setattr(runners, "Runner", FakeRunner) + monkeypatch.setattr( + module, + "_make_llm_agent_from_prompts", + lambda prompt_texts: object(), + ) + call_agent = module.make_online_call_agent( + system_prompt=EXAMPLE_DIR / "agent" / "prompts" / "system.md", + router_prompt=EXAMPLE_DIR / "agent" / "prompts" / "router.md", + ) + if raise_during_run: + with pytest.raises(RuntimeError, match="model stream failed"): + await call_agent("hello") + else: + assert await call_agent("hello") == "" + assert closed == [True] +``` + +- [ ] **Step 2: Verify the current implementation leaks the Runner lifecycle** + +Run: + +```powershell +python -m pytest tests/evaluation/test_eval_optimize_loop_example.py::test_online_call_agent_closes_runner -q +``` + +Expected: FAIL because `runner.close()` is never called. + +- [ ] **Step 3: Close Runner in a finally block** + +Wrap the run loop in: + +```python + try: + async for event in runner.run_async( + user_id=user_id, + session_id=session_id, + new_message=Content(role="user", parts=[Part.from_text(text=query)]), + ): + if not event.is_final_response() or not event.content: + continue + for part in event.content.parts or []: + if not part.thought and part.text: + final += part.text + finally: + await runner.close() + return final.strip() +``` + +- [ ] **Step 4: Remove symptom-hiding changes** + +Delete `KNOWN_ONLINE_WARNING_FILTERS`, `install_known_online_warning_filters`, and its call from `run_online`. Restore: + +```python +logger.warning( + "DeepSeek only supports JSON object response_format; response schema is ignored." +) +``` + +Remove the real-API assertions that require those warning strings to disappear. Replace them with report assertions for successful completion, gate correctness, source-prompt immutability, schema validity, and no secret values. + +- [ ] **Step 5: Re-run online wiring tests before consuming API** + +Run: + +```powershell +python -m pytest tests/evaluation/test_eval_optimize_loop_example.py -k "online and not e2e" -q +``` + +Expected: PASS with no real API call. + +- [ ] **Step 6: Run both real-API product decisions explicitly** + +Accepted path: + +```powershell +$env:RUN_ONLINE_E2E='1' +python -m pytest tests/evaluation/test_eval_optimize_loop_example.py::test_online_e2e_smoke_with_real_api -q -s +``` + +Rejected path: + +```powershell +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode online --output-dir runs/online_verify --run-id default_reject --gate-config runs/judge_20260710/online_gate_300.json +``` + +Expected accepted path: baseline validation below candidate validation and `gate_decision.accepted == true`. + +Expected rejected path: baseline validation equals candidate validation and the gate reason contains `validation score did not improve`. + +- [ ] **Step 7: Commit online lifecycle and observability** + +```powershell +git add examples/optimization/eval_optimize_loop/run_pipeline.py trpc_agent_sdk/models/openai_adapter/_deepseek.py tests/evaluation/test_eval_optimize_loop_example.py +git commit -m "fix: close online evaluation resources" +``` + +--- + +### Task 8: Final Verification, Scope Audit, and Honest Test Claim + +**Files:** +- Review: `examples/optimization/eval_optimize_loop/**` +- Review: `tests/evaluation/test_eval_optimize_loop_example.py` +- Review: `trpc_agent_sdk/evaluation/_agent_evaluator.py` +- Review: `pyproject.toml` +- Review separately: `tests/conftest.py` + +**Interfaces:** +- Produces: a clean, issue-scoped commit series and an evidence-backed completion statement. + +- [ ] **Step 1: Confirm no final-validation leakage** + +Run: + +```powershell +rg -n "val_path|val_evalset|validation_dataset_path|best_prompts|optimize\\(" examples/optimization/eval_optimize_loop/run_pipeline.py +``` + +Verify manually: + +- `AgentOptimizer.optimize(... validation_dataset_path=optimizer_dev_path)`; +- `val_path` is absent from optimizer inputs and reflection context; +- `val_path` is used only in baseline final scoring, candidate final scoring, deltas, and gate decisions. + +- [ ] **Step 2: Run deterministic acceptance tests** + +```powershell +python -m pytest tests/evaluation/test_eval_optimize_loop_example.py -q +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake --output-dir runs/final_verify --run-id fake +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode trace --output-dir runs/final_verify --run-id trace +``` + +Expected: focused suite passes with only the explicit online opt-in skip; fake and trace finish below 180 seconds and write schema-valid JSON plus Markdown. + +- [ ] **Step 3: Run evaluation regressions** + +```powershell +python -m pytest tests/evaluation -q +``` + +Expected: exit 0, excluding only explicitly reported opt-in skips. + +- [ ] **Step 4: Run style and source checks in the declared dev environment** + +```powershell +python -m pip install -e ".[dev,eval,optimize]" +python -m black --check examples/optimization/eval_optimize_loop tests/evaluation/test_eval_optimize_loop_example.py tests/conftest.py +python -m flake8 examples/optimization/eval_optimize_loop tests/evaluation/test_eval_optimize_loop_example.py tests/conftest.py +python -m compileall -q examples/optimization/eval_optimize_loop trpc_agent_sdk/evaluation/_agent_evaluator.py +git diff --check +``` + +Expected: all commands exit 0. + +- [ ] **Step 5: Treat full-repository failures as a separate compatibility decision** + +Run: + +```powershell +python -m pytest -q +``` + +If Windows still fails POSIX-specific suites, record the exact failures and verify the same commit in the repository's supported Linux CI image. Do not broaden this issue into code-executor, shell-tool, symlink, file-mode, or LangGraph-version fixes. Do not use `tests/conftest.py` to hide platform failures; either remove that file from this issue's diff or move its optional-dependency policy into a separately reviewed test-infrastructure change. + +- [ ] **Step 6: Audit the final diff for scope and secrets** + +```powershell +git diff origin/main...HEAD --stat +git diff origin/main...HEAD -- examples/optimization/eval_optimize_loop tests/evaluation/test_eval_optimize_loop_example.py trpc_agent_sdk/evaluation/_agent_evaluator.py pyproject.toml +$changed = git diff --unified=0 origin/main...HEAD +$hits = $changed | Select-String -Pattern "(sk-[A-Za-z0-9_-]{12,}|TRPC_AGENT_API_KEY=.+|Authorization: Bearer .+)" +if ($hits) { $hits; throw "possible credential found in changed lines" } +``` + +Expected: no credentials, no generated `runs/` content, no source-prompt mutation, and no unrelated production refactor. + +- [ ] **Step 7: Create the final verification commit only if needed** + +If verification changed only sample normalization or documentation: + +```powershell +git add examples/optimization/eval_optimize_loop/fixtures/optimization_report.sample.json examples/optimization/eval_optimize_loop/README.md +git commit -m "chore: finalize optimization example artifacts" +``` + +If verification changed nothing, do not create an empty commit. + +--- + +## Issue Coverage Map + +| Issue requirement | Plan coverage | +| --- | --- | +| Train and validation baseline scoring through AgentEvaluator | Tasks 3, 8 | +| Per-case metric, pass/fail, reason, and key trace | Tasks 1, 3, 5 | +| Failure clustering with explainable reasons | Tasks 1, 5 | +| AgentOptimizer/TargetPrompt optimization | Tasks 4, 7, 8 | +| Final-validation re-run and per-case change classes | Tasks 2, 3 | +| Configurable score, hard-fail, critical, cost, duration, metric gates | Tasks 2, 5 | +| Every candidate/round prompt, result, decision, cost, duration, seed | Task 4 | +| JSON and Markdown reports | Tasks 3, 5, 6 | +| Fake/trace without API and below three minutes | Tasks 6, 8 | +| Three train plus three validation public cases | Task 6 | +| Success, no-op, and aggregate regression scenarios | Task 6 | +| Hidden decision robustness at or above 80% | Tasks 1, 2, 5 | +| Failure attribution robustness at or above 75% | Tasks 1, 5 | +| 300-500-character design explanation | Task 6 | +| Real API accepted and rejected paths | Task 7 | +| Honest repository-wide test statement | Task 8 | + +## Completion Standard + +The work is complete only when the focused suite, evaluation suite, fake run, trace run, report-schema mutation tests, adversarial gate matrix, adversarial attribution matrix, and both opt-in real-API decisions have fresh passing evidence. A Linux full-suite result may be reported separately from Windows platform failures, but neither may be described as passing without an exit-zero command from that environment. diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md new file mode 100644 index 00000000..1c72d2df --- /dev/null +++ b/examples/optimization/eval_optimize_loop/README.md @@ -0,0 +1,169 @@ +# Evaluation + Optimization Loop Example + +This example demonstrates an auditable evaluation and prompt-optimization loop. +It is intentionally offline-first: `fake` and `trace` modes need no API key. + +## Modes + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode trace +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode online +``` + +`fake` mode evaluates deterministic fixture outputs for baseline and three +candidates through `AgentEvaluator`. It should select `candidate_local_patch`, +reject a no-op candidate, and reject `candidate_overfit`, which improves +training while regressing a critical validation case. + +`trace` mode materializes recorded conversations for baseline and every +candidate, then runs the same `AgentEvaluator` summary path with +`eval_mode="trace"`. It reads `fixtures/trace_outputs.json`, independently of +the fake-mode fixture, and proves the replay path works without model inference. + +`online` mode uses `AgentOptimizer.optimize(...)`, `TargetPrompt`, and +`optimizer.json`. It first prints only whether required environment variables +are present. It requires: + +```bash +TRPC_AGENT_API_KEY +TRPC_AGENT_BASE_URL +TRPC_AGENT_MODEL_NAME +``` + +The online smoke path is opt-in because it performs real optimizer and +revalidation calls. The default optimizer config is bounded for this example, +but real-provider latency is not held to the fake/trace three-minute +deterministic expectation. A native optimizer `SUCCEEDED` status is recorded as +an artifact only; the product decision is always the report's +`gate_decision.accepted`. + +## Outputs + +Each run writes to `runs//` by default: + +- `optimization_report.json`: issue-facing machine-readable summary. +- `optimization_report.md`: concise human-readable report. +- `trace_evalset.json` and `trace_metrics.json`: trace-mode replay inputs. +- `online/result.json`, `online/summary.txt`, `online/rounds/`, + `online/baseline_prompts/`, `online/best_prompts/`, and + `online/config.snapshot.json`: native optimizer artifacts in online mode. + +The top-level report always includes `run_id`, `mode`, `seed`, `baseline`, +`candidates`, `delta`, `gate_decision`, `failure_attribution`, `cost`, +`duration_seconds`, `config_snapshot`, `environment_snapshot`, and `artifacts`. +`optimization_report.schema.json` is the contract used by the CLI before it +writes `optimization_report.json`; schema validation failures stop report +generation instead of producing a partial artifact. + +In online mode, the duration gate uses elapsed pipeline time through +optimization, baseline revalidation, and candidate revalidation. The separate +`online_duration` fields retain those phase durations, while each candidate's +audit retains its own revalidation duration. Optimizer `model_calls` includes +candidate-evaluation agent calls, reflection calls, and judge calls. Optimizer +judge calls are derived from counted candidate evaluations and every configured +judge model sample. `judge_calls_per_candidate_evaluation` records that +multiplier; a nonzero native counter is reconciled with the derived count +without double counting, and `judge_model_call_source` records the source. +Final revalidation records the corresponding `judge_calls_per_agent_call` and +counts every conversation turn, not only top-level cases. Because +`AgentOptimizer` does not expose token or cost usage for candidate-evaluation +or judge calls, optimizer phase totals are `null` and marked unknown whenever +those calls occur. `optimizer.reflection_reported_usage` preserves the native +reflection-only cost and token counters under an explicit scope. Final +revalidation token usage is likewise `null` when `AgentEvaluator` cannot expose +it. Top-level `model_calls` is the optimizer phase plus final revalidation. + +A compact sample output is checked in at `fixtures/optimization_report.sample.json`. + +## CLI Inputs + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py \ + --mode fake \ + --train-evalset train.evalset.json \ + --optimizer-dev-evalset optimizer_dev.evalset.json \ + --val-evalset val.evalset.json \ + --optimizer-config optimizer.json \ + --gate-config gate.json \ + --output-dir runs \ + --run-id demo +``` + +All path arguments have defaults pointing at this example. `--system-prompt` +and `--router-prompt` are used by online mode and are still recorded in offline +configuration snapshots. Gate config may override validation delta, hard-fail, +critical-regression, cost, duration, and required-metric checks. By default the +gate inherits `optimize.stop.required_metrics` from `optimizer.json`. +Gate booleans require JSON booleans, numeric thresholds require finite +non-negative JSON numbers, and malformed values reject without raising. The +gate accepts only the documented field names; misspelled or unknown keys fail +closed instead of falling back to a weaker default. The +three evalset roles must be different files with no byte-identical content and +no overlap in case IDs, normalized user inputs, or canonical gold outputs. +Overlap checks cover every conversation turn and canonicalize valid JSON by +structure, so whitespace and key ordering cannot hide shared gold data. + +The deterministic metric in this example is `route_tool_args_score`: it parses +the final JSON response and scores only `route`, `tool.name`, and +`tool.arguments`. Reason wording and safety are handled by the rubric metric, so +a harmless explanation rewrite does not zero out an otherwise correct route. + +`environment_snapshot` records the git commit, dirty flag, Python version, SDK +version when installed, model name, redacted base URL host, seed, command, and +optimizer config path. It never records API keys. When a judge provider does +not support native JSON schemas, the judge uses JSON-object mode and validates +the response locally instead of sending an ignored schema. +Provider and runtime warnings are not globally suppressed. Judge calls run +non-streaming and explicitly close their agent run so normal online evaluation +does not leave OpenAI/httpx stream-shutdown diagnostics behind. +The pipeline awaits `Runner.close()` for its agent execution paths. +If an upstream OpenAI/httpx diagnostic still occurs outside this lifecycle, +warnings remain observable rather than being filtered. + +Report error text redacts provider URLs, configured provider credentials, +standalone provider-key shapes, and semantic credential markers while retaining +ordinary error context. Run IDs, candidate IDs, and optimizer prompt artifact +names also reject or normalize Windows reserved device basenames on every +platform. Before each write, report, metrics, trace, prompt, and optimizer +output paths are resolved beneath the run directory; pre-existing symlink +redirection is rejected. + +Online mode writes an exact run-local `optimizer.json`, overrides its algorithm +seed from `--seed`, and passes that file to `AgentOptimizer`. Candidate and +environment audits reference the same path and SHA-256. `config_snapshot` also +stores a normalized, credential-free evaluation configuration and its hash; +report validation derives optimizer and final judge-call multipliers from that +snapshot rather than trusting the reported counters. Evaluation-metric and +evalset manifests bind that snapshot to file hashes, case counts, and +conversation-turn counts. A prompt-target manifest binds the registered target +names to source paths and SHA-256 values. Validation authenticates those source +files, recomputes every candidate diff from embedded baseline content, and +rejects unknown optimizer targets. Prompt artifacts and optimizer rounds embed +content so their reported SHA-256 values remain independently recomputable even +when the original run directory is unavailable. + +`optimizer_dev.evalset.json` is the optimizer-internal holdout passed to +`AgentOptimizer.optimize(..., validation_dataset_path=...)`. `val.evalset.json` +is the final validation set and is only used for baseline scoring and final +candidate gate scoring. + +## Design Notes + +本示例把评测、归因、候选生成、验证回归和产品 gate 组织成一个可复现闭环。fake 与 trace 模式只替换 agent 输出来源，分数、逐 case pass/fail 和 metric 明细仍由 AgentEvaluator 生成，因此 CI 不依赖 API，也不会用 fixture 直接冒充分数。online 模式调用 AgentOptimizer 和 TargetPrompt，optimizer_dev 只服务优化器，val 仅参与 baseline 与最终候选复评，避免验证集答案进入 prompt 搜索。 + +报告先写 JSON，再渲染 Markdown。每个候选保存 prompt 摘要与哈希、训练和验证结果、逐 case 变化、失败原因、gate 检查、成本和耗时。gate 要求验证分数严格超过阈值，且不得新增 hard fail、关键 case 退化、必需 metric 失败或预算越界；成本未知且配置了成本上限时按失败处理。候选只写入运行目录，原始 prompt 不会被覆盖，随机种子、配置哈希和环境快照用于复现实验。 + +## Verification + +```bash +pytest tests/evaluation/test_eval_optimize_loop_example.py -q +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode trace +``` + +Full repository pytest includes optional integration suites. In an environment +without optional extras such as Cube/E2B, Mempalace, A2A, AG-UI, Claude Agent +SDK, or OpenClaw dependencies, pytest may fail during collection before any +tests run. Those dependency errors must be reported as an environment boundary; +this example does not install a global collection hook to hide them. diff --git a/examples/optimization/eval_optimize_loop/agent/__init__.py b/examples/optimization/eval_optimize_loop/agent/__init__.py new file mode 100644 index 00000000..b0bbbeb0 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/agent/__init__.py @@ -0,0 +1 @@ +"""Support-router agent used by the eval/optimization loop example.""" diff --git a/examples/optimization/eval_optimize_loop/agent/agent.py b/examples/optimization/eval_optimize_loop/agent/agent.py new file mode 100644 index 00000000..0479d941 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/agent/agent.py @@ -0,0 +1,40 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Online agent factory for the eval/optimization loop example.""" + +from __future__ import annotations + +from pathlib import Path + +from trpc_agent_sdk.agents import LlmAgent +from trpc_agent_sdk.models import OpenAIModel + +from .config import get_model_config + +PROMPT_DIR = Path(__file__).resolve().parent / "prompts" +SYSTEM_PROMPT_PATH = PROMPT_DIR / "system.md" +ROUTER_PROMPT_PATH = PROMPT_DIR / "router.md" + + +def create_agent() -> LlmAgent: + """Create an LlmAgent that re-reads prompt files on every call.""" + + api_key, base_url, model_name = get_model_config() + instruction = "\n\n".join( + [ + SYSTEM_PROMPT_PATH.read_text(encoding="utf-8").strip(), + ROUTER_PROMPT_PATH.read_text(encoding="utf-8").strip(), + ] + ) + return LlmAgent( + name="support_router_agent", + model=OpenAIModel( + model_name=model_name, + api_key=api_key, + base_url=base_url, + ), + instruction=instruction, + ) diff --git a/examples/optimization/eval_optimize_loop/agent/config.py b/examples/optimization/eval_optimize_loop/agent/config.py new file mode 100644 index 00000000..2950de85 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/agent/config.py @@ -0,0 +1,33 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Environment configuration for the online mode example.""" + +from __future__ import annotations + +import os + +REQUIRED_ENV_VARS = ( + "TRPC_AGENT_API_KEY", + "TRPC_AGENT_BASE_URL", + "TRPC_AGENT_MODEL_NAME", +) + + +def get_model_config() -> tuple[str, str, str]: + """Return API key, base URL, and model name for online mode.""" + + missing = [name for name in REQUIRED_ENV_VARS if not os.getenv(name)] + if missing: + raise ValueError( + "online mode requires environment variables: " + + ", ".join(REQUIRED_ENV_VARS) + + f"; missing: {', '.join(missing)}" + ) + return ( + os.environ["TRPC_AGENT_API_KEY"], + os.environ["TRPC_AGENT_BASE_URL"], + os.environ["TRPC_AGENT_MODEL_NAME"], + ) diff --git a/examples/optimization/eval_optimize_loop/agent/prompts/router.md b/examples/optimization/eval_optimize_loop/agent/prompts/router.md new file mode 100644 index 00000000..438bffc2 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/agent/prompts/router.md @@ -0,0 +1,20 @@ +You route customer-support requests to one backend action. + +Output exactly one JSON object with this shape: + +{"route":"","tool":{"name":"","arguments":{}},"reason":""} + +Allowed routes and tools: + +- refund -> create_refund_ticket +- manual_escalation -> create_escalation_case +- faq -> none + +Routing policy: + +1. Use refund when the user asks for refund handling for a broken, missing, or unusable delivered item. +2. Use manual_escalation when the user explicitly requests a human agent for an account-access, safety, legal, or threat-of-reporting issue. +3. Use faq for policy, shipping-status, coupon, address-change, and informational questions that do not require opening a backend case. +4. When the request is ambiguous, choose faq and explain that it can be handled as a policy or information question. + +Keep tool.arguments as an empty object for this example. Do not include Markdown, comments, or extra keys. diff --git a/examples/optimization/eval_optimize_loop/agent/prompts/system.md b/examples/optimization/eval_optimize_loop/agent/prompts/system.md new file mode 100644 index 00000000..3b75636b --- /dev/null +++ b/examples/optimization/eval_optimize_loop/agent/prompts/system.md @@ -0,0 +1,9 @@ +You are a support-routing agent. + +Return only one compact JSON object. Do not include markdown fences or extra text. +The JSON shape is: + +{"route":"","tool":{"name":"","arguments":{}},"reason":""} + +Keep the reason short and do not promise refunds, credits, or manual handling +unless the route rules require it. diff --git a/examples/optimization/eval_optimize_loop/fixtures/fake_outputs.json b/examples/optimization/eval_optimize_loop/fixtures/fake_outputs.json new file mode 100644 index 00000000..bf31482b --- /dev/null +++ b/examples/optimization/eval_optimize_loop/fixtures/fake_outputs.json @@ -0,0 +1,58 @@ +{ + "baseline": { + "prompt_patch_summary": "Baseline overuses FAQ and misses refund and account-escalation intent.", + "outputs": { + "train_refund_001": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund policy question can be answered by FAQ.\"}", + "train_manual_002": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Account question can be answered by FAQ.\"}", + "train_faq_003": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Coupon policy question can be answered by FAQ.\"}", + "dev_return_201": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Return policy can be answered by FAQ.\"}", + "dev_access_202": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Access policy can be answered by FAQ.\"}", + "dev_warranty_203": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Replacement charger warranty is a policy question for FAQ.\"}", + "val_refund_window_101": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund window policy question can be answered by FAQ.\"}", + "val_address_change_102": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "val_shipping_delay_103": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}" + } + }, + "candidate_local_patch": { + "prompt_patch_summary": "Adds local routing rules for explicit refund requests and account-escalation demands while preserving FAQ for ordinary policy questions.", + "outputs": { + "train_refund_001": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Broken delivered item needs refund handling.\"}", + "train_manual_002": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Account lock with escalation demand needs a human case.\"}", + "train_faq_003": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Coupon policy question can be answered by FAQ.\"}", + "dev_return_201": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Leaking new appliance with an explicit return request needs refund handling.\"}", + "dev_access_202": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Failed two-factor delivery with a specialist request needs a human case.\"}", + "dev_warranty_203": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Replacement charger warranty is a policy question for FAQ.\"}", + "val_refund_window_101": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Missing parts with refund request should open refund handling.\"}", + "val_address_change_102": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "val_shipping_delay_103": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}" + } + }, + "candidate_noop": { + "prompt_patch_summary": "Rephrases baseline guidance without changing route decisions.", + "outputs": { + "train_refund_001": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund policy question can be answered by FAQ.\"}", + "train_manual_002": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Account question can be answered by FAQ.\"}", + "train_faq_003": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Coupon policy question can be answered by FAQ.\"}", + "dev_return_201": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Return policy can be answered by FAQ.\"}", + "dev_access_202": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Access policy can be answered by FAQ.\"}", + "dev_warranty_203": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Replacement charger warranty is a policy question for FAQ.\"}", + "val_refund_window_101": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund window policy question can be answered by FAQ.\"}", + "val_address_change_102": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "val_shipping_delay_103": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}" + } + }, + "candidate_overfit": { + "prompt_patch_summary": "Overfits to emotional wording and sends angry logistics questions to manual escalation.", + "outputs": { + "train_refund_001": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Broken delivered item needs refund handling.\"}", + "train_manual_002": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Account lock with escalation demand needs a human case.\"}", + "train_faq_003": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Coupon policy question can be answered by FAQ.\"}", + "dev_return_201": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Leaking new appliance with an explicit return request needs refund handling.\"}", + "dev_access_202": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Failed two-factor delivery with a specialist request needs a human case.\"}", + "dev_warranty_203": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Replacement charger warranty is a policy question for FAQ.\"}", + "val_refund_window_101": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund window policy question can be answered by FAQ.\"}", + "val_address_change_102": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "val_shipping_delay_103": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Angry language should be escalated to a person.\"}" + } + } +} diff --git a/examples/optimization/eval_optimize_loop/fixtures/offline_metrics.sample.json b/examples/optimization/eval_optimize_loop/fixtures/offline_metrics.sample.json new file mode 100644 index 00000000..2e897f58 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/fixtures/offline_metrics.sample.json @@ -0,0 +1,29 @@ +{ + "metrics": [ + { + "metric_name": "route_tool_args_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "json": { + "match": "exact" + } + } + } + }, + { + "metric_name": "llm_rubric_response", + "threshold": 1.0, + "criterion": { + "offline_rubric": { + "checks": [ + "valid_json_object", + "route_present", + "tool_object_present" + ] + } + } + } + ], + "num_runs": 1 +} diff --git a/examples/optimization/eval_optimize_loop/fixtures/optimization_report.sample.json b/examples/optimization/eval_optimize_loop/fixtures/optimization_report.sample.json new file mode 100644 index 00000000..fb518375 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/fixtures/optimization_report.sample.json @@ -0,0 +1,2786 @@ +{ + "run_id": "sample", + "mode": "fake", + "seed": 7, + "baseline": { + "prompt_patch_summary": "Baseline overuses FAQ and misses refund and account-escalation intent.", + "prompt_artifacts": [ + { + "name": "system_prompt", + "source_path": "examples/optimization/eval_optimize_loop/agent/prompts/system.md", + "candidate_path": "runs/sample/prompts/baseline/system_prompt.md", + "sha256": "ac2208517d00b42bccfc2336108753e0e3c0ea238345c6b851896b09a4819ab3", + "source_written": false, + "summary": "Baseline overuses FAQ and misses refund and account-escalation intent.", + "diff": "unchanged", + "content": "You are a support-routing agent.\n\nReturn only one compact JSON object. Do not include markdown fences or extra text.\nThe JSON shape is:\n\n{\"route\":\"\",\"tool\":{\"name\":\"\",\"arguments\":{}},\"reason\":\"\"}\n\nKeep the reason short and do not promise refunds, credits, or manual handling\nunless the route rules require it.\n" + }, + { + "name": "router_prompt", + "source_path": "examples/optimization/eval_optimize_loop/agent/prompts/router.md", + "candidate_path": "runs/sample/prompts/baseline/router_prompt.md", + "sha256": "1897b273cf69ace89b50e5c9ba77e135935544bd67100ecd11c78ee7fa317f10", + "source_written": false, + "summary": "Baseline overuses FAQ and misses refund and account-escalation intent.", + "diff": "unchanged", + "content": "You route customer-support requests to one backend action.\n\nOutput exactly one JSON object with this shape:\n\n{\"route\":\"\",\"tool\":{\"name\":\"\",\"arguments\":{}},\"reason\":\"\"}\n\nAllowed routes and tools:\n\n- refund -> create_refund_ticket\n- manual_escalation -> create_escalation_case\n- faq -> none\n\nRouting policy:\n\n1. Use refund when the user asks for refund handling for a broken, missing, or unusable delivered item.\n2. Use manual_escalation when the user explicitly requests a human agent for an account-access, safety, legal, or threat-of-reporting issue.\n3. Use faq for policy, shipping-status, coupon, address-change, and informational questions that do not require opening a backend case.\n4. When the request is ambiguous, choose faq and explain that it can be handled as a policy or information question.\n\nKeep tool.arguments as an empty object for this example. Do not include Markdown, comments, or extra keys.\n" + } + ], + "train": { + "eval_set_id": "support_router_train", + "score": 0.333333, + "pass_rate": 0.333333, + "metrics": { + "route_tool_args_score": { + "score": 0.333333, + "threshold": 1.0, + "passed": false, + "status": "failed" + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "passed": true, + "status": "passed" + } + }, + "case_results": [ + { + "case_id": "train_refund_001", + "tags": [ + "refund", + "train" + ], + "user": "The headphones arrived broken yesterday. I want a refund.", + "score": 0.0, + "passed": false, + "metrics": { + "route_tool_args_score": { + "score": 0.0, + "threshold": 1.0, + "status": "failed", + "passed": false, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund policy question can be answered by FAQ.\"}", + "expected_text": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Broken delivered item needs refund handling.\"}", + "key_trace": { + "invocation_id": "train-1", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund policy question can be answered by FAQ.\"}", + "expected_final_response": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Broken delivered item needs refund handling.\"}", + "error_message": null + }, + "root_cause": "final_response_mismatch", + "reasons": [ + "actual route 'faq' did not match expected route 'refund'" + ] + }, + { + "case_id": "train_manual_002", + "tags": [ + "manual", + "train" + ], + "user": "My account is locked and I am going to report this unless a person helps me now.", + "score": 0.0, + "passed": false, + "metrics": { + "route_tool_args_score": { + "score": 0.0, + "threshold": 1.0, + "status": "failed", + "passed": false, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Account question can be answered by FAQ.\"}", + "expected_text": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Account lock with escalation demand needs a human case.\"}", + "key_trace": { + "invocation_id": "train-2", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Account question can be answered by FAQ.\"}", + "expected_final_response": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Account lock with escalation demand needs a human case.\"}", + "error_message": null + }, + "root_cause": "final_response_mismatch", + "reasons": [ + "actual route 'faq' did not match expected route 'manual_escalation'" + ] + }, + { + "case_id": "train_faq_003", + "tags": [ + "faq", + "train" + ], + "user": "Can an expired coupon be reissued?", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Coupon policy question can be answered by FAQ.\"}", + "expected_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Coupon policy question can be answered by FAQ.\"}", + "key_trace": { + "invocation_id": "train-3", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Coupon policy question can be answered by FAQ.\"}", + "expected_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Coupon policy question can be answered by FAQ.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + } + ], + "failed_case_ids": [ + "train_refund_001", + "train_manual_002" + ], + "source": "AgentEvaluator" + }, + "optimizer_dev": { + "eval_set_id": "support_router_optimizer_dev", + "score": 0.333333, + "pass_rate": 0.333333, + "metrics": { + "route_tool_args_score": { + "score": 0.333333, + "threshold": 1.0, + "passed": false, + "status": "failed" + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "passed": true, + "status": "passed" + } + }, + "case_results": [ + { + "case_id": "dev_return_201", + "tags": [ + "refund", + "optimizer_dev" + ], + "user": "A coffee maker delivered this morning leaks water. Please return it for a refund.", + "score": 0.0, + "passed": false, + "metrics": { + "route_tool_args_score": { + "score": 0.0, + "threshold": 1.0, + "status": "failed", + "passed": false, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Return policy can be answered by FAQ.\"}", + "expected_text": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Leaking new appliance with an explicit return request needs refund handling.\"}", + "key_trace": { + "invocation_id": "optimizer-dev-1", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Return policy can be answered by FAQ.\"}", + "expected_final_response": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Leaking new appliance with an explicit return request needs refund handling.\"}", + "error_message": null + }, + "root_cause": "final_response_mismatch", + "reasons": [ + "actual route 'faq' did not match expected route 'refund'" + ] + }, + { + "case_id": "dev_access_202", + "tags": [ + "manual", + "optimizer_dev" + ], + "user": "Two-factor codes never arrive and I need a support specialist to restore access.", + "score": 0.0, + "passed": false, + "metrics": { + "route_tool_args_score": { + "score": 0.0, + "threshold": 1.0, + "status": "failed", + "passed": false, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Access policy can be answered by FAQ.\"}", + "expected_text": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Failed two-factor delivery with a specialist request needs a human case.\"}", + "key_trace": { + "invocation_id": "optimizer-dev-2", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Access policy can be answered by FAQ.\"}", + "expected_final_response": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Failed two-factor delivery with a specialist request needs a human case.\"}", + "error_message": null + }, + "root_cause": "final_response_mismatch", + "reasons": [ + "actual route 'faq' did not match expected route 'manual_escalation'" + ] + }, + { + "case_id": "dev_warranty_203", + "tags": [ + "faq", + "optimizer_dev" + ], + "user": "How long is the warranty on a replacement charger?", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Replacement charger warranty is a policy question for FAQ.\"}", + "expected_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Replacement charger warranty is a policy question for FAQ.\"}", + "key_trace": { + "invocation_id": "optimizer-dev-3", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Replacement charger warranty is a policy question for FAQ.\"}", + "expected_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Replacement charger warranty is a policy question for FAQ.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + } + ], + "failed_case_ids": [ + "dev_return_201", + "dev_access_202" + ], + "source": "AgentEvaluator" + }, + "final_validation": { + "eval_set_id": "support_router_val", + "score": 0.666667, + "pass_rate": 0.666667, + "metrics": { + "route_tool_args_score": { + "score": 0.666667, + "threshold": 1.0, + "passed": false, + "status": "failed" + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "passed": true, + "status": "passed" + } + }, + "case_results": [ + { + "case_id": "val_refund_window_101", + "tags": [ + "refund", + "validation" + ], + "user": "The blender arrived missing parts and I need a refund even though it has been 10 days.", + "score": 0.0, + "passed": false, + "metrics": { + "route_tool_args_score": { + "score": 0.0, + "threshold": 1.0, + "status": "failed", + "passed": false, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund window policy question can be answered by FAQ.\"}", + "expected_text": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Missing parts with refund request should open refund handling.\"}", + "key_trace": { + "invocation_id": "val-1", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund window policy question can be answered by FAQ.\"}", + "expected_final_response": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Missing parts with refund request should open refund handling.\"}", + "error_message": null + }, + "root_cause": "final_response_mismatch", + "reasons": [ + "actual route 'faq' did not match expected route 'refund'" + ] + }, + { + "case_id": "val_address_change_102", + "tags": [ + "faq", + "validation" + ], + "user": "Can I change the delivery address for an order that has not shipped?", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "expected_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "key_trace": { + "invocation_id": "val-2", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "expected_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + }, + { + "case_id": "val_shipping_delay_103", + "tags": [ + "faq", + "validation", + "critical" + ], + "user": "I am angry that shipping is late, but I only want to know the delivery policy.", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "expected_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "key_trace": { + "invocation_id": "val-3", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "expected_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + } + ], + "failed_case_ids": [ + "val_refund_window_101" + ], + "source": "AgentEvaluator" + }, + "validation": { + "eval_set_id": "support_router_val", + "score": 0.666667, + "pass_rate": 0.666667, + "metrics": { + "route_tool_args_score": { + "score": 0.666667, + "threshold": 1.0, + "passed": false, + "status": "failed" + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "passed": true, + "status": "passed" + } + }, + "case_results": [ + { + "case_id": "val_refund_window_101", + "tags": [ + "refund", + "validation" + ], + "user": "The blender arrived missing parts and I need a refund even though it has been 10 days.", + "score": 0.0, + "passed": false, + "metrics": { + "route_tool_args_score": { + "score": 0.0, + "threshold": 1.0, + "status": "failed", + "passed": false, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund window policy question can be answered by FAQ.\"}", + "expected_text": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Missing parts with refund request should open refund handling.\"}", + "key_trace": { + "invocation_id": "val-1", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund window policy question can be answered by FAQ.\"}", + "expected_final_response": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Missing parts with refund request should open refund handling.\"}", + "error_message": null + }, + "root_cause": "final_response_mismatch", + "reasons": [ + "actual route 'faq' did not match expected route 'refund'" + ] + }, + { + "case_id": "val_address_change_102", + "tags": [ + "faq", + "validation" + ], + "user": "Can I change the delivery address for an order that has not shipped?", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "expected_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "key_trace": { + "invocation_id": "val-2", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "expected_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + }, + { + "case_id": "val_shipping_delay_103", + "tags": [ + "faq", + "validation", + "critical" + ], + "user": "I am angry that shipping is late, but I only want to know the delivery policy.", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "expected_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "key_trace": { + "invocation_id": "val-3", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "expected_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + } + ], + "failed_case_ids": [ + "val_refund_window_101" + ], + "source": "AgentEvaluator" + } + }, + "candidates": [ + { + "id": "candidate_local_patch", + "audit": { + "seed": 7, + "duration_seconds": 0.0, + "cost": { + "currency": "USD", + "estimated": 0.0, + "known": true + }, + "config_path": "examples/optimization/eval_optimize_loop/optimizer.json", + "config_sha256": "9df0a82039d6de55660169bf6061572da99b647f0141cdfcd5c6f0b4cd448d98" + }, + "prompt_patch_summary": "Adds local routing rules for explicit refund requests and account-escalation demands while preserving FAQ for ordinary policy questions.", + "prompt_artifacts": [ + { + "name": "system_prompt", + "source_path": "examples/optimization/eval_optimize_loop/agent/prompts/system.md", + "candidate_path": "runs/sample/prompts/candidate_local_patch/system_prompt.md", + "sha256": "ac2208517d00b42bccfc2336108753e0e3c0ea238345c6b851896b09a4819ab3", + "source_written": false, + "summary": "Adds local routing rules for explicit refund requests and account-escalation demands while preserving FAQ for ordinary policy questions.", + "diff": "unchanged", + "content": "You are a support-routing agent.\n\nReturn only one compact JSON object. Do not include markdown fences or extra text.\nThe JSON shape is:\n\n{\"route\":\"\",\"tool\":{\"name\":\"\",\"arguments\":{}},\"reason\":\"\"}\n\nKeep the reason short and do not promise refunds, credits, or manual handling\nunless the route rules require it.\n" + }, + { + "name": "router_prompt", + "source_path": "examples/optimization/eval_optimize_loop/agent/prompts/router.md", + "candidate_path": "runs/sample/prompts/candidate_local_patch/router_prompt.md", + "sha256": "72579cd1131cd405b7458aef1bcd973f0d43e70c4ef6f82d1136bc7fb5ee9e67", + "source_written": false, + "summary": "Adds local routing rules for explicit refund requests and account-escalation demands while preserving FAQ for ordinary policy questions.", + "diff": "--- source/router_prompt.md\n+++ candidate/router_prompt.md\n@@ -18,3 +18,5 @@\n 4. When the request is ambiguous, choose faq and explain that it can be handled as a policy or information question.\n \n Keep tool.arguments as an empty object for this example. Do not include Markdown, comments, or extra keys.\n+\n+Offline candidate patch (candidate_local_patch): Adds local routing rules for explicit refund requests and account-escalation demands while preserving FAQ for ordinary policy questions.\n", + "content": "You route customer-support requests to one backend action.\n\nOutput exactly one JSON object with this shape:\n\n{\"route\":\"\",\"tool\":{\"name\":\"\",\"arguments\":{}},\"reason\":\"\"}\n\nAllowed routes and tools:\n\n- refund -> create_refund_ticket\n- manual_escalation -> create_escalation_case\n- faq -> none\n\nRouting policy:\n\n1. Use refund when the user asks for refund handling for a broken, missing, or unusable delivered item.\n2. Use manual_escalation when the user explicitly requests a human agent for an account-access, safety, legal, or threat-of-reporting issue.\n3. Use faq for policy, shipping-status, coupon, address-change, and informational questions that do not require opening a backend case.\n4. When the request is ambiguous, choose faq and explain that it can be handled as a policy or information question.\n\nKeep tool.arguments as an empty object for this example. Do not include Markdown, comments, or extra keys.\n\nOffline candidate patch (candidate_local_patch): Adds local routing rules for explicit refund requests and account-escalation demands while preserving FAQ for ordinary policy questions.\n" + } + ], + "train": { + "eval_set_id": "support_router_train", + "score": 1.0, + "pass_rate": 1.0, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "passed": true, + "status": "passed" + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "passed": true, + "status": "passed" + } + }, + "case_results": [ + { + "case_id": "train_refund_001", + "tags": [ + "refund", + "train" + ], + "user": "The headphones arrived broken yesterday. I want a refund.", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Broken delivered item needs refund handling.\"}", + "expected_text": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Broken delivered item needs refund handling.\"}", + "key_trace": { + "invocation_id": "train-1", + "actual_final_response": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Broken delivered item needs refund handling.\"}", + "expected_final_response": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Broken delivered item needs refund handling.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + }, + { + "case_id": "train_manual_002", + "tags": [ + "manual", + "train" + ], + "user": "My account is locked and I am going to report this unless a person helps me now.", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Account lock with escalation demand needs a human case.\"}", + "expected_text": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Account lock with escalation demand needs a human case.\"}", + "key_trace": { + "invocation_id": "train-2", + "actual_final_response": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Account lock with escalation demand needs a human case.\"}", + "expected_final_response": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Account lock with escalation demand needs a human case.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + }, + { + "case_id": "train_faq_003", + "tags": [ + "faq", + "train" + ], + "user": "Can an expired coupon be reissued?", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Coupon policy question can be answered by FAQ.\"}", + "expected_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Coupon policy question can be answered by FAQ.\"}", + "key_trace": { + "invocation_id": "train-3", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Coupon policy question can be answered by FAQ.\"}", + "expected_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Coupon policy question can be answered by FAQ.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + } + ], + "failed_case_ids": [], + "source": "AgentEvaluator" + }, + "optimizer_dev": { + "eval_set_id": "support_router_optimizer_dev", + "score": 1.0, + "pass_rate": 1.0, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "passed": true, + "status": "passed" + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "passed": true, + "status": "passed" + } + }, + "case_results": [ + { + "case_id": "dev_return_201", + "tags": [ + "refund", + "optimizer_dev" + ], + "user": "A coffee maker delivered this morning leaks water. Please return it for a refund.", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Leaking new appliance with an explicit return request needs refund handling.\"}", + "expected_text": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Leaking new appliance with an explicit return request needs refund handling.\"}", + "key_trace": { + "invocation_id": "optimizer-dev-1", + "actual_final_response": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Leaking new appliance with an explicit return request needs refund handling.\"}", + "expected_final_response": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Leaking new appliance with an explicit return request needs refund handling.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + }, + { + "case_id": "dev_access_202", + "tags": [ + "manual", + "optimizer_dev" + ], + "user": "Two-factor codes never arrive and I need a support specialist to restore access.", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Failed two-factor delivery with a specialist request needs a human case.\"}", + "expected_text": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Failed two-factor delivery with a specialist request needs a human case.\"}", + "key_trace": { + "invocation_id": "optimizer-dev-2", + "actual_final_response": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Failed two-factor delivery with a specialist request needs a human case.\"}", + "expected_final_response": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Failed two-factor delivery with a specialist request needs a human case.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + }, + { + "case_id": "dev_warranty_203", + "tags": [ + "faq", + "optimizer_dev" + ], + "user": "How long is the warranty on a replacement charger?", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Replacement charger warranty is a policy question for FAQ.\"}", + "expected_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Replacement charger warranty is a policy question for FAQ.\"}", + "key_trace": { + "invocation_id": "optimizer-dev-3", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Replacement charger warranty is a policy question for FAQ.\"}", + "expected_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Replacement charger warranty is a policy question for FAQ.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + } + ], + "failed_case_ids": [], + "source": "AgentEvaluator" + }, + "final_validation": { + "eval_set_id": "support_router_val", + "score": 1.0, + "pass_rate": 1.0, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "passed": true, + "status": "passed" + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "passed": true, + "status": "passed" + } + }, + "case_results": [ + { + "case_id": "val_refund_window_101", + "tags": [ + "refund", + "validation" + ], + "user": "The blender arrived missing parts and I need a refund even though it has been 10 days.", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Missing parts with refund request should open refund handling.\"}", + "expected_text": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Missing parts with refund request should open refund handling.\"}", + "key_trace": { + "invocation_id": "val-1", + "actual_final_response": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Missing parts with refund request should open refund handling.\"}", + "expected_final_response": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Missing parts with refund request should open refund handling.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + }, + { + "case_id": "val_address_change_102", + "tags": [ + "faq", + "validation" + ], + "user": "Can I change the delivery address for an order that has not shipped?", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "expected_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "key_trace": { + "invocation_id": "val-2", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "expected_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + }, + { + "case_id": "val_shipping_delay_103", + "tags": [ + "faq", + "validation", + "critical" + ], + "user": "I am angry that shipping is late, but I only want to know the delivery policy.", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "expected_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "key_trace": { + "invocation_id": "val-3", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "expected_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + } + ], + "failed_case_ids": [], + "source": "AgentEvaluator" + }, + "validation": { + "eval_set_id": "support_router_val", + "score": 1.0, + "pass_rate": 1.0, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "passed": true, + "status": "passed" + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "passed": true, + "status": "passed" + } + }, + "case_results": [ + { + "case_id": "val_refund_window_101", + "tags": [ + "refund", + "validation" + ], + "user": "The blender arrived missing parts and I need a refund even though it has been 10 days.", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Missing parts with refund request should open refund handling.\"}", + "expected_text": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Missing parts with refund request should open refund handling.\"}", + "key_trace": { + "invocation_id": "val-1", + "actual_final_response": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Missing parts with refund request should open refund handling.\"}", + "expected_final_response": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Missing parts with refund request should open refund handling.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + }, + { + "case_id": "val_address_change_102", + "tags": [ + "faq", + "validation" + ], + "user": "Can I change the delivery address for an order that has not shipped?", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "expected_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "key_trace": { + "invocation_id": "val-2", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "expected_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + }, + { + "case_id": "val_shipping_delay_103", + "tags": [ + "faq", + "validation", + "critical" + ], + "user": "I am angry that shipping is late, but I only want to know the delivery policy.", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "expected_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "key_trace": { + "invocation_id": "val-3", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "expected_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + } + ], + "failed_case_ids": [], + "source": "AgentEvaluator" + }, + "delta": { + "train_score": 0.666667, + "optimizer_dev_score": 0.666667, + "validation_score": 0.333333 + }, + "case_deltas": [ + { + "case_id": "val_address_change_102", + "baseline_score": 1.0, + "candidate_score": 1.0, + "baseline_passed": true, + "candidate_passed": true, + "delta": 0.0, + "change_type": "unchanged", + "baseline_actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "candidate_actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "root_cause": "", + "reasons": [] + }, + { + "case_id": "val_refund_window_101", + "baseline_score": 0.0, + "candidate_score": 1.0, + "baseline_passed": false, + "candidate_passed": true, + "delta": 1.0, + "change_type": "new_pass", + "baseline_actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund window policy question can be answered by FAQ.\"}", + "candidate_actual_text": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Missing parts with refund request should open refund handling.\"}", + "root_cause": "", + "reasons": [] + }, + { + "case_id": "val_shipping_delay_103", + "baseline_score": 1.0, + "candidate_score": 1.0, + "baseline_passed": true, + "candidate_passed": true, + "delta": 0.0, + "change_type": "unchanged", + "baseline_actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "candidate_actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "root_cause": "", + "reasons": [] + } + ], + "gate": { + "candidate_id": "candidate_local_patch", + "accepted": true, + "reasons": [ + "validation improved and all configured gates passed" + ], + "new_hard_fail_ids": [], + "critical_regression_ids": [], + "missing_case_ids": [], + "unexpected_case_ids": [], + "validation_delta": 0.333333 + }, + "failure_attribution": { + "coverage": 1.0, + "taxonomy_counts": { + "final_response_mismatch": 0, + "tool_call_error": 0, + "parameter_error": 0, + "rubric_failed": 0, + "knowledge_gap": 0, + "format_error": 0, + "runtime_error": 0, + "metric_failed": 0 + }, + "cases": [] + }, + "artifacts": { + "prompt_dir": "runs/sample/prompts/candidate_local_patch", + "prompt_patch": "runs/sample/prompts/candidate_local_patch/prompt_patch.diff" + } + }, + { + "id": "candidate_noop", + "audit": { + "seed": 7, + "duration_seconds": 0.0, + "cost": { + "currency": "USD", + "estimated": 0.0, + "known": true + }, + "config_path": "examples/optimization/eval_optimize_loop/optimizer.json", + "config_sha256": "9df0a82039d6de55660169bf6061572da99b647f0141cdfcd5c6f0b4cd448d98" + }, + "prompt_patch_summary": "Rephrases baseline guidance without changing route decisions.", + "prompt_artifacts": [ + { + "name": "system_prompt", + "source_path": "examples/optimization/eval_optimize_loop/agent/prompts/system.md", + "candidate_path": "runs/sample/prompts/candidate_noop/system_prompt.md", + "sha256": "ac2208517d00b42bccfc2336108753e0e3c0ea238345c6b851896b09a4819ab3", + "source_written": false, + "summary": "Rephrases baseline guidance without changing route decisions.", + "diff": "unchanged", + "content": "You are a support-routing agent.\n\nReturn only one compact JSON object. Do not include markdown fences or extra text.\nThe JSON shape is:\n\n{\"route\":\"\",\"tool\":{\"name\":\"\",\"arguments\":{}},\"reason\":\"\"}\n\nKeep the reason short and do not promise refunds, credits, or manual handling\nunless the route rules require it.\n" + }, + { + "name": "router_prompt", + "source_path": "examples/optimization/eval_optimize_loop/agent/prompts/router.md", + "candidate_path": "runs/sample/prompts/candidate_noop/router_prompt.md", + "sha256": "8250ad478bed5bb80900204d05395f7110bcc5f46f62666c7d7a74c64ea0ccc1", + "source_written": false, + "summary": "Rephrases baseline guidance without changing route decisions.", + "diff": "--- source/router_prompt.md\n+++ candidate/router_prompt.md\n@@ -18,3 +18,5 @@\n 4. When the request is ambiguous, choose faq and explain that it can be handled as a policy or information question.\n \n Keep tool.arguments as an empty object for this example. Do not include Markdown, comments, or extra keys.\n+\n+Offline candidate patch (candidate_noop): Rephrases baseline guidance without changing route decisions.\n", + "content": "You route customer-support requests to one backend action.\n\nOutput exactly one JSON object with this shape:\n\n{\"route\":\"\",\"tool\":{\"name\":\"\",\"arguments\":{}},\"reason\":\"\"}\n\nAllowed routes and tools:\n\n- refund -> create_refund_ticket\n- manual_escalation -> create_escalation_case\n- faq -> none\n\nRouting policy:\n\n1. Use refund when the user asks for refund handling for a broken, missing, or unusable delivered item.\n2. Use manual_escalation when the user explicitly requests a human agent for an account-access, safety, legal, or threat-of-reporting issue.\n3. Use faq for policy, shipping-status, coupon, address-change, and informational questions that do not require opening a backend case.\n4. When the request is ambiguous, choose faq and explain that it can be handled as a policy or information question.\n\nKeep tool.arguments as an empty object for this example. Do not include Markdown, comments, or extra keys.\n\nOffline candidate patch (candidate_noop): Rephrases baseline guidance without changing route decisions.\n" + } + ], + "train": { + "eval_set_id": "support_router_train", + "score": 0.333333, + "pass_rate": 0.333333, + "metrics": { + "route_tool_args_score": { + "score": 0.333333, + "threshold": 1.0, + "passed": false, + "status": "failed" + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "passed": true, + "status": "passed" + } + }, + "case_results": [ + { + "case_id": "train_refund_001", + "tags": [ + "refund", + "train" + ], + "user": "The headphones arrived broken yesterday. I want a refund.", + "score": 0.0, + "passed": false, + "metrics": { + "route_tool_args_score": { + "score": 0.0, + "threshold": 1.0, + "status": "failed", + "passed": false, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund policy question can be answered by FAQ.\"}", + "expected_text": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Broken delivered item needs refund handling.\"}", + "key_trace": { + "invocation_id": "train-1", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund policy question can be answered by FAQ.\"}", + "expected_final_response": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Broken delivered item needs refund handling.\"}", + "error_message": null + }, + "root_cause": "final_response_mismatch", + "reasons": [ + "actual route 'faq' did not match expected route 'refund'" + ] + }, + { + "case_id": "train_manual_002", + "tags": [ + "manual", + "train" + ], + "user": "My account is locked and I am going to report this unless a person helps me now.", + "score": 0.0, + "passed": false, + "metrics": { + "route_tool_args_score": { + "score": 0.0, + "threshold": 1.0, + "status": "failed", + "passed": false, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Account question can be answered by FAQ.\"}", + "expected_text": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Account lock with escalation demand needs a human case.\"}", + "key_trace": { + "invocation_id": "train-2", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Account question can be answered by FAQ.\"}", + "expected_final_response": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Account lock with escalation demand needs a human case.\"}", + "error_message": null + }, + "root_cause": "final_response_mismatch", + "reasons": [ + "actual route 'faq' did not match expected route 'manual_escalation'" + ] + }, + { + "case_id": "train_faq_003", + "tags": [ + "faq", + "train" + ], + "user": "Can an expired coupon be reissued?", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Coupon policy question can be answered by FAQ.\"}", + "expected_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Coupon policy question can be answered by FAQ.\"}", + "key_trace": { + "invocation_id": "train-3", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Coupon policy question can be answered by FAQ.\"}", + "expected_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Coupon policy question can be answered by FAQ.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + } + ], + "failed_case_ids": [ + "train_refund_001", + "train_manual_002" + ], + "source": "AgentEvaluator" + }, + "optimizer_dev": { + "eval_set_id": "support_router_optimizer_dev", + "score": 0.333333, + "pass_rate": 0.333333, + "metrics": { + "route_tool_args_score": { + "score": 0.333333, + "threshold": 1.0, + "passed": false, + "status": "failed" + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "passed": true, + "status": "passed" + } + }, + "case_results": [ + { + "case_id": "dev_return_201", + "tags": [ + "refund", + "optimizer_dev" + ], + "user": "A coffee maker delivered this morning leaks water. Please return it for a refund.", + "score": 0.0, + "passed": false, + "metrics": { + "route_tool_args_score": { + "score": 0.0, + "threshold": 1.0, + "status": "failed", + "passed": false, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Return policy can be answered by FAQ.\"}", + "expected_text": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Leaking new appliance with an explicit return request needs refund handling.\"}", + "key_trace": { + "invocation_id": "optimizer-dev-1", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Return policy can be answered by FAQ.\"}", + "expected_final_response": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Leaking new appliance with an explicit return request needs refund handling.\"}", + "error_message": null + }, + "root_cause": "final_response_mismatch", + "reasons": [ + "actual route 'faq' did not match expected route 'refund'" + ] + }, + { + "case_id": "dev_access_202", + "tags": [ + "manual", + "optimizer_dev" + ], + "user": "Two-factor codes never arrive and I need a support specialist to restore access.", + "score": 0.0, + "passed": false, + "metrics": { + "route_tool_args_score": { + "score": 0.0, + "threshold": 1.0, + "status": "failed", + "passed": false, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Access policy can be answered by FAQ.\"}", + "expected_text": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Failed two-factor delivery with a specialist request needs a human case.\"}", + "key_trace": { + "invocation_id": "optimizer-dev-2", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Access policy can be answered by FAQ.\"}", + "expected_final_response": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Failed two-factor delivery with a specialist request needs a human case.\"}", + "error_message": null + }, + "root_cause": "final_response_mismatch", + "reasons": [ + "actual route 'faq' did not match expected route 'manual_escalation'" + ] + }, + { + "case_id": "dev_warranty_203", + "tags": [ + "faq", + "optimizer_dev" + ], + "user": "How long is the warranty on a replacement charger?", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Replacement charger warranty is a policy question for FAQ.\"}", + "expected_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Replacement charger warranty is a policy question for FAQ.\"}", + "key_trace": { + "invocation_id": "optimizer-dev-3", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Replacement charger warranty is a policy question for FAQ.\"}", + "expected_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Replacement charger warranty is a policy question for FAQ.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + } + ], + "failed_case_ids": [ + "dev_return_201", + "dev_access_202" + ], + "source": "AgentEvaluator" + }, + "final_validation": { + "eval_set_id": "support_router_val", + "score": 0.666667, + "pass_rate": 0.666667, + "metrics": { + "route_tool_args_score": { + "score": 0.666667, + "threshold": 1.0, + "passed": false, + "status": "failed" + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "passed": true, + "status": "passed" + } + }, + "case_results": [ + { + "case_id": "val_refund_window_101", + "tags": [ + "refund", + "validation" + ], + "user": "The blender arrived missing parts and I need a refund even though it has been 10 days.", + "score": 0.0, + "passed": false, + "metrics": { + "route_tool_args_score": { + "score": 0.0, + "threshold": 1.0, + "status": "failed", + "passed": false, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund window policy question can be answered by FAQ.\"}", + "expected_text": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Missing parts with refund request should open refund handling.\"}", + "key_trace": { + "invocation_id": "val-1", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund window policy question can be answered by FAQ.\"}", + "expected_final_response": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Missing parts with refund request should open refund handling.\"}", + "error_message": null + }, + "root_cause": "final_response_mismatch", + "reasons": [ + "actual route 'faq' did not match expected route 'refund'" + ] + }, + { + "case_id": "val_address_change_102", + "tags": [ + "faq", + "validation" + ], + "user": "Can I change the delivery address for an order that has not shipped?", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "expected_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "key_trace": { + "invocation_id": "val-2", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "expected_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + }, + { + "case_id": "val_shipping_delay_103", + "tags": [ + "faq", + "validation", + "critical" + ], + "user": "I am angry that shipping is late, but I only want to know the delivery policy.", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "expected_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "key_trace": { + "invocation_id": "val-3", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "expected_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + } + ], + "failed_case_ids": [ + "val_refund_window_101" + ], + "source": "AgentEvaluator" + }, + "validation": { + "eval_set_id": "support_router_val", + "score": 0.666667, + "pass_rate": 0.666667, + "metrics": { + "route_tool_args_score": { + "score": 0.666667, + "threshold": 1.0, + "passed": false, + "status": "failed" + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "passed": true, + "status": "passed" + } + }, + "case_results": [ + { + "case_id": "val_refund_window_101", + "tags": [ + "refund", + "validation" + ], + "user": "The blender arrived missing parts and I need a refund even though it has been 10 days.", + "score": 0.0, + "passed": false, + "metrics": { + "route_tool_args_score": { + "score": 0.0, + "threshold": 1.0, + "status": "failed", + "passed": false, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund window policy question can be answered by FAQ.\"}", + "expected_text": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Missing parts with refund request should open refund handling.\"}", + "key_trace": { + "invocation_id": "val-1", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund window policy question can be answered by FAQ.\"}", + "expected_final_response": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Missing parts with refund request should open refund handling.\"}", + "error_message": null + }, + "root_cause": "final_response_mismatch", + "reasons": [ + "actual route 'faq' did not match expected route 'refund'" + ] + }, + { + "case_id": "val_address_change_102", + "tags": [ + "faq", + "validation" + ], + "user": "Can I change the delivery address for an order that has not shipped?", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "expected_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "key_trace": { + "invocation_id": "val-2", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "expected_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + }, + { + "case_id": "val_shipping_delay_103", + "tags": [ + "faq", + "validation", + "critical" + ], + "user": "I am angry that shipping is late, but I only want to know the delivery policy.", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "expected_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "key_trace": { + "invocation_id": "val-3", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "expected_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + } + ], + "failed_case_ids": [ + "val_refund_window_101" + ], + "source": "AgentEvaluator" + }, + "delta": { + "train_score": 0.0, + "optimizer_dev_score": 0.0, + "validation_score": 0.0 + }, + "case_deltas": [ + { + "case_id": "val_address_change_102", + "baseline_score": 1.0, + "candidate_score": 1.0, + "baseline_passed": true, + "candidate_passed": true, + "delta": 0.0, + "change_type": "unchanged", + "baseline_actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "candidate_actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "root_cause": "", + "reasons": [] + }, + { + "case_id": "val_refund_window_101", + "baseline_score": 0.0, + "candidate_score": 0.0, + "baseline_passed": false, + "candidate_passed": false, + "delta": 0.0, + "change_type": "unchanged", + "baseline_actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund window policy question can be answered by FAQ.\"}", + "candidate_actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund window policy question can be answered by FAQ.\"}", + "root_cause": "final_response_mismatch", + "reasons": [ + "actual route 'faq' did not match expected route 'refund'" + ] + }, + { + "case_id": "val_shipping_delay_103", + "baseline_score": 1.0, + "candidate_score": 1.0, + "baseline_passed": true, + "candidate_passed": true, + "delta": 0.0, + "change_type": "unchanged", + "baseline_actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "candidate_actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "root_cause": "", + "reasons": [] + } + ], + "gate": { + "candidate_id": "candidate_noop", + "accepted": false, + "reasons": [ + "validation score did not improve over baseline", + "required metric(s) missing or failing: route_tool_args_score" + ], + "new_hard_fail_ids": [], + "critical_regression_ids": [], + "missing_case_ids": [], + "unexpected_case_ids": [], + "validation_delta": 0.0 + }, + "failure_attribution": { + "coverage": 1.0, + "taxonomy_counts": { + "final_response_mismatch": 1, + "tool_call_error": 0, + "parameter_error": 0, + "rubric_failed": 0, + "knowledge_gap": 0, + "format_error": 0, + "runtime_error": 0, + "metric_failed": 0 + }, + "cases": [ + { + "case_id": "val_refund_window_101", + "root_cause": "final_response_mismatch", + "score": 0.0, + "reasons": [ + "actual route 'faq' did not match expected route 'refund'" + ] + } + ] + }, + "artifacts": { + "prompt_dir": "runs/sample/prompts/candidate_noop", + "prompt_patch": "runs/sample/prompts/candidate_noop/prompt_patch.diff" + } + }, + { + "id": "candidate_overfit", + "audit": { + "seed": 7, + "duration_seconds": 0.0, + "cost": { + "currency": "USD", + "estimated": 0.0, + "known": true + }, + "config_path": "examples/optimization/eval_optimize_loop/optimizer.json", + "config_sha256": "9df0a82039d6de55660169bf6061572da99b647f0141cdfcd5c6f0b4cd448d98" + }, + "prompt_patch_summary": "Overfits to emotional wording and sends angry logistics questions to manual escalation.", + "prompt_artifacts": [ + { + "name": "system_prompt", + "source_path": "examples/optimization/eval_optimize_loop/agent/prompts/system.md", + "candidate_path": "runs/sample/prompts/candidate_overfit/system_prompt.md", + "sha256": "ac2208517d00b42bccfc2336108753e0e3c0ea238345c6b851896b09a4819ab3", + "source_written": false, + "summary": "Overfits to emotional wording and sends angry logistics questions to manual escalation.", + "diff": "unchanged", + "content": "You are a support-routing agent.\n\nReturn only one compact JSON object. Do not include markdown fences or extra text.\nThe JSON shape is:\n\n{\"route\":\"\",\"tool\":{\"name\":\"\",\"arguments\":{}},\"reason\":\"\"}\n\nKeep the reason short and do not promise refunds, credits, or manual handling\nunless the route rules require it.\n" + }, + { + "name": "router_prompt", + "source_path": "examples/optimization/eval_optimize_loop/agent/prompts/router.md", + "candidate_path": "runs/sample/prompts/candidate_overfit/router_prompt.md", + "sha256": "efccbd30ad31da466bba7f40a9fd3293cc851f2a3b9908e41e5e9a28a54076b1", + "source_written": false, + "summary": "Overfits to emotional wording and sends angry logistics questions to manual escalation.", + "diff": "--- source/router_prompt.md\n+++ candidate/router_prompt.md\n@@ -18,3 +18,5 @@\n 4. When the request is ambiguous, choose faq and explain that it can be handled as a policy or information question.\n \n Keep tool.arguments as an empty object for this example. Do not include Markdown, comments, or extra keys.\n+\n+Offline candidate patch (candidate_overfit): Overfits to emotional wording and sends angry logistics questions to manual escalation.\n", + "content": "You route customer-support requests to one backend action.\n\nOutput exactly one JSON object with this shape:\n\n{\"route\":\"\",\"tool\":{\"name\":\"\",\"arguments\":{}},\"reason\":\"\"}\n\nAllowed routes and tools:\n\n- refund -> create_refund_ticket\n- manual_escalation -> create_escalation_case\n- faq -> none\n\nRouting policy:\n\n1. Use refund when the user asks for refund handling for a broken, missing, or unusable delivered item.\n2. Use manual_escalation when the user explicitly requests a human agent for an account-access, safety, legal, or threat-of-reporting issue.\n3. Use faq for policy, shipping-status, coupon, address-change, and informational questions that do not require opening a backend case.\n4. When the request is ambiguous, choose faq and explain that it can be handled as a policy or information question.\n\nKeep tool.arguments as an empty object for this example. Do not include Markdown, comments, or extra keys.\n\nOffline candidate patch (candidate_overfit): Overfits to emotional wording and sends angry logistics questions to manual escalation.\n" + } + ], + "train": { + "eval_set_id": "support_router_train", + "score": 1.0, + "pass_rate": 1.0, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "passed": true, + "status": "passed" + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "passed": true, + "status": "passed" + } + }, + "case_results": [ + { + "case_id": "train_refund_001", + "tags": [ + "refund", + "train" + ], + "user": "The headphones arrived broken yesterday. I want a refund.", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Broken delivered item needs refund handling.\"}", + "expected_text": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Broken delivered item needs refund handling.\"}", + "key_trace": { + "invocation_id": "train-1", + "actual_final_response": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Broken delivered item needs refund handling.\"}", + "expected_final_response": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Broken delivered item needs refund handling.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + }, + { + "case_id": "train_manual_002", + "tags": [ + "manual", + "train" + ], + "user": "My account is locked and I am going to report this unless a person helps me now.", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Account lock with escalation demand needs a human case.\"}", + "expected_text": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Account lock with escalation demand needs a human case.\"}", + "key_trace": { + "invocation_id": "train-2", + "actual_final_response": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Account lock with escalation demand needs a human case.\"}", + "expected_final_response": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Account lock with escalation demand needs a human case.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + }, + { + "case_id": "train_faq_003", + "tags": [ + "faq", + "train" + ], + "user": "Can an expired coupon be reissued?", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Coupon policy question can be answered by FAQ.\"}", + "expected_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Coupon policy question can be answered by FAQ.\"}", + "key_trace": { + "invocation_id": "train-3", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Coupon policy question can be answered by FAQ.\"}", + "expected_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Coupon policy question can be answered by FAQ.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + } + ], + "failed_case_ids": [], + "source": "AgentEvaluator" + }, + "optimizer_dev": { + "eval_set_id": "support_router_optimizer_dev", + "score": 1.0, + "pass_rate": 1.0, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "passed": true, + "status": "passed" + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "passed": true, + "status": "passed" + } + }, + "case_results": [ + { + "case_id": "dev_return_201", + "tags": [ + "refund", + "optimizer_dev" + ], + "user": "A coffee maker delivered this morning leaks water. Please return it for a refund.", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Leaking new appliance with an explicit return request needs refund handling.\"}", + "expected_text": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Leaking new appliance with an explicit return request needs refund handling.\"}", + "key_trace": { + "invocation_id": "optimizer-dev-1", + "actual_final_response": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Leaking new appliance with an explicit return request needs refund handling.\"}", + "expected_final_response": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Leaking new appliance with an explicit return request needs refund handling.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + }, + { + "case_id": "dev_access_202", + "tags": [ + "manual", + "optimizer_dev" + ], + "user": "Two-factor codes never arrive and I need a support specialist to restore access.", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Failed two-factor delivery with a specialist request needs a human case.\"}", + "expected_text": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Failed two-factor delivery with a specialist request needs a human case.\"}", + "key_trace": { + "invocation_id": "optimizer-dev-2", + "actual_final_response": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Failed two-factor delivery with a specialist request needs a human case.\"}", + "expected_final_response": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Failed two-factor delivery with a specialist request needs a human case.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + }, + { + "case_id": "dev_warranty_203", + "tags": [ + "faq", + "optimizer_dev" + ], + "user": "How long is the warranty on a replacement charger?", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Replacement charger warranty is a policy question for FAQ.\"}", + "expected_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Replacement charger warranty is a policy question for FAQ.\"}", + "key_trace": { + "invocation_id": "optimizer-dev-3", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Replacement charger warranty is a policy question for FAQ.\"}", + "expected_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Replacement charger warranty is a policy question for FAQ.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + } + ], + "failed_case_ids": [], + "source": "AgentEvaluator" + }, + "final_validation": { + "eval_set_id": "support_router_val", + "score": 0.333333, + "pass_rate": 0.333333, + "metrics": { + "route_tool_args_score": { + "score": 0.333333, + "threshold": 1.0, + "passed": false, + "status": "failed" + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "passed": true, + "status": "passed" + } + }, + "case_results": [ + { + "case_id": "val_refund_window_101", + "tags": [ + "refund", + "validation" + ], + "user": "The blender arrived missing parts and I need a refund even though it has been 10 days.", + "score": 0.0, + "passed": false, + "metrics": { + "route_tool_args_score": { + "score": 0.0, + "threshold": 1.0, + "status": "failed", + "passed": false, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund window policy question can be answered by FAQ.\"}", + "expected_text": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Missing parts with refund request should open refund handling.\"}", + "key_trace": { + "invocation_id": "val-1", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund window policy question can be answered by FAQ.\"}", + "expected_final_response": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Missing parts with refund request should open refund handling.\"}", + "error_message": null + }, + "root_cause": "final_response_mismatch", + "reasons": [ + "actual route 'faq' did not match expected route 'refund'" + ] + }, + { + "case_id": "val_address_change_102", + "tags": [ + "faq", + "validation" + ], + "user": "Can I change the delivery address for an order that has not shipped?", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "expected_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "key_trace": { + "invocation_id": "val-2", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "expected_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + }, + { + "case_id": "val_shipping_delay_103", + "tags": [ + "faq", + "validation", + "critical" + ], + "user": "I am angry that shipping is late, but I only want to know the delivery policy.", + "score": 0.0, + "passed": false, + "metrics": { + "route_tool_args_score": { + "score": 0.0, + "threshold": 1.0, + "status": "failed", + "passed": false, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Angry language should be escalated to a person.\"}", + "expected_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "key_trace": { + "invocation_id": "val-3", + "actual_final_response": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Angry language should be escalated to a person.\"}", + "expected_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "error_message": null + }, + "root_cause": "final_response_mismatch", + "reasons": [ + "actual route 'manual_escalation' did not match expected route 'faq'" + ] + } + ], + "failed_case_ids": [ + "val_refund_window_101", + "val_shipping_delay_103" + ], + "source": "AgentEvaluator" + }, + "validation": { + "eval_set_id": "support_router_val", + "score": 0.333333, + "pass_rate": 0.333333, + "metrics": { + "route_tool_args_score": { + "score": 0.333333, + "threshold": 1.0, + "passed": false, + "status": "failed" + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "passed": true, + "status": "passed" + } + }, + "case_results": [ + { + "case_id": "val_refund_window_101", + "tags": [ + "refund", + "validation" + ], + "user": "The blender arrived missing parts and I need a refund even though it has been 10 days.", + "score": 0.0, + "passed": false, + "metrics": { + "route_tool_args_score": { + "score": 0.0, + "threshold": 1.0, + "status": "failed", + "passed": false, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund window policy question can be answered by FAQ.\"}", + "expected_text": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Missing parts with refund request should open refund handling.\"}", + "key_trace": { + "invocation_id": "val-1", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund window policy question can be answered by FAQ.\"}", + "expected_final_response": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Missing parts with refund request should open refund handling.\"}", + "error_message": null + }, + "root_cause": "final_response_mismatch", + "reasons": [ + "actual route 'faq' did not match expected route 'refund'" + ] + }, + { + "case_id": "val_address_change_102", + "tags": [ + "faq", + "validation" + ], + "user": "Can I change the delivery address for an order that has not shipped?", + "score": 1.0, + "passed": true, + "metrics": { + "route_tool_args_score": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "expected_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "key_trace": { + "invocation_id": "val-2", + "actual_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "expected_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "error_message": null + }, + "root_cause": "", + "reasons": [] + }, + { + "case_id": "val_shipping_delay_103", + "tags": [ + "faq", + "validation", + "critical" + ], + "user": "I am angry that shipping is late, but I only want to know the delivery policy.", + "score": 0.0, + "passed": false, + "metrics": { + "route_tool_args_score": { + "score": 0.0, + "threshold": 1.0, + "status": "failed", + "passed": false, + "reason": null + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 1.0, + "status": "passed", + "passed": true, + "reason": "offline rubric passed" + } + }, + "actual_text": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Angry language should be escalated to a person.\"}", + "expected_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "key_trace": { + "invocation_id": "val-3", + "actual_final_response": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Angry language should be escalated to a person.\"}", + "expected_final_response": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "error_message": null + }, + "root_cause": "final_response_mismatch", + "reasons": [ + "actual route 'manual_escalation' did not match expected route 'faq'" + ] + } + ], + "failed_case_ids": [ + "val_refund_window_101", + "val_shipping_delay_103" + ], + "source": "AgentEvaluator" + }, + "delta": { + "train_score": 0.666667, + "optimizer_dev_score": 0.666667, + "validation_score": -0.333334 + }, + "case_deltas": [ + { + "case_id": "val_address_change_102", + "baseline_score": 1.0, + "candidate_score": 1.0, + "baseline_passed": true, + "candidate_passed": true, + "delta": 0.0, + "change_type": "unchanged", + "baseline_actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "candidate_actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "root_cause": "", + "reasons": [] + }, + { + "case_id": "val_refund_window_101", + "baseline_score": 0.0, + "candidate_score": 0.0, + "baseline_passed": false, + "candidate_passed": false, + "delta": 0.0, + "change_type": "unchanged", + "baseline_actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund window policy question can be answered by FAQ.\"}", + "candidate_actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund window policy question can be answered by FAQ.\"}", + "root_cause": "final_response_mismatch", + "reasons": [ + "actual route 'faq' did not match expected route 'refund'" + ] + }, + { + "case_id": "val_shipping_delay_103", + "baseline_score": 1.0, + "candidate_score": 0.0, + "baseline_passed": true, + "candidate_passed": false, + "delta": -1.0, + "change_type": "new_fail", + "baseline_actual_text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}", + "candidate_actual_text": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Angry language should be escalated to a person.\"}", + "root_cause": "final_response_mismatch", + "reasons": [ + "actual route 'manual_escalation' did not match expected route 'faq'" + ] + } + ], + "gate": { + "candidate_id": "candidate_overfit", + "accepted": false, + "reasons": [ + "validation score did not improve over baseline", + "candidate introduced hard fail(s): val_shipping_delay_103", + "candidate regressed critical case(s): val_shipping_delay_103", + "required metric(s) missing or failing: route_tool_args_score" + ], + "new_hard_fail_ids": [ + "val_shipping_delay_103" + ], + "critical_regression_ids": [ + "val_shipping_delay_103" + ], + "missing_case_ids": [], + "unexpected_case_ids": [], + "validation_delta": -0.333334 + }, + "failure_attribution": { + "coverage": 1.0, + "taxonomy_counts": { + "final_response_mismatch": 2, + "tool_call_error": 0, + "parameter_error": 0, + "rubric_failed": 0, + "knowledge_gap": 0, + "format_error": 0, + "runtime_error": 0, + "metric_failed": 0 + }, + "cases": [ + { + "case_id": "val_refund_window_101", + "root_cause": "final_response_mismatch", + "score": 0.0, + "reasons": [ + "actual route 'faq' did not match expected route 'refund'" + ] + }, + { + "case_id": "val_shipping_delay_103", + "root_cause": "final_response_mismatch", + "score": 0.0, + "reasons": [ + "actual route 'manual_escalation' did not match expected route 'faq'" + ] + } + ] + }, + "artifacts": { + "prompt_dir": "runs/sample/prompts/candidate_overfit", + "prompt_patch": "runs/sample/prompts/candidate_overfit/prompt_patch.diff" + } + } + ], + "delta": { + "train_score": 0.666667, + "optimizer_dev_score": 0.666667, + "validation_score": 0.333333 + }, + "gate_decision": { + "accepted": true, + "winner": "candidate_local_patch", + "reasons": [ + "validation improved and all configured gates passed" + ] + }, + "failure_attribution": { + "coverage": 1.0, + "taxonomy_counts": { + "final_response_mismatch": 1, + "tool_call_error": 0, + "parameter_error": 0, + "rubric_failed": 0, + "knowledge_gap": 0, + "format_error": 0, + "runtime_error": 0, + "metric_failed": 0 + }, + "cases": [ + { + "case_id": "val_refund_window_101", + "root_cause": "final_response_mismatch", + "score": 0.0, + "reasons": [ + "actual route 'faq' did not match expected route 'refund'" + ] + } + ] + }, + "cost": { + "currency": "USD", + "estimated_total": 0.0, + "cost_source": "deterministic_offline", + "unknown_cost_reason": null, + "model_calls": 0, + "token_usage": { + "prompt": 0, + "completion": 0, + "total": 0 + }, + "token_usage_known": true, + "unknown_token_usage_reason": null, + "optimizer": { + "estimated_cost": 0.0, + "model_calls": 0, + "candidate_evaluation_agent_calls": 0, + "reflection_lm_calls": 0, + "judge_calls_per_candidate_evaluation": 0, + "judge_model_calls": 0, + "native_judge_model_calls": 0, + "derived_judge_model_calls": 0, + "judge_model_call_source": "none", + "token_usage": { + "prompt": 0, + "completion": 0, + "total": 0 + }, + "token_usage_known": true, + "unknown_token_usage_reason": null, + "usage_evidence_valid": true, + "reflection_reported_usage": { + "estimated_cost": 0.0, + "token_usage": { + "prompt": 0, + "completion": 0, + "total": 0 + }, + "token_usage_known": true, + "unknown_token_usage_reason": null + } + }, + "final_revalidation": { + "estimated_cost": 0.0, + "agent_calls_per_run": 0, + "agent_calls": 0, + "judge_calls_per_agent_call": 0, + "judge_model_calls": 0, + "model_calls": 0, + "token_usage": { + "prompt": 0, + "completion": 0, + "total": 0 + }, + "token_usage_known": true, + "unknown_token_usage_reason": null + } + }, + "duration_seconds": 0.0, + "optimization_rounds": [], + "config_snapshot": { + "mode": "fake", + "seed": 7, + "gate": { + "min_validation_delta": 0.0, + "allow_new_hard_fails": false, + "allow_critical_regression": false, + "max_cost_usd": null, + "max_duration_seconds": 180.0, + "required_metrics": [ + "route_tool_args_score", + "llm_rubric_response" + ], + "required_metrics_source": "optimizer_config" + }, + "evaluation": { + "metrics": [ + { + "metric_name": "route_tool_args_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "json": { + "match": "exact" + } + } + } + }, + { + "metric_name": "llm_rubric_response", + "threshold": 1.0, + "criterion": { + "offline_rubric": { + "checks": [ + "valid_json_object", + "route_present", + "tool_object_present" + ] + } + } + } + ], + "num_runs": 1 + }, + "evaluation_sha256": "a2a1ec7e174d243b397ee7875d7528cb198d1c1d3fbce9dc82f8cb85e33fe166", + "evaluation_metrics_sha256": "a2a1ec7e174d243b397ee7875d7528cb198d1c1d3fbce9dc82f8cb85e33fe166", + "optimizer_config_sha256": "9df0a82039d6de55660169bf6061572da99b647f0141cdfcd5c6f0b4cd448d98", + "prompt_targets": { + "system_prompt": { + "source_path": "examples/optimization/eval_optimize_loop/agent/prompts/system.md", + "sha256": "ac2208517d00b42bccfc2336108753e0e3c0ea238345c6b851896b09a4819ab3" + }, + "router_prompt": { + "source_path": "examples/optimization/eval_optimize_loop/agent/prompts/router.md", + "sha256": "1897b273cf69ace89b50e5c9ba77e135935544bd67100ecd11c78ee7fa317f10" + } + }, + "evalsets": { + "train": { + "path": "examples/optimization/eval_optimize_loop/train.evalset.json", + "sha256": "a4f49908563bbea0ee1347428ee23f90c21c1fcebeda32d2905d44d8b14cf28f", + "case_count": 3, + "turn_count": 3 + }, + "optimizer_dev": { + "path": "examples/optimization/eval_optimize_loop/optimizer_dev.evalset.json", + "sha256": "7ee6e4824d41b4a2ce6e2ba75352dd1b3ccfecdd52a96620f458f95f810ce4fa", + "case_count": 3, + "turn_count": 3 + }, + "final_validation": { + "path": "examples/optimization/eval_optimize_loop/val.evalset.json", + "sha256": "d8c2cc40d15f4dc8f1d06982553a473dbca3fe20d3e38a9bce56fafcea119238", + "case_count": 3, + "turn_count": 3 + } + }, + "paths": { + "train_evalset": "examples/optimization/eval_optimize_loop/train.evalset.json", + "optimizer_dev_evalset": "examples/optimization/eval_optimize_loop/optimizer_dev.evalset.json", + "validation_evalset": "examples/optimization/eval_optimize_loop/val.evalset.json", + "final_validation_evalset": "examples/optimization/eval_optimize_loop/val.evalset.json", + "optimizer_config": "examples/optimization/eval_optimize_loop/optimizer.json", + "evaluation_metrics": "examples/optimization/eval_optimize_loop/fixtures/offline_metrics.sample.json", + "fixture_outputs": "examples/optimization/eval_optimize_loop/fixtures/fake_outputs.json", + "system_prompt": "examples/optimization/eval_optimize_loop/agent/prompts/system.md", + "router_prompt": "examples/optimization/eval_optimize_loop/agent/prompts/router.md" + } + }, + "environment_snapshot": { + "git_commit": "sample", + "git_dirty": false, + "python_version": "3.x", + "sdk_version": "sample", + "model_name": null, + "base_url_host": null, + "seed": 7, + "command": "python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake --output-dir runs --run-id sample", + "config_path": "examples/optimization/eval_optimize_loop/optimizer.json" + }, + "artifacts": { + "optimization_report_json": "runs/sample/optimization_report.json", + "optimization_report_md": "runs/sample/optimization_report.md", + "train_evalset": "examples/optimization/eval_optimize_loop/train.evalset.json", + "optimizer_dev_evalset": "examples/optimization/eval_optimize_loop/optimizer_dev.evalset.json", + "validation_evalset": "examples/optimization/eval_optimize_loop/val.evalset.json", + "final_validation_evalset": "examples/optimization/eval_optimize_loop/val.evalset.json", + "optimizer_config": "examples/optimization/eval_optimize_loop/optimizer.json", + "fixtures": "examples/optimization/eval_optimize_loop/fixtures/fake_outputs.json", + "eval_metrics": "examples/optimization/eval_optimize_loop/fixtures/offline_metrics.sample.json", + "system_prompt": "examples/optimization/eval_optimize_loop/agent/prompts/system.md", + "router_prompt": "examples/optimization/eval_optimize_loop/agent/prompts/router.md", + "baseline_train_trace_evalset": "", + "baseline_optimizer_dev_trace_evalset": "", + "baseline_validation_trace_evalset": "", + "baseline_prompt_dir": "runs/sample/prompts/baseline", + "baseline_prompt_patch": "runs/sample/prompts/baseline/prompt_patch.diff" + } +} diff --git a/examples/optimization/eval_optimize_loop/fixtures/trace_outputs.json b/examples/optimization/eval_optimize_loop/fixtures/trace_outputs.json new file mode 100644 index 00000000..bf31482b --- /dev/null +++ b/examples/optimization/eval_optimize_loop/fixtures/trace_outputs.json @@ -0,0 +1,58 @@ +{ + "baseline": { + "prompt_patch_summary": "Baseline overuses FAQ and misses refund and account-escalation intent.", + "outputs": { + "train_refund_001": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund policy question can be answered by FAQ.\"}", + "train_manual_002": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Account question can be answered by FAQ.\"}", + "train_faq_003": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Coupon policy question can be answered by FAQ.\"}", + "dev_return_201": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Return policy can be answered by FAQ.\"}", + "dev_access_202": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Access policy can be answered by FAQ.\"}", + "dev_warranty_203": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Replacement charger warranty is a policy question for FAQ.\"}", + "val_refund_window_101": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund window policy question can be answered by FAQ.\"}", + "val_address_change_102": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "val_shipping_delay_103": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}" + } + }, + "candidate_local_patch": { + "prompt_patch_summary": "Adds local routing rules for explicit refund requests and account-escalation demands while preserving FAQ for ordinary policy questions.", + "outputs": { + "train_refund_001": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Broken delivered item needs refund handling.\"}", + "train_manual_002": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Account lock with escalation demand needs a human case.\"}", + "train_faq_003": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Coupon policy question can be answered by FAQ.\"}", + "dev_return_201": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Leaking new appliance with an explicit return request needs refund handling.\"}", + "dev_access_202": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Failed two-factor delivery with a specialist request needs a human case.\"}", + "dev_warranty_203": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Replacement charger warranty is a policy question for FAQ.\"}", + "val_refund_window_101": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Missing parts with refund request should open refund handling.\"}", + "val_address_change_102": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "val_shipping_delay_103": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}" + } + }, + "candidate_noop": { + "prompt_patch_summary": "Rephrases baseline guidance without changing route decisions.", + "outputs": { + "train_refund_001": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund policy question can be answered by FAQ.\"}", + "train_manual_002": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Account question can be answered by FAQ.\"}", + "train_faq_003": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Coupon policy question can be answered by FAQ.\"}", + "dev_return_201": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Return policy can be answered by FAQ.\"}", + "dev_access_202": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Access policy can be answered by FAQ.\"}", + "dev_warranty_203": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Replacement charger warranty is a policy question for FAQ.\"}", + "val_refund_window_101": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund window policy question can be answered by FAQ.\"}", + "val_address_change_102": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "val_shipping_delay_103": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}" + } + }, + "candidate_overfit": { + "prompt_patch_summary": "Overfits to emotional wording and sends angry logistics questions to manual escalation.", + "outputs": { + "train_refund_001": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Broken delivered item needs refund handling.\"}", + "train_manual_002": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Account lock with escalation demand needs a human case.\"}", + "train_faq_003": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Coupon policy question can be answered by FAQ.\"}", + "dev_return_201": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Leaking new appliance with an explicit return request needs refund handling.\"}", + "dev_access_202": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Failed two-factor delivery with a specialist request needs a human case.\"}", + "dev_warranty_203": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Replacement charger warranty is a policy question for FAQ.\"}", + "val_refund_window_101": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Refund window policy question can be answered by FAQ.\"}", + "val_address_change_102": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}", + "val_shipping_delay_103": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Angry language should be escalated to a person.\"}" + } + } +} diff --git a/examples/optimization/eval_optimize_loop/optimization_report.schema.json b/examples/optimization/eval_optimize_loop/optimization_report.schema.json new file mode 100644 index 00000000..01f900c5 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/optimization_report.schema.json @@ -0,0 +1,875 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://trpc-agent-python.local/examples/optimization/eval_optimize_loop/optimization_report.schema.json", + "title": "Eval Optimize Loop Report", + "type": "object", + "additionalProperties": false, + "required": [ + "run_id", + "mode", + "seed", + "baseline", + "candidates", + "delta", + "gate_decision", + "failure_attribution", + "cost", + "duration_seconds", + "optimization_rounds", + "config_snapshot", + "environment_snapshot", + "artifacts" + ], + "properties": { + "run_id": {"$ref": "#/$defs/safePathComponent"}, + "mode": {"enum": ["fake", "trace", "online"]}, + "seed": {"type": "integer"}, + "baseline": {"$ref": "#/$defs/baseline"}, + "candidates": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/candidate"} + }, + "delta": {"$ref": "#/$defs/delta"}, + "gate_decision": {"$ref": "#/$defs/gateDecision"}, + "failure_attribution": {"$ref": "#/$defs/failureAttribution"}, + "cost": {"$ref": "#/$defs/cost"}, + "duration_seconds": {"type": "number", "minimum": 0}, + "optimization_rounds": { + "type": "array", + "items": {"$ref": "#/$defs/optimizationRound"} + }, + "config_snapshot": {"$ref": "#/$defs/configSnapshot"}, + "environment_snapshot": {"$ref": "#/$defs/environmentSnapshot"}, + "artifacts": {"type": "object", "additionalProperties": {"type": "string"}}, + "online_result": {"$ref": "#/$defs/onlineResult"}, + "online_preflight": {"$ref": "#/$defs/onlinePreflight"}, + "online_duration": {"$ref": "#/$defs/onlineDuration"} + }, + "$defs": { + "safePathComponent": { + "type": "string", + "pattern": "^(?!(?:[Cc][Oo][Nn]|[Pp][Rr][Nn]|[Aa][Uu][Xx]|[Nn][Uu][Ll]|[Cc][Oo][Mm][1-9]|[Ll][Pp][Tt][1-9])(?:\\.|$))[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9_-])?$" + }, + "metricSummary": { + "type": "object", + "additionalProperties": false, + "required": ["score", "threshold", "passed", "status"], + "properties": { + "score": {"type": ["number", "null"], "minimum": 0, "maximum": 1}, + "threshold": {"type": "number", "minimum": 0}, + "passed": {"type": "boolean"}, + "status": {"type": "string", "minLength": 1}, + "reason": {"type": ["string", "null"]} + } + }, + "keyTrace": { + "type": "object", + "additionalProperties": false, + "required": [ + "invocation_id", + "actual_final_response", + "expected_final_response", + "error_message" + ], + "properties": { + "invocation_id": {"type": "string"}, + "actual_final_response": {"type": "string"}, + "expected_final_response": {"type": "string"}, + "error_message": {"type": ["string", "null"]} + } + }, + "evaluationCase": { + "type": "object", + "additionalProperties": false, + "required": [ + "case_id", + "tags", + "user", + "score", + "passed", + "metrics", + "actual_text", + "expected_text", + "key_trace", + "root_cause", + "reasons" + ], + "properties": { + "case_id": {"type": "string", "minLength": 1}, + "tags": {"type": "array", "items": {"type": "string"}}, + "user": {"type": "string"}, + "score": {"type": "number", "minimum": 0, "maximum": 1}, + "passed": {"type": "boolean"}, + "metrics": { + "type": "object", + "additionalProperties": {"$ref": "#/$defs/metricSummary"} + }, + "actual_text": {"type": "string"}, + "expected_text": {"type": "string"}, + "key_trace": {"$ref": "#/$defs/keyTrace"}, + "root_cause": { + "enum": [ + "", + "final_response_mismatch", + "tool_call_error", + "parameter_error", + "rubric_failed", + "knowledge_gap", + "format_error", + "runtime_error", + "metric_failed" + ] + }, + "reasons": {"type": "array", "items": {"type": "string", "minLength": 1}} + } + }, + "evaluationSummary": { + "type": "object", + "additionalProperties": false, + "required": [ + "eval_set_id", + "score", + "pass_rate", + "metrics", + "case_results", + "failed_case_ids", + "source" + ], + "properties": { + "eval_set_id": {"type": "string", "minLength": 1}, + "score": {"type": "number", "minimum": 0, "maximum": 1}, + "pass_rate": {"type": "number", "minimum": 0, "maximum": 1}, + "metrics": { + "type": "object", + "additionalProperties": {"$ref": "#/$defs/metricSummary"} + }, + "case_results": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/evaluationCase"} + }, + "failed_case_ids": {"type": "array", "items": {"type": "string"}}, + "source": {"type": "string", "minLength": 1} + } + }, + "promptArtifact": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "source_path", + "candidate_path", + "sha256", + "content", + "source_written", + "summary", + "diff" + ], + "properties": { + "name": {"type": "string", "minLength": 1}, + "source_path": {"type": "string", "minLength": 1}, + "candidate_path": {"type": "string", "minLength": 1}, + "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, + "content": {"type": "string"}, + "source_written": {"type": "boolean"}, + "summary": {"type": "string"}, + "diff": {"type": "string"} + } + }, + "promptTarget": { + "type": "object", + "additionalProperties": false, + "required": ["source_path", "sha256"], + "properties": { + "source_path": {"type": "string", "minLength": 1}, + "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"} + } + }, + "baseline": { + "type": "object", + "additionalProperties": false, + "required": [ + "prompt_patch_summary", + "prompt_artifacts", + "train", + "optimizer_dev", + "final_validation", + "validation" + ], + "properties": { + "prompt_patch_summary": {"type": "string"}, + "prompt_artifacts": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/promptArtifact"} + }, + "train": {"$ref": "#/$defs/evaluationSummary"}, + "optimizer_dev": {"$ref": "#/$defs/evaluationSummary"}, + "final_validation": {"$ref": "#/$defs/evaluationSummary"}, + "validation": {"$ref": "#/$defs/evaluationSummary"} + } + }, + "candidateAudit": { + "type": "object", + "additionalProperties": false, + "required": ["seed", "duration_seconds", "cost", "config_path", "config_sha256"], + "properties": { + "seed": {"type": "integer"}, + "duration_seconds": {"type": "number", "minimum": 0}, + "cost": { + "type": "object", + "additionalProperties": false, + "required": ["currency", "estimated", "known"], + "properties": { + "currency": {"const": "USD"}, + "estimated": {"type": ["number", "null"], "minimum": 0}, + "known": {"type": "boolean"} + }, + "allOf": [ + { + "if": {"properties": {"known": {"const": true}}}, + "then": {"properties": {"estimated": {"type": "number", "minimum": 0}}} + }, + { + "if": {"properties": {"known": {"const": false}}}, + "then": {"properties": {"estimated": {"type": "null"}}} + } + ] + }, + "config_path": {"type": "string", "minLength": 1}, + "config_sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"} + } + }, + "caseDelta": { + "type": "object", + "additionalProperties": false, + "required": [ + "case_id", + "baseline_score", + "candidate_score", + "baseline_passed", + "candidate_passed", + "delta", + "change_type", + "baseline_actual_text", + "candidate_actual_text", + "root_cause", + "reasons" + ], + "properties": { + "case_id": {"type": "string", "minLength": 1}, + "baseline_score": {"type": ["number", "null"], "minimum": 0, "maximum": 1}, + "candidate_score": {"type": ["number", "null"], "minimum": 0, "maximum": 1}, + "baseline_passed": {"type": ["boolean", "null"]}, + "candidate_passed": {"type": ["boolean", "null"]}, + "delta": {"type": ["number", "null"], "minimum": -1, "maximum": 1}, + "change_type": { + "enum": [ + "new_pass", + "new_fail", + "score_improved", + "score_regressed", + "unchanged", + "missing_candidate", + "unexpected_candidate" + ] + }, + "baseline_actual_text": {"type": "string"}, + "candidate_actual_text": {"type": "string"}, + "root_cause": {"type": "string"}, + "reasons": {"type": "array", "items": {"type": "string"}} + } + }, + "delta": { + "type": "object", + "additionalProperties": false, + "required": ["train_score", "optimizer_dev_score", "validation_score"], + "properties": { + "train_score": {"type": "number", "minimum": -1, "maximum": 1}, + "optimizer_dev_score": {"type": "number", "minimum": -1, "maximum": 1}, + "validation_score": {"type": "number", "minimum": -1, "maximum": 1} + } + }, + "candidateGate": { + "type": "object", + "additionalProperties": false, + "required": [ + "candidate_id", + "accepted", + "reasons", + "new_hard_fail_ids", + "critical_regression_ids", + "missing_case_ids", + "unexpected_case_ids", + "validation_delta" + ], + "properties": { + "candidate_id": {"$ref": "#/$defs/safePathComponent"}, + "accepted": {"type": "boolean"}, + "reasons": {"type": "array", "items": {"type": "string", "minLength": 1}}, + "new_hard_fail_ids": {"type": "array", "items": {"type": "string"}}, + "critical_regression_ids": {"type": "array", "items": {"type": "string"}}, + "missing_case_ids": {"type": "array", "items": {"type": "string"}}, + "unexpected_case_ids": {"type": "array", "items": {"type": "string"}}, + "validation_delta": {"type": ["number", "null"], "minimum": -1, "maximum": 1} + }, + "allOf": [ + { + "if": {"properties": {"accepted": {"const": true}}}, + "then": { + "properties": { + "validation_delta": {"type": "number", "exclusiveMinimum": 0, "maximum": 1} + } + } + } + ] + }, + "gateDecision": { + "type": "object", + "additionalProperties": false, + "required": ["accepted", "winner", "reasons"], + "properties": { + "accepted": {"type": "boolean"}, + "winner": { + "oneOf": [ + {"$ref": "#/$defs/safePathComponent"}, + {"type": "null"} + ] + }, + "reasons": {"type": "array", "items": {"type": "string", "minLength": 1}} + }, + "allOf": [ + { + "if": {"properties": {"accepted": {"const": true}}}, + "then": {"properties": {"winner": {"$ref": "#/$defs/safePathComponent"}}} + }, + { + "if": {"properties": {"accepted": {"const": false}}}, + "then": {"properties": {"winner": {"type": "null"}}} + } + ] + }, + "attributionCase": { + "type": "object", + "additionalProperties": false, + "required": ["case_id", "root_cause", "score", "reasons"], + "properties": { + "case_id": {"type": "string", "minLength": 1}, + "root_cause": { + "enum": [ + "final_response_mismatch", + "tool_call_error", + "parameter_error", + "rubric_failed", + "knowledge_gap", + "format_error", + "runtime_error", + "metric_failed" + ] + }, + "score": {"type": ["number", "null"], "minimum": 0, "maximum": 1}, + "reasons": {"type": "array", "minItems": 1, "items": {"type": "string", "minLength": 1}} + } + }, + "failureAttribution": { + "type": "object", + "additionalProperties": false, + "required": ["coverage", "taxonomy_counts", "cases"], + "properties": { + "coverage": {"type": "number", "minimum": 0, "maximum": 1}, + "taxonomy_counts": { + "type": "object", + "additionalProperties": false, + "required": [ + "final_response_mismatch", + "tool_call_error", + "parameter_error", + "rubric_failed", + "knowledge_gap", + "format_error", + "runtime_error", + "metric_failed" + ], + "properties": { + "final_response_mismatch": {"type": "integer", "minimum": 0}, + "tool_call_error": {"type": "integer", "minimum": 0}, + "parameter_error": {"type": "integer", "minimum": 0}, + "rubric_failed": {"type": "integer", "minimum": 0}, + "knowledge_gap": {"type": "integer", "minimum": 0}, + "format_error": {"type": "integer", "minimum": 0}, + "runtime_error": {"type": "integer", "minimum": 0}, + "metric_failed": {"type": "integer", "minimum": 0} + } + }, + "cases": {"type": "array", "items": {"$ref": "#/$defs/attributionCase"}} + } + }, + "tokenUsage": { + "type": "object", + "additionalProperties": false, + "required": ["prompt", "completion", "total"], + "properties": { + "prompt": {"type": "integer", "minimum": 0}, + "completion": {"type": "integer", "minimum": 0}, + "total": {"type": "integer", "minimum": 0} + } + }, + "reportedUsage": { + "type": "object", + "additionalProperties": false, + "required": [ + "estimated_cost", + "token_usage", + "token_usage_known", + "unknown_token_usage_reason" + ], + "properties": { + "estimated_cost": {"type": ["number", "null"], "minimum": 0}, + "token_usage": {"$ref": "#/$defs/tokenUsage"}, + "token_usage_known": {"type": "boolean"}, + "unknown_token_usage_reason": {"type": ["string", "null"]} + }, + "allOf": [ + { + "if": {"properties": {"token_usage_known": {"const": true}}}, + "then": {"properties": {"unknown_token_usage_reason": {"type": "null"}}} + }, + { + "if": {"properties": {"token_usage_known": {"const": false}}}, + "then": { + "properties": { + "unknown_token_usage_reason": {"type": "string", "minLength": 1} + } + } + } + ] + }, + "optimizerCost": { + "type": "object", + "additionalProperties": false, + "required": [ + "estimated_cost", + "model_calls", + "candidate_evaluation_agent_calls", + "reflection_lm_calls", + "judge_calls_per_candidate_evaluation", + "judge_model_calls", + "native_judge_model_calls", + "derived_judge_model_calls", + "judge_model_call_source", + "token_usage", + "token_usage_known", + "unknown_token_usage_reason", + "usage_evidence_valid", + "reflection_reported_usage" + ], + "properties": { + "estimated_cost": {"type": ["number", "null"], "minimum": 0}, + "model_calls": {"type": "integer", "minimum": 0}, + "candidate_evaluation_agent_calls": {"type": "integer", "minimum": 0}, + "reflection_lm_calls": {"type": "integer", "minimum": 0}, + "judge_calls_per_candidate_evaluation": {"type": "integer", "minimum": 0}, + "judge_model_calls": {"type": "integer", "minimum": 0}, + "native_judge_model_calls": {"type": "integer", "minimum": 0}, + "derived_judge_model_calls": {"type": "integer", "minimum": 0}, + "judge_model_call_source": { + "enum": [ + "none", + "native_optimizer_counter", + "derived_from_candidate_calls_and_llm_metrics", + "native_and_derived_agree", + "reconciled_native_and_derived_max" + ] + }, + "token_usage": { + "oneOf": [ + {"$ref": "#/$defs/tokenUsage"}, + {"type": "null"} + ] + }, + "token_usage_known": {"type": "boolean"}, + "unknown_token_usage_reason": {"type": ["string", "null"]}, + "usage_evidence_valid": {"type": "boolean"}, + "reflection_reported_usage": {"$ref": "#/$defs/reportedUsage"} + }, + "allOf": [ + { + "if": {"properties": {"token_usage_known": {"const": true}}}, + "then": { + "properties": { + "token_usage": {"$ref": "#/$defs/tokenUsage"}, + "unknown_token_usage_reason": {"type": "null"} + } + } + }, + { + "if": {"properties": {"token_usage_known": {"const": false}}}, + "then": { + "properties": { + "token_usage": {"type": "null"}, + "unknown_token_usage_reason": {"type": "string", "minLength": 1} + } + } + } + ] + }, + "finalRevalidationCost": { + "type": "object", + "additionalProperties": false, + "required": [ + "estimated_cost", + "agent_calls_per_run", + "agent_calls", + "judge_calls_per_agent_call", + "judge_model_calls", + "model_calls", + "token_usage", + "token_usage_known", + "unknown_token_usage_reason" + ], + "properties": { + "estimated_cost": {"type": ["number", "null"], "minimum": 0}, + "agent_calls_per_run": {"type": "integer", "minimum": 0}, + "agent_calls": {"type": "integer", "minimum": 0}, + "judge_calls_per_agent_call": {"type": "integer", "minimum": 0}, + "judge_model_calls": {"type": "integer", "minimum": 0}, + "model_calls": {"type": "integer", "minimum": 0}, + "token_usage": { + "oneOf": [ + {"$ref": "#/$defs/tokenUsage"}, + {"type": "null"} + ] + }, + "token_usage_known": {"type": "boolean"}, + "unknown_token_usage_reason": {"type": ["string", "null"]} + }, + "allOf": [ + { + "if": {"properties": {"token_usage_known": {"const": true}}}, + "then": { + "properties": { + "token_usage": {"$ref": "#/$defs/tokenUsage"}, + "unknown_token_usage_reason": {"type": "null"} + } + } + }, + { + "if": {"properties": {"token_usage_known": {"const": false}}}, + "then": { + "properties": { + "token_usage": {"type": "null"}, + "unknown_token_usage_reason": {"type": "string", "minLength": 1} + } + } + } + ] + }, + "cost": { + "type": "object", + "additionalProperties": false, + "required": [ + "currency", + "estimated_total", + "cost_source", + "unknown_cost_reason", + "model_calls", + "token_usage", + "token_usage_known", + "unknown_token_usage_reason", + "optimizer", + "final_revalidation" + ], + "properties": { + "currency": {"const": "USD"}, + "estimated_total": {"type": ["number", "null"], "minimum": 0}, + "cost_source": {"type": "string", "minLength": 1}, + "unknown_cost_reason": {"type": ["string", "null"]}, + "model_calls": {"type": "integer", "minimum": 0}, + "token_usage": { + "oneOf": [ + {"$ref": "#/$defs/tokenUsage"}, + {"type": "null"} + ] + }, + "token_usage_known": {"type": "boolean"}, + "unknown_token_usage_reason": {"type": ["string", "null"]}, + "optimizer": {"$ref": "#/$defs/optimizerCost"}, + "final_revalidation": {"$ref": "#/$defs/finalRevalidationCost"} + }, + "allOf": [ + { + "if": {"properties": {"token_usage_known": {"const": true}}}, + "then": { + "properties": { + "token_usage": {"$ref": "#/$defs/tokenUsage"}, + "unknown_token_usage_reason": {"type": "null"} + } + } + }, + { + "if": {"properties": {"token_usage_known": {"const": false}}}, + "then": { + "properties": { + "token_usage": {"type": "null"}, + "unknown_token_usage_reason": {"type": "string", "minLength": 1} + } + } + } + ] + }, + "optimizationRound": { + "type": "object", + "additionalProperties": false, + "required": [ + "round", + "optimized_field_names", + "prompt_paths", + "prompt_sha256", + "prompt_contents", + "validation_pass_rate", + "metric_breakdown", + "accepted", + "decision_reason", + "failed_case_ids", + "cost_usd", + "token_usage", + "duration_seconds" + ], + "properties": { + "round": {"type": "integer", "minimum": 1}, + "optimized_field_names": {"type": "array", "items": {"type": "string"}}, + "prompt_paths": { + "type": "object", + "additionalProperties": {"type": "string", "minLength": 1} + }, + "prompt_sha256": { + "type": "object", + "additionalProperties": {"type": "string", "pattern": "^[a-f0-9]{64}$"} + }, + "prompt_contents": { + "type": "object", + "additionalProperties": {"type": "string"} + }, + "validation_pass_rate": {"type": "number", "minimum": 0, "maximum": 1}, + "metric_breakdown": {"type": "object", "additionalProperties": {"type": "number"}}, + "accepted": {"type": "boolean"}, + "decision_reason": {"type": "string", "minLength": 1}, + "failed_case_ids": {"type": "array", "items": {"type": "string"}}, + "cost_usd": {"type": "number", "minimum": 0}, + "token_usage": {"$ref": "#/$defs/tokenUsage"}, + "duration_seconds": {"type": "number", "minimum": 0} + } + }, + "candidate": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "audit", + "prompt_patch_summary", + "prompt_artifacts", + "train", + "optimizer_dev", + "final_validation", + "validation", + "delta", + "case_deltas", + "gate", + "failure_attribution", + "artifacts" + ], + "properties": { + "id": {"$ref": "#/$defs/safePathComponent"}, + "audit": {"$ref": "#/$defs/candidateAudit"}, + "prompt_patch_summary": {"type": "string"}, + "prompt_artifacts": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/promptArtifact"} + }, + "train": {"$ref": "#/$defs/evaluationSummary"}, + "optimizer_dev": {"$ref": "#/$defs/evaluationSummary"}, + "final_validation": {"$ref": "#/$defs/evaluationSummary"}, + "validation": {"$ref": "#/$defs/evaluationSummary"}, + "delta": {"$ref": "#/$defs/delta"}, + "case_deltas": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/caseDelta"} + }, + "gate": {"$ref": "#/$defs/candidateGate"}, + "failure_attribution": {"$ref": "#/$defs/failureAttribution"}, + "artifacts": {"type": "object", "additionalProperties": {"type": "string"}} + } + }, + "onlineResult": { + "type": "object", + "additionalProperties": false, + "required": [ + "status", + "error_message", + "baseline_pass_rate", + "best_pass_rate", + "pass_rate_improvement", + "stop_reason", + "baseline_metric_breakdown", + "best_metric_breakdown" + ], + "properties": { + "status": {"type": "string", "minLength": 1}, + "error_message": {"type": "string"}, + "baseline_pass_rate": {"type": "number"}, + "best_pass_rate": {"type": "number"}, + "pass_rate_improvement": {"type": "number"}, + "stop_reason": {"type": ["string", "null"]}, + "baseline_metric_breakdown": { + "type": "object", + "additionalProperties": {"type": "number"} + }, + "best_metric_breakdown": { + "type": "object", + "additionalProperties": {"type": "number"} + } + } + }, + "onlinePreflight": { + "type": "object", + "additionalProperties": false, + "required": [ + "TRPC_AGENT_API_KEY", + "TRPC_AGENT_BASE_URL", + "TRPC_AGENT_MODEL_NAME" + ], + "properties": { + "TRPC_AGENT_API_KEY": {"type": "boolean"}, + "TRPC_AGENT_BASE_URL": {"type": "boolean"}, + "TRPC_AGENT_MODEL_NAME": {"type": "boolean"} + } + }, + "onlineDuration": { + "type": "object", + "additionalProperties": false, + "required": [ + "optimization_seconds", + "baseline_revalidation_seconds", + "candidate_revalidation_seconds", + "gate_elapsed_seconds" + ], + "properties": { + "optimization_seconds": {"type": "number", "minimum": 0}, + "baseline_revalidation_seconds": {"type": "number", "minimum": 0}, + "candidate_revalidation_seconds": {"type": "number", "minimum": 0}, + "gate_elapsed_seconds": {"type": "number", "minimum": 0} + } + }, + "configSnapshot": { + "type": "object", + "additionalProperties": false, + "required": [ + "mode", + "seed", + "gate", + "evaluation", + "evaluation_sha256", + "evaluation_metrics_sha256", + "optimizer_config_sha256", + "prompt_targets", + "evalsets", + "paths" + ], + "properties": { + "mode": {"enum": ["fake", "trace", "online"]}, + "seed": {"type": "integer"}, + "gate": {"type": "object"}, + "evaluation": {"type": "object"}, + "evaluation_sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, + "evaluation_metrics_sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, + "optimizer_config_sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, + "prompt_targets": { + "type": "object", + "additionalProperties": false, + "required": ["system_prompt", "router_prompt"], + "properties": { + "system_prompt": {"$ref": "#/$defs/promptTarget"}, + "router_prompt": {"$ref": "#/$defs/promptTarget"} + } + }, + "evalsets": { + "type": "object", + "additionalProperties": false, + "required": ["train", "optimizer_dev", "final_validation"], + "properties": { + "train": {"$ref": "#/$defs/evalsetManifest"}, + "optimizer_dev": {"$ref": "#/$defs/evalsetManifest"}, + "final_validation": {"$ref": "#/$defs/evalsetManifest"} + } + }, + "paths": { + "type": "object", + "additionalProperties": false, + "required": [ + "train_evalset", + "optimizer_dev_evalset", + "validation_evalset", + "final_validation_evalset", + "optimizer_config", + "evaluation_metrics", + "system_prompt", + "router_prompt" + ], + "properties": { + "train_evalset": {"type": "string", "minLength": 1}, + "optimizer_dev_evalset": {"type": "string", "minLength": 1}, + "validation_evalset": {"type": "string", "minLength": 1}, + "final_validation_evalset": {"type": "string", "minLength": 1}, + "optimizer_config": {"type": "string", "minLength": 1}, + "evaluation_metrics": {"type": "string", "minLength": 1}, + "optimizer_source_config": {"type": "string", "minLength": 1}, + "fixture_outputs": {"type": "string", "minLength": 1}, + "online_eval_metrics": {"type": "string", "minLength": 1}, + "system_prompt": {"type": "string", "minLength": 1}, + "router_prompt": {"type": "string", "minLength": 1} + } + } + } + }, + "evalsetManifest": { + "type": "object", + "additionalProperties": false, + "required": ["path", "sha256", "case_count", "turn_count"], + "properties": { + "path": {"type": "string", "minLength": 1}, + "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, + "case_count": {"type": "integer", "minimum": 1}, + "turn_count": {"type": "integer", "minimum": 1} + } + }, + "environmentSnapshot": { + "type": "object", + "additionalProperties": false, + "required": [ + "git_commit", + "git_dirty", + "python_version", + "sdk_version", + "model_name", + "base_url_host", + "seed", + "command", + "config_path" + ], + "properties": { + "git_commit": {"type": ["string", "null"]}, + "git_dirty": {"type": "boolean"}, + "python_version": {"type": "string"}, + "sdk_version": {"type": ["string", "null"]}, + "model_name": {"type": ["string", "null"]}, + "base_url_host": {"type": ["string", "null"]}, + "seed": {"type": "integer"}, + "command": {"type": "string"}, + "config_path": {"type": "string", "minLength": 1} + } + } + } +} diff --git a/examples/optimization/eval_optimize_loop/optimizer.json b/examples/optimization/eval_optimize_loop/optimizer.json new file mode 100644 index 00000000..c3dc2c32 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/optimizer.json @@ -0,0 +1,87 @@ +{ + "evaluate": { + "metrics": [ + { + "metric_name": "route_tool_args_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "json": { + "match": "exact" + } + } + } + }, + { + "metric_name": "llm_rubric_response", + "threshold": 0.66, + "criterion": { + "llm_judge": { + "judge_model": { + "model_name": "${TRPC_AGENT_MODEL_NAME}", + "base_url": "${TRPC_AGENT_BASE_URL}", + "api_key": "${TRPC_AGENT_API_KEY}", + "generation_config": { + "max_tokens": 1024, + "temperature": 0.2 + } + }, + "rubrics": [ + { + "id": "route_correct", + "content": { + "text": "The route and tool must match the expected support action." + }, + "description": "Correct route and tool", + "type": "FINAL_RESPONSE_QUALITY" + }, + { + "id": "json_only", + "content": { + "text": "The answer must be exactly one valid JSON object without extra prose." + }, + "description": "JSON only", + "type": "FINAL_RESPONSE_QUALITY" + }, + { + "id": "safe_commitments", + "content": { + "text": "The answer must not promise refunds, credits, or manual handling beyond policy." + }, + "description": "No unsafe commitment", + "type": "FINAL_RESPONSE_QUALITY" + } + ] + } + } + } + ], + "num_runs": 1 + }, + "optimize": { + "eval_case_parallelism": 2, + "stop": { + "required_metrics": "all" + }, + "algorithm": { + "name": "gepa_reflective", + "seed": 7, + "reflection_lm": { + "model_name": "${TRPC_AGENT_MODEL_NAME}", + "base_url": "${TRPC_AGENT_BASE_URL}", + "api_key": "${TRPC_AGENT_API_KEY}", + "generation_config": { + "max_tokens": 4096, + "temperature": 0.6 + } + }, + "candidate_selection_strategy": "pareto", + "module_selector": "round_robin", + "reflection_minibatch_size": 3, + "skip_perfect_score": false, + "max_metric_calls": 12, + "max_iterations_without_improvement": 2, + "timeout_seconds": 120 + } + } +} diff --git a/examples/optimization/eval_optimize_loop/optimizer_dev.evalset.json b/examples/optimization/eval_optimize_loop/optimizer_dev.evalset.json new file mode 100644 index 00000000..7bde77f7 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/optimizer_dev.evalset.json @@ -0,0 +1,109 @@ +{ + "eval_set_id": "support_router_optimizer_dev", + "name": "Support router optimizer dev set", + "description": "Three optimizer-internal development cases with IDs, user inputs, and gold outputs distinct from both training and final validation. Final validation gold is only used for baseline scoring and final gate decisions.", + "eval_cases": [ + { + "eval_id": "dev_return_201", + "conversation": [ + { + "invocation_id": "optimizer-dev-1", + "user_content": { + "parts": [ + { + "text": "A coffee maker delivered this morning leaks water. Please return it for a refund." + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "text": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Leaking new appliance with an explicit return request needs refund handling.\"}" + } + ], + "role": "model" + } + } + ], + "session_input": { + "app_name": "support_router_optimizer", + "user_id": "optimizer-dev", + "state": { + "tags": [ + "refund", + "optimizer_dev" + ] + } + } + }, + { + "eval_id": "dev_access_202", + "conversation": [ + { + "invocation_id": "optimizer-dev-2", + "user_content": { + "parts": [ + { + "text": "Two-factor codes never arrive and I need a support specialist to restore access." + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "text": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Failed two-factor delivery with a specialist request needs a human case.\"}" + } + ], + "role": "model" + } + } + ], + "session_input": { + "app_name": "support_router_optimizer", + "user_id": "optimizer-dev", + "state": { + "tags": [ + "manual", + "optimizer_dev" + ] + } + } + }, + { + "eval_id": "dev_warranty_203", + "conversation": [ + { + "invocation_id": "optimizer-dev-3", + "user_content": { + "parts": [ + { + "text": "How long is the warranty on a replacement charger?" + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Replacement charger warranty is a policy question for FAQ.\"}" + } + ], + "role": "model" + } + } + ], + "session_input": { + "app_name": "support_router_optimizer", + "user_id": "optimizer-dev", + "state": { + "tags": [ + "faq", + "optimizer_dev" + ] + } + } + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py new file mode 100644 index 00000000..fe687ae9 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -0,0 +1,3965 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Evaluation + optimization loop example. + +The default mode is deterministic and offline. Fixtures provide only agent +outputs; AgentEvaluator remains the source of scores, pass/fail status, metric +details, and actual conversations. Online mode is explicit and gated by model +environment variables. +""" + +from __future__ import annotations + +import argparse +import asyncio +import copy +import difflib +import hashlib +import importlib.metadata +import json +import math +import os +import platform +import re +import subprocess +import sys +import time +import uuid +from collections import Counter +from collections.abc import Awaitable +from collections.abc import Callable +from collections.abc import Mapping +from datetime import datetime +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +HERE = Path(__file__).resolve().parent +REPO_ROOT = HERE.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) +if str(HERE) not in sys.path: + sys.path.insert(0, str(HERE)) + +from trpc_agent_sdk.log import logger + +TRAIN_PATH = HERE / "train.evalset.json" +OPTIMIZER_DEV_PATH = HERE / "optimizer_dev.evalset.json" +VAL_PATH = HERE / "val.evalset.json" +FIXTURE_PATH = HERE / "fixtures" / "fake_outputs.json" +TRACE_FIXTURE_PATH = HERE / "fixtures" / "trace_outputs.json" +OPTIMIZER_CONFIG_PATH = HERE / "optimizer.json" +REPORT_SCHEMA_PATH = HERE / "optimization_report.schema.json" +PROMPT_DIR = HERE / "agent" / "prompts" +SYSTEM_PROMPT_PATH = PROMPT_DIR / "system.md" +ROUTER_PROMPT_PATH = PROMPT_DIR / "router.md" +PROMPT_TARGET_NAMES = ("system_prompt", "router_prompt") +DEFAULT_RUNS_DIR = HERE / "runs" +DEFAULT_SEED = 7 +DEFAULT_MAX_SECONDS = 180.0 +PRIMARY_METRIC = "route_tool_args_score" +OFFLINE_RUBRIC_METRIC = "llm_rubric_response" +ONLINE_ENV_VARS = ( + "TRPC_AGENT_API_KEY", + "TRPC_AGENT_BASE_URL", + "TRPC_AGENT_MODEL_NAME", +) + +TAXONOMY = ( + "final_response_mismatch", + "tool_call_error", + "parameter_error", + "rubric_failed", + "knowledge_gap", + "format_error", + "runtime_error", + "metric_failed", +) + +OFFLINE_METRICS_CONFIG = { + "metrics": [ + { + "metric_name": PRIMARY_METRIC, + "threshold": 1.0, + "criterion": {"final_response": {"json": {"match": "exact"}}}, + }, + { + "metric_name": OFFLINE_RUBRIC_METRIC, + "threshold": 1.0, + "criterion": {"offline_rubric": {"checks": ["valid_json_object", "route_present", "tool_object_present"]}}, + }, + ], + "num_runs": 1, +} + +DEFAULT_GATE_CONFIG = { + "min_validation_delta": 0.0, + "allow_new_hard_fails": False, + "allow_critical_regression": False, + "max_cost_usd": None, + "max_duration_seconds": DEFAULT_MAX_SECONDS, + "required_metrics": [PRIMARY_METRIC], +} +GATE_CONFIG_KEYS = frozenset(DEFAULT_GATE_CONFIG) +GATE_RUNTIME_KEYS = GATE_CONFIG_KEYS | {"required_metrics_source"} + +SAFE_PATH_COMPONENT = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9_-])?$") +TOKEN_USAGE_KEYS = ("prompt", "completion", "total") +WINDOWS_RESERVED_BASENAMES = { + "con", + "prn", + "aux", + "nul", + *(f"com{number}" for number in range(1, 10)), + *(f"lpt{number}" for number in range(1, 10)), +} +INVALID_JSON_EVIDENCE = object() + + +def load_json(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8") + + +def validate_report_schema(report: dict[str, Any]) -> None: + from jsonschema import Draft202012Validator + from jsonschema import ValidationError + + def reject_nonfinite_numbers(value: Any) -> None: + if isinstance(value, float) and not math.isfinite(value): + raise ValidationError("report contains a non-finite number") + if isinstance(value, dict): + for item in value.values(): + reject_nonfinite_numbers(item) + elif isinstance(value, (list, tuple)): + for item in value: + reject_nonfinite_numbers(item) + + reject_nonfinite_numbers(report) + schema = load_json(REPORT_SCHEMA_PATH) + Draft202012Validator(schema).validate(report) + + def reject(message: str) -> None: + raise ValidationError(message) + + if _is_windows_reserved_name(report.get("run_id")): + reject("run_id uses a Windows reserved device basename") + + baseline = report["baseline"] + if baseline["validation"] != baseline["final_validation"]: + reject("baseline validation must equal final_validation") + + def validate_evaluation_summary(summary: dict[str, Any], label: str) -> None: + case_results = summary["case_results"] + expected_score = round(sum(case["score"] for case in case_results) / len(case_results), 6) + if round(summary["score"], 6) != expected_score: + reject(f"{label} summary score does not match case-derived score") + expected_pass_rate = round( + sum(1 for case in case_results if case["passed"]) / len(case_results), + 6, + ) + if round(summary["pass_rate"], 6) != expected_pass_rate: + reject(f"{label} summary pass_rate does not match case results") + expected_failed_ids = [case["case_id"] for case in case_results if not case["passed"]] + if summary["failed_case_ids"] != expected_failed_ids: + reject(f"{label} summary failed_case_ids do not match case results") + if summary["source"] != "AgentEvaluator": + reject(f"{label} summary source must be AgentEvaluator") + numeric_case_metric_names: set[str] = set() + explicit_no_run_case_ids: set[str] = set() + for case in case_results: + case_metrics = case["metrics"] + if not case_metrics: + if not case["passed"] and case["key_trace"]["error_message"]: + if case["score"] != 0.0: + reject(f"{label} explicit no-run case score must be zero") + explicit_no_run_case_ids.add(case["case_id"]) + continue + reject(f"{label} case metric evidence is empty without an explicit failed run") + for metric_name, metric in case_metrics.items(): + metric_score = _finite_float(metric["score"]) + metric_threshold = _finite_float(metric["threshold"]) + if metric_threshold is None: + reject(f"{label} case metric {metric_name} has an invalid threshold") + if metric_score is None: + if metric["passed"] or metric["status"].lower() == "passed": + reject(f"{label} case metric {metric_name} has contradictory missing-score evidence") + continue + numeric_case_metric_names.add(metric_name) + expected_passed = metric_score >= metric_threshold + expected_status = "passed" if expected_passed else "failed" + if metric["passed"] is not expected_passed or metric["status"].lower() != expected_status: + reject(f"{label} case metric {metric_name} has contradictory score evidence") + if case["passed"] and any(metric["passed"] is not True for metric in case_metrics.values()): + reject(f"{label} passed case contains failed metric evidence") + primary_metric = case_metrics.get(PRIMARY_METRIC) + primary_score = _finite_float(primary_metric.get("score")) if isinstance(primary_metric, Mapping) else None + if primary_score is not None: + expected_case_score = round(primary_score, 6) + else: + fallback_scores = [ + metric_score + for metric in case_metrics.values() + for metric_score in [_finite_float(metric.get("score"))] + if metric_score is not None + ] + expected_case_score = ( + round(sum(fallback_scores) / len(fallback_scores), 6) + if fallback_scores + else (1.0 if case["passed"] else 0.0) + ) + if round(case["score"], 6) != expected_case_score: + reject(f"{label} case score does not match metric-derived score") + if set(summary["metrics"]) != numeric_case_metric_names: + reject(f"{label} metric coverage does not match numeric case metric evidence") + for metric_name, metric in summary["metrics"].items(): + metric_score = _finite_float(metric["score"]) + metric_threshold = _finite_float(metric["threshold"]) + if metric_score is None or metric_threshold is None: + reject(f"{label} summary metric {metric_name} must have numeric score and threshold") + expected_passed = metric_score >= metric_threshold + expected_status = "passed" if expected_passed else "failed" + if metric["passed"] is not expected_passed or metric["status"].lower() != expected_status: + reject(f"{label} summary metric {metric_name} has contradictory score evidence") + case_metric_scores: list[float] = [] + for case in case_results: + case_metric = case["metrics"].get(metric_name) + if not isinstance(case_metric, Mapping): + if case["case_id"] in explicit_no_run_case_ids: + continue + reject(f"{label} metric coverage is missing {metric_name} for an executed case") + case_metric_score = _finite_float(case_metric.get("score")) + if case_metric_score is None: + continue + case_metric_threshold = _finite_float(case_metric.get("threshold")) + if case_metric_threshold != metric_threshold: + reject(f"{label} aggregate metric {metric_name} threshold does not match case evidence") + case_metric_scores.append(case_metric_score) + if not case_metric_scores: + reject(f"{label} aggregate metric {metric_name} has no numeric case evidence") + expected_metric_score = round(sum(case_metric_scores) / len(case_metric_scores), 6) + if round(metric_score, 6) != expected_metric_score: + reject(f"{label} aggregate metric {metric_name} score does not match case evidence") + for case in case_results: + trace = case["key_trace"] + if trace["actual_final_response"] != case["actual_text"]: + reject(f"{label} case {case['case_id']} key trace actual response does not match") + if trace["expected_final_response"] != case["expected_text"]: + reject(f"{label} case {case['case_id']} key trace expected response does not match") + if case["passed"]: + if case["root_cause"] or case["reasons"]: + reject(f"{label} passed case {case['case_id']} must not carry failure attribution") + elif case["root_cause"] not in TAXONOMY or not case["reasons"]: + reject(f"{label} failed case {case['case_id']} requires an explainable root cause") + + def reject_duplicate_cases(summary: dict[str, Any], label: str) -> None: + case_ids = [case["case_id"] for case in summary["case_results"]] + duplicates = sorted(case_id for case_id, count in Counter(case_ids).items() if count > 1) + if duplicates: + reject(f"{label} contains duplicate case_id values: {', '.join(duplicates)}") + + def validate_prompt_artifacts(artifacts: list[dict[str, Any]], label: str) -> None: + names = [artifact["name"] for artifact in artifacts] + if len(names) != len(set(names)): + reject(f"{label} prompt artifact names must be unique") + for artifact in artifacts: + content = artifact["content"] + if artifact["sha256"] != sha256_text(content): + reject(f"{label} prompt artifact hash does not match embedded content") + if artifact["source_written"]: + reject(f"{label} prompt audit must not claim source writes") + candidate_path = Path(artifact["candidate_path"]) + if not candidate_path.is_absolute(): + candidate_path = REPO_ROOT / candidate_path + if candidate_path.is_file() and candidate_path.read_text(encoding="utf-8") != content: + reject(f"{label} prompt artifact content does not match the referenced file") + + for summary_name in ("train", "optimizer_dev", "validation", "final_validation"): + reject_duplicate_cases(baseline[summary_name], f"baseline.{summary_name}") + validate_evaluation_summary(baseline[summary_name], f"baseline.{summary_name}") + validate_prompt_artifacts(baseline["prompt_artifacts"], "baseline") + baseline_prompts_by_name = {artifact["name"]: artifact for artifact in baseline["prompt_artifacts"]} + + if report["failure_attribution"] != attribution_for(baseline["validation"]): + reject("top-level failure attribution does not match baseline validation failures") + + config_snapshot = report["config_snapshot"] + if config_snapshot["mode"] != report["mode"] or config_snapshot["seed"] != report["seed"]: + reject("config snapshot mode and seed must match the report") + recorded_gate_config = config_snapshot.get("gate") + if not isinstance(recorded_gate_config, Mapping): + reject("config_snapshot.gate must be an object") + evaluation_snapshot = config_snapshot["evaluation"] + try: + renormalized_evaluation = normalized_evaluation_config(evaluation_snapshot) + except ValueError as error: + reject(f"config snapshot evaluation is invalid: {error}") + if renormalized_evaluation != evaluation_snapshot: + reject("config snapshot evaluation must be normalized and credential-free") + if config_snapshot["evaluation_sha256"] != sha256_json(evaluation_snapshot): + reject("evaluation config hash does not match the normalized snapshot") + expected_judge_multiplier = judge_calls_per_agent_call(evaluation_snapshot) + config_paths = config_snapshot["paths"] + expected_prompt_names = set(PROMPT_TARGET_NAMES) + prompt_targets = config_snapshot["prompt_targets"] + if set(prompt_targets) != expected_prompt_names: + reject("prompt target manifest must contain exactly the registered target names") + if set(baseline_prompts_by_name) != expected_prompt_names: + reject("baseline prompt artifacts must match the registered target names") + for name in PROMPT_TARGET_NAMES: + target = prompt_targets[name] + baseline_artifact = baseline_prompts_by_name[name] + expected_source_path = config_paths[name] + if target["source_path"] != expected_source_path: + reject(f"prompt target source path does not match config paths: {name}") + if report["artifacts"].get(name) != expected_source_path: + reject(f"prompt source path does not match report artifacts: {name}") + if baseline_artifact["source_path"] != expected_source_path: + reject(f"baseline prompt source path does not match target manifest: {name}") + if target["sha256"] != baseline_artifact["sha256"]: + reject(f"prompt target hash does not match baseline content: {name}") + if baseline_artifact["diff"] != prompt_diff(baseline_artifact["content"], baseline_artifact["content"], name): + reject(f"baseline prompt diff does not match embedded baseline content: {name}") + source_path = Path(expected_source_path) + if not source_path.is_absolute(): + source_path = REPO_ROOT / source_path + if not source_path.is_file(): + reject(f"referenced prompt source artifact must exist: {name}") + source_text = source_path.read_text(encoding="utf-8") + if sha256_text(source_text) != target["sha256"] or source_text != baseline_artifact["content"]: + reject(f"prompt target manifest does not match the referenced source artifact: {name}") + expected_optimizer_config_path = config_paths["optimizer_config"] + if report["artifacts"].get("optimizer_config") != expected_optimizer_config_path: + reject("optimizer config path does not match report artifacts") + expected_optimizer_config_hash = config_snapshot["optimizer_config_sha256"] + optimizer_config_artifact = Path(expected_optimizer_config_path) + if not optimizer_config_artifact.is_absolute(): + optimizer_config_artifact = REPO_ROOT / optimizer_config_artifact + if not optimizer_config_artifact.is_file(): + reject("referenced optimizer config artifact must exist") + if sha256_json_file(optimizer_config_artifact) != expected_optimizer_config_hash: + reject("optimizer config hash does not match the referenced config artifact") + if report["mode"] == "online": + runtime_evaluation = normalized_evaluation_config( + validated_optimizer_evaluate_config(optimizer_config_artifact) + ) + if runtime_evaluation != evaluation_snapshot: + reject("evaluation snapshot does not match the runtime optimizer config") + evaluation_metrics_path = Path(config_paths["evaluation_metrics"]) + if report["artifacts"].get("eval_metrics") != config_paths["evaluation_metrics"]: + reject("evaluation metrics path does not match report artifacts") + if not evaluation_metrics_path.is_absolute(): + evaluation_metrics_path = REPO_ROOT / evaluation_metrics_path + if not evaluation_metrics_path.is_file(): + reject("referenced evaluation metrics artifact must exist") + if sha256_json_file(evaluation_metrics_path) != config_snapshot["evaluation_metrics_sha256"]: + reject("evaluation metrics hash does not match the referenced artifact") + if normalized_evaluation_config(load_json(evaluation_metrics_path)) != evaluation_snapshot: + reject("evaluation snapshot does not match the evaluation metrics artifact") + + manifest_path_keys = { + "train": "train_evalset", + "optimizer_dev": "optimizer_dev_evalset", + "final_validation": "final_validation_evalset", + } + evalset_manifests = config_snapshot["evalsets"] + for role, path_key in manifest_path_keys.items(): + manifest = evalset_manifests[role] + if manifest["path"] != config_paths[path_key]: + reject(f"{role} evalset manifest path does not match config paths") + artifact_key = path_key + if report["artifacts"].get(artifact_key) != manifest["path"]: + reject(f"{role} evalset manifest path does not match report artifacts") + evalset_path = Path(manifest["path"]) + if not evalset_path.is_absolute(): + evalset_path = REPO_ROOT / evalset_path + if not evalset_path.is_file(): + reject(f"referenced {role} evalset artifact must exist") + observed_manifest = build_evalset_manifest(evalset_path) + for evidence_key in ("sha256", "case_count", "turn_count"): + if manifest[evidence_key] != observed_manifest[evidence_key]: + reject(f"{role} evalset manifest {evidence_key} does not match the referenced artifact") + environment = report["environment_snapshot"] + if environment["seed"] != report["seed"]: + reject("environment snapshot seed must match the report") + if environment["config_path"] != expected_optimizer_config_path: + reject("environment config path must match the config snapshot") + + round_ids = [round_record["round"] for round_record in report["optimization_rounds"]] + if len(round_ids) != len(set(round_ids)): + reject("optimization round identifiers must be unique") + for round_record in report["optimization_rounds"]: + prompt_keys = set(round_record["prompt_paths"]) + if not prompt_keys.issubset(expected_prompt_names): + reject("optimization round contains an unknown prompt target") + if prompt_keys != set(round_record["prompt_sha256"]) or prompt_keys != set(round_record["prompt_contents"]): + reject("optimization round prompt path, hash, and content keys must match") + for name in prompt_keys: + content = round_record["prompt_contents"][name] + if round_record["prompt_sha256"][name] != sha256_text(content): + reject("optimization round prompt hash does not match embedded content") + prompt_path = Path(round_record["prompt_paths"][name]) + if not prompt_path.is_absolute(): + prompt_path = REPO_ROOT / prompt_path + if prompt_path.is_file() and prompt_path.read_text(encoding="utf-8") != content: + reject("optimization round prompt content does not match the referenced file") + optimized_field_names = set(round_record["optimized_field_names"]) + if (round_record["accepted"] or optimized_field_names) and not prompt_keys: + reject("accepted or optimizing rounds must contain prompt evidence") + if not optimized_field_names.issubset(prompt_keys): + reject("optimization round fields must be present in prompt evidence") + cost = report["cost"] + if report["mode"] == "online": + online_duration = report.get("online_duration") + if not isinstance(online_duration, Mapping): + reject("online report must include online_duration evidence") + expected_gate_elapsed = round( + online_duration["optimization_seconds"] + + online_duration["baseline_revalidation_seconds"] + + online_duration["candidate_revalidation_seconds"], + 6, + ) + if not math.isclose( + online_duration["gate_elapsed_seconds"], + expected_gate_elapsed, + rel_tol=0.0, + abs_tol=0.000002, + ): + reject("online gate elapsed duration does not match recorded phases") + if report["duration_seconds"] + 0.000002 < online_duration["gate_elapsed_seconds"]: + reject("total duration must cover online gate elapsed duration") + + candidates_by_id: dict[str, dict[str, Any]] = {} + for candidate in report["candidates"]: + candidate_id = candidate["id"] + if candidate_id in candidates_by_id: + reject(f"report contains duplicate candidate id: {candidate_id}") + if _is_windows_reserved_name(candidate_id): + reject("candidate id uses a Windows reserved device basename") + if candidate["gate"]["candidate_id"] != candidate_id: + reject("candidate gate candidate_id must equal candidate id") + if candidate["validation"] != candidate["final_validation"]: + reject("candidate validation must equal final_validation") + for summary_name in ("train", "optimizer_dev", "validation", "final_validation"): + reject_duplicate_cases(candidate[summary_name], f"candidate {candidate_id}.{summary_name}") + validate_evaluation_summary(candidate[summary_name], f"candidate {candidate_id}.{summary_name}") + for summary_name in ("train", "optimizer_dev"): + baseline_case_ids = {case["case_id"] for case in baseline[summary_name]["case_results"]} + candidate_case_ids = {case["case_id"] for case in candidate[summary_name]["case_results"]} + if candidate_case_ids != baseline_case_ids: + reject(f"candidate {candidate_id}.{summary_name} case set must match baseline") + expected_delta = { + "train_score": _score_delta(candidate["train"]["score"], baseline["train"]["score"]), + "optimizer_dev_score": _score_delta( + candidate["optimizer_dev"]["score"], + baseline["optimizer_dev"]["score"], + ), + "validation_score": _score_delta( + candidate["validation"]["score"], + baseline["validation"]["score"], + ), + } + if candidate["delta"] != expected_delta: + reject(f"candidate delta does not match recomputed six-decimal deltas: {candidate_id}") + if candidate["gate"]["validation_delta"] != expected_delta["validation_score"]: + reject(f"candidate gate validation_delta does not match candidate delta: {candidate_id}") + expected_case_deltas = build_case_deltas(baseline["validation"], candidate["validation"]) + if candidate["case_deltas"] != expected_case_deltas: + reject(f"candidate case_deltas do not match recomputed validation case deltas: {candidate_id}") + if candidate["failure_attribution"] != attribution_for(candidate["validation"]): + reject(f"candidate failure attribution does not match validation failures: {candidate_id}") + + candidate_audit = candidate["audit"] + if candidate_audit["seed"] != report["seed"]: + reject(f"candidate audit seed does not match report seed: {candidate_id}") + if candidate_audit["config_path"] != expected_optimizer_config_path: + reject(f"candidate audit config path does not match config snapshot: {candidate_id}") + if candidate_audit["config_sha256"] != expected_optimizer_config_hash: + reject(f"candidate audit config hash does not match config snapshot: {candidate_id}") + validate_prompt_artifacts(candidate["prompt_artifacts"], f"candidate {candidate_id}") + candidate_prompts_by_name = {artifact["name"]: artifact for artifact in candidate["prompt_artifacts"]} + if set(candidate_prompts_by_name) != expected_prompt_names: + reject(f"candidate prompt artifact names must match baseline targets: {candidate_id}") + for name in PROMPT_TARGET_NAMES: + candidate_artifact = candidate_prompts_by_name[name] + baseline_artifact = baseline_prompts_by_name[name] + if candidate_artifact["source_path"] != prompt_targets[name]["source_path"]: + reject(f"candidate prompt source path must match target manifest: {candidate_id}/{name}") + expected_diff = prompt_diff( + baseline_artifact["content"], + candidate_artifact["content"], + name, + ) + if candidate_artifact["diff"] != expected_diff: + reject(f"candidate prompt diff does not match embedded baseline: {candidate_id}/{name}") + pipeline_cost = cost["estimated_total"] + audit_cost = candidate_audit["cost"] + if audit_cost["known"] is not (pipeline_cost is not None) or audit_cost["estimated"] != pipeline_cost: + reject(f"candidate audit cost does not match pipeline cost evidence: {candidate_id}") + if candidate_audit["duration_seconds"] > report["duration_seconds"]: + reject(f"candidate audit duration exceeds total report duration: {candidate_id}") + + gate_duration = candidate_audit["duration_seconds"] + if report["mode"] == "online": + gate_duration = online_duration.get("gate_elapsed_seconds") + if candidate_audit["duration_seconds"] != online_duration["candidate_revalidation_seconds"]: + reject("candidate audit duration does not match online candidate revalidation phase") + expected_gate = apply_gate( + candidate_id=candidate_id, + baseline_val=baseline["validation"], + candidate_val=candidate["validation"], + gate_config=recorded_gate_config, + duration_seconds=gate_duration, + cost_usd=audit_cost["estimated"] if audit_cost["known"] else None, + ) + if report["mode"] == "online": + online_result = report.get("online_result") + if not isinstance(online_result, Mapping): + reject("online report must include online_result evidence") + if online_result.get("status") != "SUCCEEDED": + expected_gate["accepted"] = False + expected_gate["reasons"].append(f"native optimizer status was {online_result.get('status')}") + if online_result.get("error_message"): + expected_gate["reasons"].append(online_result["error_message"]) + if not cost["optimizer"]["usage_evidence_valid"]: + expected_gate["accepted"] = False + expected_gate["reasons"].append("optimizer usage evidence was malformed and cannot be audited") + if candidate["gate"] != expected_gate: + reject(f"candidate gate evidence does not match recomputed gate evidence: {candidate_id}") + if candidate["gate"]["accepted"]: + if candidate["gate"]["validation_delta"] <= 0: + reject("accepted candidate validation delta must be strictly positive") + candidates_by_id[candidate_id] = candidate + + expected_winner = pick_winner(report["candidates"]) + if expected_winner is not None: + expected_decision = { + "accepted": True, + "winner": expected_winner["id"], + "reasons": expected_winner["gate"]["reasons"], + } + expected_top_delta = expected_winner["delta"] + else: + rejection_reasons = ["no candidate passed all gates"] + for candidate in report["candidates"]: + rejection_reasons.extend(f"{candidate['id']}: {reason}" for reason in candidate["gate"]["reasons"]) + expected_decision = { + "accepted": False, + "winner": None, + "reasons": rejection_reasons, + } + expected_top_delta = { + "validation_score": 0.0, + "optimizer_dev_score": 0.0, + "train_score": 0.0, + } + if report["gate_decision"] != expected_decision: + reject("top-level gate decision does not match recomputed candidate decisions") + if report["delta"] != expected_top_delta: + reject("top-level delta does not match recomputed winner delta") + + optimizer_cost = cost["optimizer"] + final_cost = cost["final_revalidation"] + if report["mode"] == "online": + if optimizer_cost["judge_calls_per_candidate_evaluation"] != expected_judge_multiplier: + reject("optimizer judge multiplier does not match the recorded evaluation config") + if final_cost["judge_calls_per_agent_call"] != expected_judge_multiplier: + reject("final judge multiplier does not match the recorded evaluation config") + else: + offline_call_evidence = ( + optimizer_cost["model_calls"], + optimizer_cost["candidate_evaluation_agent_calls"], + optimizer_cost["reflection_lm_calls"], + optimizer_cost["judge_calls_per_candidate_evaluation"], + optimizer_cost["judge_model_calls"], + optimizer_cost["native_judge_model_calls"], + optimizer_cost["derived_judge_model_calls"], + final_cost["agent_calls_per_run"], + final_cost["agent_calls"], + final_cost["judge_calls_per_agent_call"], + final_cost["judge_model_calls"], + final_cost["model_calls"], + cost["model_calls"], + ) + if any(offline_call_evidence): + reject("offline reports must record zero optimizer and provider model calls") + native_judge_calls = optimizer_cost["native_judge_model_calls"] + derived_judge_calls = optimizer_cost["derived_judge_model_calls"] + expected_derived_judge_calls = ( + optimizer_cost["candidate_evaluation_agent_calls"] * optimizer_cost["judge_calls_per_candidate_evaluation"] + ) + if derived_judge_calls != expected_derived_judge_calls: + reject("optimizer derived judge calls do not match candidate calls and recorded multiplier") + expected_judge_calls = max(native_judge_calls, derived_judge_calls) + if optimizer_cost["judge_model_calls"] != expected_judge_calls: + reject("optimizer judge_model_calls must reconcile native and derived counts without double counting") + if native_judge_calls and derived_judge_calls: + expected_judge_source = ( + "native_and_derived_agree" + if native_judge_calls == derived_judge_calls + else "reconciled_native_and_derived_max" + ) + elif native_judge_calls: + expected_judge_source = "native_optimizer_counter" + elif derived_judge_calls: + expected_judge_source = "derived_from_candidate_calls_and_llm_metrics" + else: + expected_judge_source = "none" + if optimizer_cost["judge_model_call_source"] != expected_judge_source: + reject("optimizer judge_model_call_source does not match reconciled judge evidence") + expected_optimizer_calls = ( + optimizer_cost["candidate_evaluation_agent_calls"] + + optimizer_cost["reflection_lm_calls"] + + optimizer_cost["judge_model_calls"] + ) + if optimizer_cost["model_calls"] != expected_optimizer_calls: + reject("optimizer model_calls must equal candidate, reflection, and judge calls") + if report["mode"] == "online": + expected_agent_calls_per_run = (1 + len(report["candidates"])) * sum( + manifest["turn_count"] for manifest in evalset_manifests.values() + ) + if final_cost["agent_calls_per_run"] != expected_agent_calls_per_run: + reject("final revalidation per-run calls do not match authenticated evalset turns") + expected_final_agent_calls = final_cost["agent_calls_per_run"] * evaluation_snapshot["num_runs"] + if final_cost["agent_calls"] != expected_final_agent_calls: + reject("final revalidation agent calls do not match per-run calls and evaluation num_runs") + expected_final_judge_calls = final_cost["agent_calls"] * final_cost["judge_calls_per_agent_call"] + if final_cost["judge_model_calls"] != expected_final_judge_calls: + reject("final revalidation judge calls do not match agent calls and recorded multiplier") + if final_cost["model_calls"] != final_cost["agent_calls"] + final_cost["judge_model_calls"]: + reject("final revalidation model_calls must equal agent plus judge calls") + expected_model_calls = cost["optimizer"]["model_calls"] + cost["final_revalidation"]["model_calls"] + if cost["model_calls"] != expected_model_calls: + reject("top-level model_calls must equal optimizer plus final revalidation calls") + + reflection_usage = optimizer_cost["reflection_reported_usage"] + if reflection_usage["token_usage_known"]: + if reflection_usage["unknown_token_usage_reason"] is not None: + reject("known optimizer reflection token usage must not carry an unknown reason") + elif not reflection_usage["unknown_token_usage_reason"]: + reject("unknown optimizer reflection token usage must carry a reason") + for scope, label in ( + (optimizer_cost, "optimizer"), + (final_cost, "final revalidation"), + (cost, "top-level"), + ): + if scope["token_usage_known"]: + if scope["token_usage"] is None or scope["unknown_token_usage_reason"] is not None: + reject(f"{label} known token usage must provide counters without an unknown reason") + elif scope["token_usage"] is not None or not scope["unknown_token_usage_reason"]: + reject(f"{label} unknown token usage must be null with a reason") + + unknown_optimizer_calls = ( + optimizer_cost["candidate_evaluation_agent_calls"] > 0 or optimizer_cost["judge_model_calls"] > 0 + ) + if unknown_optimizer_calls or not optimizer_cost["usage_evidence_valid"]: + if ( + optimizer_cost["estimated_cost"] is not None + or optimizer_cost["token_usage"] is not None + or optimizer_cost["token_usage_known"] + ): + reject("optimizer calls with unscoped usage must force unknown phase cost and tokens") + else: + if optimizer_cost["estimated_cost"] != reflection_usage["estimated_cost"]: + reject("optimizer phase cost must match scoped reflection cost when fully known") + if optimizer_cost["token_usage"] != reflection_usage["token_usage"]: + reject("optimizer phase tokens must match scoped reflection tokens when fully known") + if optimizer_cost["usage_evidence_valid"] and ( + reflection_usage["estimated_cost"] is None or not reflection_usage["token_usage_known"] + ): + reject("valid optimizer usage evidence requires known scoped reflection usage") + if ( + optimizer_cost["usage_evidence_valid"] + and optimizer_cost["reflection_lm_calls"] == 0 + and (reflection_usage["estimated_cost"] != 0 or reflection_usage["token_usage"]["total"] != 0) + ): + reject("zero reflection calls require zero scoped reflection cost and tokens") + if optimizer_cost["candidate_evaluation_agent_calls"] > 0 and "candidate-evaluation" not in ( + optimizer_cost["unknown_token_usage_reason"] or "" + ): + reject("optimizer candidate-evaluation unknown token reason must name that scope") + if optimizer_cost["candidate_evaluation_agent_calls"] > 0 and ( + "candidate-evaluation" not in (cost["unknown_cost_reason"] or "") + or "candidate-evaluation" not in (cost["unknown_token_usage_reason"] or "") + ): + reject("top-level unknown usage reasons must preserve candidate-evaluation scope") + if optimizer_cost["judge_model_calls"] > 0 and "judge" not in (optimizer_cost["unknown_token_usage_reason"] or ""): + reject("optimizer judge unknown token reason must name that scope") + if optimizer_cost["judge_model_calls"] > 0 and ( + "judge" not in (cost["unknown_cost_reason"] or "") or "judge" not in (cost["unknown_token_usage_reason"] or "") + ): + reject("top-level unknown usage reasons must preserve optimizer judge scope") + + if final_cost["model_calls"] == 0: + if final_cost["estimated_cost"] != 0 or not final_cost["token_usage_known"]: + reject("zero-call final revalidation must have known zero cost and tokens") + elif final_cost["estimated_cost"] is not None or final_cost["token_usage_known"]: + reject("final revalidation calls without usage counters must remain unknown") + + scoped_costs_known = optimizer_cost["estimated_cost"] is not None and final_cost["estimated_cost"] is not None + if scoped_costs_known: + if cost["estimated_total"] != optimizer_cost["estimated_cost"] + final_cost["estimated_cost"]: + reject("top-level estimated cost must equal known scoped costs") + if cost["cost_source"] == "unknown" or cost["unknown_cost_reason"] is not None: + reject("known top-level cost must not carry unknown cost evidence") + elif cost["estimated_total"] is not None or cost["cost_source"] != "unknown" or not cost["unknown_cost_reason"]: + reject("unknown scoped cost must propagate to top-level cost fields") + + scoped_tokens_known = optimizer_cost["token_usage_known"] and final_cost["token_usage_known"] + if scoped_tokens_known: + expected_token_usage = { + key: optimizer_cost["token_usage"][key] + final_cost["token_usage"][key] for key in TOKEN_USAGE_KEYS + } + if not cost["token_usage_known"] or cost["token_usage"] != expected_token_usage: + reject("top-level token usage must equal known scoped token usage") + elif cost["token_usage_known"] or cost["token_usage"] is not None or not cost["unknown_token_usage_reason"]: + reject("unknown scoped token usage must propagate to top-level token fields") + + def validate_token_totals(value: Any) -> None: + if isinstance(value, dict): + if set(value) == set(TOKEN_USAGE_KEYS) and all( + isinstance(value[key], int) and not isinstance(value[key], bool) for key in TOKEN_USAGE_KEYS + ): + if value["total"] != value["prompt"] + value["completion"]: + reject("token usage total must equal prompt plus completion") + for nested in value.values(): + validate_token_totals(nested) + elif isinstance(value, list): + for nested in value: + validate_token_totals(nested) + + validate_token_totals(report) + + +def sha256_text(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def sha256_json(value: Any) -> str: + canonical = json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ) + return sha256_text(canonical) + + +def sha256_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def sha256_json_file(path: Path) -> str: + return sha256_json(load_json(path)) + + +def _redact_evaluation_config(value: Any) -> Any: + if isinstance(value, dict): + redacted: dict[str, Any] = {} + for key, item in value.items(): + normalized_key = re.sub(r"[^a-z0-9]", "", str(key).lower()) + if normalized_key in { + "apikey", + "authorization", + "proxyauthorization", + "baseurl", + "headers", + "cookies", + "accesstoken", + "sessiontoken", + }: + continue + redacted[str(key)] = _redact_evaluation_config(item) + return redacted + if isinstance(value, list): + return [_redact_evaluation_config(item) for item in value] + return value + + +def normalized_evaluation_config(metrics_config: Mapping[str, Any]) -> dict[str, Any]: + from trpc_agent_sdk.evaluation._eval_config import EvalConfig + + try: + eval_config = EvalConfig.model_validate(dict(metrics_config)) + except Exception as error: + raise ValueError(f"invalid evaluation metrics config: {error}") from error + metrics = [] + for metric in eval_config.get_eval_metrics(): + metrics.append( + { + "metric_name": metric.metric_name, + "threshold": float(metric.threshold), + "criterion": _redact_evaluation_config(copy.deepcopy(metric.criterion)), + } + ) + snapshot: dict[str, Any] = { + "metrics": metrics, + "num_runs": eval_config.num_runs, + } + if eval_config.user_simulator_config is not None: + user_simulator = eval_config.user_simulator_config + if hasattr(user_simulator, "model_dump"): + user_simulator = user_simulator.model_dump(mode="json") + snapshot["user_simulator_config"] = _redact_evaluation_config(user_simulator) + return snapshot + + +def build_evalset_manifest(path: Path) -> dict[str, Any]: + payload = load_json(path) + cases = payload.get("eval_cases") if isinstance(payload, dict) else None + if not isinstance(cases, list) or not cases: + raise ValueError(f"evalset manifest requires non-empty eval_cases: {path}") + turn_count = 0 + for case in cases: + conversation = case.get("conversation") if isinstance(case, dict) else None + if not isinstance(conversation, list) or not conversation: + raise ValueError(f"evalset manifest requires non-empty conversations: {path}") + turn_count += len(conversation) + return { + "path": str(path), + "sha256": sha256_json_file(path), + "case_count": len(cases), + "turn_count": turn_count, + } + + +def build_evalset_manifests( + train_evalset: Path, + optimizer_dev_evalset: Path, + val_evalset: Path, +) -> dict[str, dict[str, Any]]: + return { + "train": build_evalset_manifest(train_evalset), + "optimizer_dev": build_evalset_manifest(optimizer_dev_evalset), + "final_validation": build_evalset_manifest(val_evalset), + } + + +def build_candidate_audit( + *, + seed: int, + duration_seconds: float, + cost_usd: float | None, + optimizer_config: Path, +) -> dict[str, Any]: + return { + "seed": seed, + "duration_seconds": round(duration_seconds, 6), + "cost": { + "currency": "USD", + "estimated": cost_usd, + "known": cost_usd is not None, + }, + "config_path": str(optimizer_config), + "config_sha256": sha256_json_file(optimizer_config), + } + + +def _normalized_round_number(value: Any, *, nonnegative: bool = False) -> tuple[int | float, bool]: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return 0.0, True + if not math.isfinite(value) or (nonnegative and value < 0): + return 0.0, True + return value, False + + +def _normalized_round_count(value: Any) -> tuple[int, bool]: + normalized, invalid = _normalized_round_number(value, nonnegative=True) + if invalid: + return 0, True + if isinstance(normalized, int): + return normalized, False + if normalized.is_integer(): + return int(normalized), False + return 0, True + + +def _normalized_token_usage(value: Any) -> tuple[dict[str, int], bool]: + empty = {name: 0 for name in TOKEN_USAGE_KEYS} + if not isinstance(value, dict) or set(value) != set(TOKEN_USAGE_KEYS): + return empty, True + normalized: dict[str, int] = {} + for name in TOKEN_USAGE_KEYS: + count, invalid = _normalized_round_count(value[name]) + if invalid: + return empty, True + normalized[name] = count + if normalized["total"] != normalized["prompt"] + normalized["completion"]: + return empty, True + return normalized, False + + +def _normalized_round_identifier(value: Any) -> tuple[int, bool]: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return 0, True + if isinstance(value, float) and not math.isfinite(value): + return 0, True + if value <= 0 or (isinstance(value, float) and not value.is_integer()): + return 0, True + return int(value), False + + +def _normalized_round_list(value: Any) -> tuple[list[Any], bool]: + if not isinstance(value, (list, tuple)): + return [], True + normalized_values: list[str] = [] + invalid_member = False + for member in value: + if isinstance(member, str): + normalized_values.append(member) + else: + invalid_member = True + return normalized_values, invalid_member + + +def _prompt_sort_key(item: tuple[Any, Any]) -> tuple[str, str]: + name = item[0] + return type(name).__name__, repr(name) + + +def write_optimizer_round_artifacts( + *, + run_dir: Path, + rounds: list[Any], +) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + reserved_round_ids = { + round_id + for round_record in rounds + for round_id, invalid in [_normalized_round_identifier(getattr(round_record, "round", None))] + if not invalid + } + used_round_ids: set[int] = set() + next_fallback_round_id = 1 + for round_record in rounds: + round_id, invalid_round_id = _normalized_round_identifier(getattr(round_record, "round", None)) + duplicate_round_id = not invalid_round_id and round_id in used_round_ids + if invalid_round_id or duplicate_round_id: + while next_fallback_round_id in reserved_round_ids or next_fallback_round_id in used_round_ids: + next_fallback_round_id += 1 + round_id = next_fallback_round_id + next_fallback_round_id += 1 + used_round_ids.add(round_id) + round_component = f"optimizer_round_{round_id:03d}" + round_dir = _resolved_run_descendant( + run_dir, + "prompts", + round_component, + label="optimizer round prompt directory", + ) + round_dir.mkdir(parents=True, exist_ok=True) + prompt_paths: dict[str, str] = {} + prompt_hashes: dict[str, str] = {} + prompt_contents: dict[str, str] = {} + prompt_reasons: list[str] = [] + raw_candidate_prompts = getattr(round_record, "candidate_prompts", None) + invalid_prompt_evidence = not isinstance(raw_candidate_prompts, dict) + if invalid_prompt_evidence: + prompt_reasons.append("candidate_prompts was not a mapping and was normalized to an empty mapping") + raw_candidate_prompts = {} + raw_prompt_items = sorted(raw_candidate_prompts.items(), key=_prompt_sort_key) + for raw_name, raw_content in raw_prompt_items: + if not isinstance(raw_name, str) or raw_name not in PROMPT_TARGET_NAMES: + invalid_prompt_evidence = True + if "unknown prompt target was dropped" not in prompt_reasons: + prompt_reasons.append("unknown prompt target was dropped") + continue + name = raw_name + if isinstance(raw_content, str): + content = raw_content + else: + content = "" + invalid_prompt_evidence = True + if "prompt content was normalized to an empty string" not in prompt_reasons: + prompt_reasons.append("prompt content was normalized to an empty string") + prompt_path = _resolved_run_descendant( + run_dir, + "prompts", + round_component, + f"{name}.md", + label="optimizer round prompt artifact", + ) + prompt_path.write_text(content, encoding="utf-8") + prompt_paths[name] = str(prompt_path) + prompt_hashes[name] = sha256_text(content) + prompt_contents[name] = content + validation_pass_rate, invalid_validation_pass_rate = _normalized_round_number( + getattr(round_record, "validation_pass_rate", None), + nonnegative=True, + ) + invalid_rate_bounds = invalid_validation_pass_rate or validation_pass_rate > 1 + if invalid_rate_bounds: + validation_pass_rate = 0.0 + metric_breakdown: dict[str, int | float] = {} + invalid_numeric_evidence = invalid_validation_pass_rate + raw_metric_breakdown = getattr(round_record, "metric_breakdown", {}) + if not isinstance(raw_metric_breakdown, dict): + raw_metric_breakdown = {} + invalid_numeric_evidence = True + invalid_mapping_key_evidence = False + for name, value in raw_metric_breakdown.items(): + if not isinstance(name, str): + invalid_mapping_key_evidence = True + continue + normalized, invalid = _normalized_round_number(value) + metric_breakdown[name] = normalized + invalid_numeric_evidence = invalid_numeric_evidence or invalid + cost_usd, invalid_cost = _normalized_round_number( + getattr(round_record, "round_llm_cost", None), + nonnegative=True, + ) + duration_seconds, invalid_duration = _normalized_round_number( + getattr(round_record, "duration_seconds", None), + nonnegative=True, + ) + invalid_numeric_evidence = invalid_numeric_evidence or invalid_cost or invalid_duration + raw_token_usage = getattr(round_record, "round_token_usage", {}) + token_usage, invalid_token_usage = _normalized_token_usage(raw_token_usage) + invalid_numeric_evidence = invalid_numeric_evidence or invalid_token_usage + invalid_collection_evidence = False + optimized_field_names, invalid_optimized_field_names = _normalized_round_list( + getattr(round_record, "optimized_field_names", None) + ) + failed_case_ids, invalid_failed_case_ids = _normalized_round_list( + getattr(round_record, "failed_case_ids", None) + ) + invalid_collection_evidence = invalid_optimized_field_names or invalid_failed_case_ids + reason_values: list[str] = [] + invalid_reason_evidence = False + for field_name in ("acceptance_reason", "skip_reason", "error_message"): + value = getattr(round_record, field_name, None) + if value is None: + continue + if not isinstance(value, str): + invalid_reason_evidence = True + continue + if value: + reason_values.append(value) + break + decision_reason = sanitize_report_text(reason_values[0] if reason_values else "") or ( + "optimizer reported no reason" + ) + accepted = getattr(round_record, "accepted", False) is True + if ( + invalid_round_id + or duplicate_round_id + or invalid_numeric_evidence + or invalid_rate_bounds + or invalid_prompt_evidence + or invalid_collection_evidence + or invalid_reason_evidence + or invalid_mapping_key_evidence + ): + accepted = False + if invalid_numeric_evidence: + decision_reason += "; invalid numeric round evidence was normalized and rejected" + if invalid_token_usage: + decision_reason += "; invalid token usage was normalized to prompt/completion/total zeros" + if invalid_round_id: + decision_reason += f"; invalid round identifier was normalized to {round_id} and rejected" + if duplicate_round_id: + decision_reason += f"; duplicate round identifier was normalized to {round_id} and rejected" + if invalid_rate_bounds: + decision_reason += "; validation_pass_rate was out of bounds and normalized to 0.0; round rejected" + if invalid_collection_evidence: + decision_reason += "; invalid round collections were normalized and rejected" + if invalid_reason_evidence: + decision_reason += "; invalid round reason fields were ignored and rejected" + if invalid_mapping_key_evidence: + decision_reason += "; invalid mapping keys were dropped and rejected" + for prompt_reason in prompt_reasons: + decision_reason += f"; {prompt_reason}; round rejected" + missing_prompt_fields = [name for name in optimized_field_names if name not in prompt_paths] + if missing_prompt_fields: + accepted = False + optimized_field_names = [name for name in optimized_field_names if name in prompt_paths] + decision_reason += "; optimized fields without prompt evidence were dropped and rejected" + if accepted and (not optimized_field_names or not prompt_paths): + accepted = False + decision_reason += "; accepted round lacked auditable optimized prompt evidence and was rejected" + records.append( + { + "round": round_id, + "optimized_field_names": optimized_field_names, + "prompt_paths": prompt_paths, + "prompt_sha256": prompt_hashes, + "prompt_contents": prompt_contents, + "validation_pass_rate": float(validation_pass_rate), + "metric_breakdown": metric_breakdown, + "accepted": accepted, + "decision_reason": decision_reason, + "failed_case_ids": failed_case_ids, + "cost_usd": float(cost_usd), + "token_usage": token_usage, + "duration_seconds": float(duration_seconds), + } + ) + return records + + +def _git_output(*args: str) -> str | None: + try: + proc = subprocess.run( + ["git", *args], + cwd=REPO_ROOT, + check=True, + capture_output=True, + text=True, + ) + except Exception: # noqa: BLE001 - environment snapshot must not fail the run. + return None + return proc.stdout.strip() + + +def sdk_version() -> str | None: + try: + return importlib.metadata.version("trpc-agent-py") + except importlib.metadata.PackageNotFoundError: + return None + + +def base_url_host(base_url: str | None) -> str | None: + if not base_url: + return None + parsed = urlparse(base_url) + return parsed.hostname or None + + +def environment_snapshot( + *, + seed: int, + command: str | None, + config_path: str, +) -> dict[str, Any]: + return { + "git_commit": _git_output("rev-parse", "HEAD"), + "git_dirty": bool(_git_output("status", "--porcelain")), + "python_version": platform.python_version(), + "sdk_version": sdk_version(), + "model_name": os.getenv("TRPC_AGENT_MODEL_NAME"), + "base_url_host": base_url_host(os.getenv("TRPC_AGENT_BASE_URL")), + "seed": seed, + "command": command or "programmatic", + "config_path": config_path, + } + + +def resolve_path(path: Path | None, default: Path) -> Path: + return (path or default).expanduser().resolve() + + +def optimizer_metric_names(config_path: Path) -> list[str]: + payload = load_json(config_path) + evaluate = payload.get("evaluate") if isinstance(payload, dict) else None + metrics = evaluate.get("metrics") if isinstance(evaluate, dict) else None + if not isinstance(metrics, list): + raise ValueError("optimizer evaluate.metrics must be an array") + names: list[str] = [] + for position, metric in enumerate(metrics): + if not isinstance(metric, dict): + raise ValueError(f"optimizer evaluate.metrics[{position}] must be an object") + name = metric.get("metric_name") or metric.get("metricName") + if not isinstance(name, str) or not name.strip(): + raise ValueError(f"optimizer evaluate.metrics[{position}].metric_name must be a non-empty string") + names.append(name) + if len(set(names)) != len(names): + raise ValueError("optimizer evaluate.metrics metric_name values must be unique") + return names + + +def optimizer_required_metrics(config_path: Path) -> tuple[list[str], str]: + payload = load_json(config_path) + optimize = payload.get("optimize") if isinstance(payload, dict) else None + stop = optimize.get("stop") if isinstance(optimize, dict) else None + if not isinstance(stop, dict): + raise ValueError("optimizer optimize.stop must be an object") + required = stop.get("required_metrics") + if required is None: + return [], "optimizer_config" + if required == "all": + return optimizer_metric_names(config_path), "optimizer_config" + if not isinstance(required, list): + raise ValueError("optimizer required_metrics must be 'all', null, or an array of strings") + if any(not isinstance(name, str) or not name.strip() for name in required): + raise ValueError("optimizer required_metrics must contain only non-empty strings") + if len(set(required)) != len(required): + raise ValueError("optimizer required_metrics must contain unique strings") + unknown = sorted(set(required) - set(optimizer_metric_names(config_path))) + if unknown: + raise ValueError("optimizer required_metrics references unknown metrics: " + ", ".join(unknown)) + return required, "optimizer_config" + + +def validated_optimizer_evaluate_config(config_path: Path) -> dict[str, Any]: + from trpc_agent_sdk.evaluation import load_optimize_config + + try: + load_optimize_config(str(config_path)) + optimizer_required_metrics(config_path) + except Exception as error: + raise ValueError(f"invalid optimizer config: {error}") from error + payload = load_json(config_path) + return payload["evaluate"] + + +def materialize_optimizer_config( + *, + run_dir: Path, + source_config: Path, + seed: int, +) -> Path: + if isinstance(seed, bool) or not isinstance(seed, int): + raise ValueError("optimizer seed must be an integer") + validated_optimizer_evaluate_config(source_config) + payload = load_json(source_config) + optimize = payload.get("optimize") if isinstance(payload, dict) else None + algorithm = optimize.get("algorithm") if isinstance(optimize, dict) else None + if not isinstance(algorithm, dict): + raise ValueError("optimizer optimize.algorithm must be an object") + runtime_payload = copy.deepcopy(payload) + runtime_payload["optimize"]["algorithm"]["seed"] = seed + runtime_path = _resolved_run_descendant( + run_dir, + "optimizer.json", + label="runtime optimizer config artifact", + ) + write_json(runtime_path, runtime_payload) + validated_optimizer_evaluate_config(runtime_path) + return runtime_path + + +def online_preflight() -> dict[str, bool]: + return {name: bool(os.getenv(name)) for name in ONLINE_ENV_VARS} + + +def format_online_preflight(preflight: dict[str, bool]) -> str: + parts = [f"{name}={'present' if preflight.get(name) else 'missing'}" for name in ONLINE_ENV_VARS] + return "online preflight: " + " ".join(parts) + + +def require_online_preflight() -> dict[str, bool]: + preflight = online_preflight() + missing = [name for name, exists in preflight.items() if not exists] + if missing: + raise ValueError( + format_online_preflight(preflight) + + "; online mode requires environment variables: " + + ", ".join(ONLINE_ENV_VARS) + + f"; missing: {', '.join(missing)}" + ) + return preflight + + +def final_text_from_content(content: Any) -> str: + if content is None: + return "" + if isinstance(content, str): + return content.strip() + if isinstance(content, dict): + parts = content.get("parts") or [] + else: + parts = getattr(content, "parts", None) or [] + return "\n".join( + str((part.get("text", "") if isinstance(part, dict) else getattr(part, "text", "")) or "") + for part in parts + if not (part.get("thought", False) if isinstance(part, dict) else getattr(part, "thought", False)) + ).strip() + + +def parsed_json_evidence(content: Any) -> Any: + visible_text = final_text_from_content(content) + try: + parsed = json.loads(visible_text) + json.dumps( + parsed, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ) + except (TypeError, ValueError, json.JSONDecodeError): + return INVALID_JSON_EVIDENCE + return parsed + + +def canonical_gold_evidence(content: Any) -> str: + visible_text = final_text_from_content(content) + parsed = parsed_json_evidence(content) + if parsed is INVALID_JSON_EVIDENCE: + return "text:" + " ".join(visible_text.split()) + return "json:" + json.dumps( + parsed, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ) + + +_SENSITIVE_REPORT_TEXT = re.compile( + r"""\b(?: + authorization|proxy-authorization|bearer|set-cookie|cookies?|headers?| + (?:x[-_])?api[-_]?key|api[-_ ]?key|access[-_ ]?key| + (?:access|session|security)[-_ ]?tokens?| + secrets?|credentials?| + (?:[a-z0-9]+[-_])+[a-z0-9_-]*(?:token|key|secret|credential)[a-z0-9_-]* + )\b""", + re.IGNORECASE | re.VERBOSE, +) +_PROVIDER_URL = re.compile(r"https?://\S+", re.IGNORECASE) +_STANDALONE_PROVIDER_KEY = re.compile(r"\bsk-[A-Za-z0-9_-]{16,}\b", re.IGNORECASE) + + +def sanitize_report_text(value: Any) -> str | None: + if value is None: + return None + message = str(value).strip() + sensitive_starts = [ + match.start() + for pattern in (_SENSITIVE_REPORT_TEXT, _PROVIDER_URL, _STANDALONE_PROVIDER_KEY) + for match in [pattern.search(message)] + if match is not None + ] + for variable in ("TRPC_AGENT_API_KEY", "TRPC_AGENT_BASE_URL"): + configured_value = os.getenv(variable) + if configured_value and configured_value in message: + sensitive_starts.append(message.index(configured_value)) + if not sensitive_starts: + return message + context = message[: min(sensitive_starts)].rstrip(" :;,-") + return f"{context}: provider details redacted" if context else "provider details redacted" + + +def final_text(invocation: dict[str, Any]) -> str: + return final_text_from_content(invocation.get("final_response")) + + +def case_user_text(case: dict[str, Any]) -> str: + invocation = case["conversation"][0] + parts = invocation["user_content"]["parts"] + return "".join(str(part.get("text", "")) for part in parts).strip() + + +def case_expected_text(case: dict[str, Any]) -> str: + return final_text(case["conversation"][0]) + + +def case_tags(case: dict[str, Any]) -> list[str]: + state = (case.get("session_input") or {}).get("state") or {} + tags = state.get("tags") or [] + return [str(tag) for tag in tags] + + +def load_gate_config( + path: Path | None = None, + overrides: dict[str, Any] | None = None, + optimizer_config: Path | None = None, +) -> dict[str, Any]: + config = copy.deepcopy(DEFAULT_GATE_CONFIG) + required_source = "default" + path_payload: dict[str, Any] = {} + if path is not None: + raw_path_payload = load_json(path) + if not isinstance(raw_path_payload, Mapping): + raise ValueError("gate config must be a JSON object") + path_payload = dict(raw_path_payload) + unknown_keys = sorted(str(key) for key in set(path_payload) - GATE_CONFIG_KEYS) + if unknown_keys: + raise ValueError("unknown gate config field(s): " + ", ".join(unknown_keys)) + config.update(path_payload) + if "required_metrics" in path_payload: + required_source = "gate_config" + if overrides is not None: + if not isinstance(overrides, Mapping): + raise ValueError("gate config overrides must be a mapping") + override_payload = dict(overrides) + unknown_keys = sorted(str(key) for key in set(override_payload) - GATE_CONFIG_KEYS) + if unknown_keys: + raise ValueError("unknown gate config override field(s): " + ", ".join(unknown_keys)) + config.update(override_payload) + if "required_metrics" in overrides: + required_source = "override" + if optimizer_config is not None and required_source == "default": + required, required_source = optimizer_required_metrics(optimizer_config) + config["required_metrics"] = required + elif optimizer_config is not None and config.get("required_metrics") == "all": + config["required_metrics"] = optimizer_metric_names(optimizer_config) + if config.get("required_metrics") is None: + config["required_metrics"] = [] + config["required_metrics_source"] = required_source + return config + + +def json_criteria_from_evaluation_config(metrics_config: Mapping[str, Any] | None) -> list[Any]: + from trpc_agent_sdk.evaluation._eval_config import EvalConfig + from trpc_agent_sdk.evaluation._eval_criterion import FinalResponseCriterion + from trpc_agent_sdk.evaluation._eval_criterion import JSONCriterion + + criteria: list[JSONCriterion] = [] + + def collect(value: Any) -> None: + if isinstance(value, Mapping): + for key, nested in value.items(): + normalized_key = str(key).replace("_", "").lower() + if normalized_key == "finalresponse" and isinstance(nested, Mapping): + final_response = FinalResponseCriterion.from_dict(dict(nested)) + if final_response is not None and final_response.json_config is not None: + criteria.append(JSONCriterion.model_validate(final_response.json_config)) + elif normalized_key == "json" and isinstance(nested, Mapping): + criteria.append(JSONCriterion.model_validate(dict(nested))) + else: + collect(nested) + elif isinstance(value, list): + for nested in value: + collect(nested) + + if metrics_config is not None: + eval_config = EvalConfig.model_validate(dict(metrics_config)) + for metric in eval_config.get_eval_metrics(): + collect(metric.criterion) + return criteria or [JSONCriterion()] + + +def validate_inputs( + train_evalset: Path, + optimizer_dev_evalset: Path, + val_evalset: Path, + *, + metrics_config: Mapping[str, Any] | None = None, +) -> None: + paths = { + "train": train_evalset, + "optimizer_dev": optimizer_dev_evalset, + "final_validation": val_evalset, + } + for path in paths.values(): + if not path.is_file(): + raise FileNotFoundError(path) + + role_pairs = (("train", "optimizer_dev"), ("train", "final_validation"), ("optimizer_dev", "final_validation")) + json_criteria = json_criteria_from_evaluation_config(metrics_config) + for left_role, right_role in role_pairs: + left = paths[left_role] + right = paths[right_role] + if left.resolve() == right.resolve() or os.path.samefile(left, right): + raise ValueError(f"{left_role} and {right_role} evalsets resolve to the same file") + if sha256_file(left) == sha256_file(right): + raise ValueError(f"{left_role} and {right_role} evalsets are byte-identical copies") + + evidence: dict[str, dict[str, Any]] = {} + for role, path in paths.items(): + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise ValueError(f"{role} evalset must be valid UTF-8 JSON: {error}") from error + if not isinstance(payload, dict) or not isinstance(payload.get("eval_cases"), list): + raise ValueError(f"{role} evalset must be an object with an eval_cases array") + if not payload["eval_cases"]: + raise ValueError(f"{role} evalset eval_cases must not be empty") + role_evidence = {"id": set(), "input": set(), "gold": set(), "json_gold": []} + for position, case in enumerate(payload["eval_cases"]): + prefix = f"{role} evalset eval_cases[{position}]" + if not isinstance(case, dict): + raise ValueError(f"{prefix} must be an object") + case_id = case.get("eval_id") + conversation = case.get("conversation") + if not isinstance(case_id, str) or not case_id.strip(): + raise ValueError(f"{prefix}.eval_id must be a non-empty string") + if case_id in role_evidence["id"]: + raise ValueError(f"{role} evalset contains duplicate eval_id: {case_id}") + if not isinstance(conversation, list) or not conversation: + raise ValueError(f"{prefix}.conversation must be a non-empty array of objects") + role_evidence["id"].add(case_id) + for invocation_position, invocation in enumerate(conversation): + invocation_prefix = f"{prefix}.conversation[{invocation_position}]" + if not isinstance(invocation, dict): + raise ValueError(f"{invocation_prefix} must be an object") + user_content = invocation.get("user_content") + final_response = invocation.get("final_response") + if not isinstance(user_content, dict) or not isinstance(user_content.get("parts"), list): + raise ValueError(f"{invocation_prefix} must contain user_content.parts as an array") + if not isinstance(final_response, dict) or not isinstance(final_response.get("parts"), list): + raise ValueError(f"{invocation_prefix} must contain final_response.parts as an array") + user_parts = user_content["parts"] + gold_parts = final_response["parts"] + if not user_parts or not all( + isinstance(part, dict) and isinstance(part.get("text"), str) for part in user_parts + ): + raise ValueError(f"{invocation_prefix} user_content.parts must contain text strings") + if not gold_parts or not all( + isinstance(part, dict) and isinstance(part.get("text"), str) for part in gold_parts + ): + raise ValueError(f"{invocation_prefix} final_response.parts must contain text strings") + normalized_input = " ".join("".join(part["text"] for part in user_parts).split()).casefold() + visible_gold = final_text_from_content(final_response) + if not normalized_input: + raise ValueError(f"{invocation_prefix} normalized user input must be non-empty") + if not visible_gold: + raise ValueError(f"{invocation_prefix} visible final response must be non-empty") + role_evidence["input"].add(normalized_input) + role_evidence["gold"].add(canonical_gold_evidence(final_response)) + parsed_gold = parsed_json_evidence(final_response) + if parsed_gold is not INVALID_JSON_EVIDENCE: + role_evidence["json_gold"].append(parsed_gold) + evidence[role] = role_evidence + + for left_role, right_role in role_pairs: + for evidence_type in ("id", "input", "gold"): + overlap = evidence[left_role][evidence_type] & evidence[right_role][evidence_type] + if overlap: + raise ValueError(f"{left_role} and {right_role} evalsets overlap in {evidence_type} evidence") + if any( + criterion.matches(left_gold, right_gold) + for criterion in json_criteria + for left_gold in evidence[left_role]["json_gold"] + for right_gold in evidence[right_role]["json_gold"] + ): + raise ValueError(f"{left_role} and {right_role} evalsets overlap in gold evidence") + + +def _is_windows_reserved_name(value: Any) -> bool: + if not isinstance(value, str): + return False + basename = value.rstrip(" .").split(".", 1)[0] + return basename.casefold() in WINDOWS_RESERVED_BASENAMES + + +def _validate_safe_path_component(value: Any, *, label: str) -> str: + if not isinstance(value, str) or SAFE_PATH_COMPONENT.fullmatch(value) is None or _is_windows_reserved_name(value): + raise ValueError(f"{label} must be a safe single path component") + return value + + +def _resolved_artifact_child(parent: Path, component: str, *, label: str) -> Path: + safe_component = _validate_safe_path_component(component, label=label) + resolved_parent = parent.resolve() + child = resolved_parent / safe_component + if child.is_symlink(): + raise ValueError(f"resolved {label} artifact path must not be a symlink") + resolved_child = child.resolve() + if not resolved_child.is_relative_to(resolved_parent): + raise ValueError(f"resolved {label} artifact path must stay beneath its parent") + return resolved_child + + +def _resolved_run_descendant(run_dir: Path, *components: str, label: str) -> Path: + if run_dir.is_symlink(): + raise ValueError(f"resolved {label} must remain beneath run_dir and must not traverse a symlink") + lexical_descendant = run_dir.joinpath(*components) + try: + relative_parts = lexical_descendant.relative_to(run_dir).parts + except ValueError as error: + raise ValueError(f"resolved {label} must remain beneath run_dir") from error + cursor = run_dir + for component in relative_parts: + cursor = cursor / component + if cursor.is_symlink(): + raise ValueError(f"resolved {label} must remain beneath run_dir and must not traverse a symlink") + resolved_run_dir = run_dir.resolve() + resolved_descendant = lexical_descendant.resolve() + if not resolved_descendant.is_relative_to(resolved_run_dir): + raise ValueError(f"resolved {label} must remain beneath run_dir") + return resolved_descendant + + +def make_run_dir(output_dir: Path | None, run_id: str) -> Path: + base = output_dir or DEFAULT_RUNS_DIR + base = base.expanduser() + if not base.is_absolute(): + base = (Path.cwd() / base).resolve() + else: + base = base.resolve() + run_dir = _resolved_artifact_child(base, run_id, label="run_id") + run_dir.mkdir(parents=True, exist_ok=True) + return run_dir + + +def offline_metrics_path(run_dir: Path) -> Path: + path = _resolved_run_descendant(run_dir, "offline_metrics.json", label="offline metrics artifact") + write_json(path, OFFLINE_METRICS_CONFIG) + return path + + +def online_metrics_path(run_dir: Path, optimizer_config: Path) -> Path: + path = _resolved_run_descendant(run_dir, "online_eval_metrics.json", label="online metrics artifact") + write_json(path, load_json(optimizer_config)["evaluate"]) + return path + + +def read_source_prompts(system_prompt: Path, router_prompt: Path) -> dict[str, tuple[Path, str]]: + return { + "system_prompt": (system_prompt, system_prompt.read_text(encoding="utf-8")), + "router_prompt": (router_prompt, router_prompt.read_text(encoding="utf-8")), + } + + +def build_prompt_target_manifest( + source_prompts: Mapping[str, tuple[Path, str]], +) -> dict[str, dict[str, str]]: + return { + name: {"source_path": str(source_path), "sha256": sha256_text(source_text)} + for name in PROMPT_TARGET_NAMES + for source_path, source_text in [source_prompts[name]] + } + + +def offline_candidate_prompts( + source_prompts: dict[str, tuple[Path, str]], + candidate_id: str, + summary: str, +) -> dict[str, str]: + prompts = {name: text for name, (_, text) in source_prompts.items()} + if candidate_id != "baseline": + prompts["router_prompt"] = ( + prompts["router_prompt"].rstrip() + "\n\n" + f"Offline candidate patch ({candidate_id}): {summary}\n" + ) + return prompts + + +def prompt_diff(source: str, candidate: str, name: str) -> str: + if source == candidate: + return "unchanged" + return "".join( + difflib.unified_diff( + source.splitlines(keepends=True), + candidate.splitlines(keepends=True), + fromfile=f"source/{name}.md", + tofile=f"candidate/{name}.md", + ) + ) + + +def write_prompt_artifacts( + *, + run_dir: Path, + candidate_id: str, + source_prompts: dict[str, tuple[Path, str]], + candidate_prompts: dict[str, str], + summary: str, + source_written: bool, +) -> tuple[list[dict[str, Any]], dict[str, str]]: + safe_candidate_id = _validate_safe_path_component(candidate_id, label="candidate_id") + prompt_dir = _resolved_run_descendant( + run_dir, + "prompts", + safe_candidate_id, + label="candidate prompt directory", + ) + prompt_dir.mkdir(parents=True, exist_ok=True) + audit: list[dict[str, Any]] = [] + patch_lines = [f"candidate: {candidate_id}", f"summary: {summary}", ""] + for name, (source_path, source_text) in source_prompts.items(): + candidate_text = candidate_prompts.get(name, source_text) + candidate_path = _resolved_run_descendant( + run_dir, + "prompts", + safe_candidate_id, + f"{name}.md", + label="candidate prompt artifact", + ) + candidate_path.write_text(candidate_text, encoding="utf-8") + diff_text = prompt_diff(source_text, candidate_text, name) + patch_lines.extend([f"## {name}", diff_text, ""]) + audit.append( + { + "name": name, + "source_path": str(source_path), + "candidate_path": str(candidate_path), + "sha256": sha256_text(candidate_text), + "content": candidate_text, + "source_written": source_written, + "summary": summary, + "diff": diff_text, + } + ) + patch_path = _resolved_run_descendant( + run_dir, + "prompts", + safe_candidate_id, + "prompt_patch.diff", + label="candidate prompt patch", + ) + patch_path.write_text("\n".join(patch_lines), encoding="utf-8") + return audit, { + "prompt_dir": str(prompt_dir), + "prompt_patch": str(patch_path), + } + + +def _json_or_none(text: str) -> Any | None: + try: + parsed = json.loads((text or "").strip()) + except Exception: # noqa: BLE001 - malformed model output becomes attribution. + return None + return parsed if isinstance(parsed, dict) else None + + +def _route_tool_args(value: Any) -> dict[str, Any] | None: + parsed = _json_or_none(final_text_from_content(value)) + if not isinstance(parsed, dict): + return None + tool = parsed.get("tool") + if not isinstance(tool, dict): + return None + if "route" not in parsed or "name" not in tool or "arguments" not in tool: + return None + arguments = tool.get("arguments") + if not isinstance(arguments, dict): + return None + return { + "route": str(parsed.get("route")), + "tool": { + "name": str(tool.get("name")), + "arguments": arguments, + }, + } + + +def route_tool_args_match(actual: Any, expected: Any) -> bool: + """Compare router outputs by route, tool name, and arguments only.""" + + actual_structured = _route_tool_args(actual) + expected_structured = _route_tool_args(expected) + return actual_structured is not None and actual_structured == expected_structured + + +_MISSING = object() + + +def _install_route_tool_args_metric(): + from trpc_agent_sdk.evaluation._evaluator_registry import EVALUATOR_REGISTRY + from trpc_agent_sdk.evaluation._final_response_evaluator import FinalResponseEvaluator + + previous_evaluator = EVALUATOR_REGISTRY._registry.get(PRIMARY_METRIC, _MISSING) + previous_compare = EVALUATOR_REGISTRY._criterion_compares.get(PRIMARY_METRIC, _MISSING) + EVALUATOR_REGISTRY.register(PRIMARY_METRIC, FinalResponseEvaluator) + EVALUATOR_REGISTRY.set_criterion_compare(PRIMARY_METRIC, route_tool_args_match) + return previous_evaluator, previous_compare + + +def _restore_route_tool_args_metric(state: tuple[Any, Any]) -> None: + from trpc_agent_sdk.evaluation._evaluator_registry import EVALUATOR_REGISTRY + + previous_evaluator, previous_compare = state + if previous_evaluator is _MISSING: + EVALUATOR_REGISTRY._registry.pop(PRIMARY_METRIC, None) + else: + EVALUATOR_REGISTRY.register(PRIMARY_METRIC, previous_evaluator) + if previous_compare is _MISSING: + EVALUATOR_REGISTRY._criterion_compares.pop(PRIMARY_METRIC, None) + else: + EVALUATOR_REGISTRY.set_criterion_compare(PRIMARY_METRIC, previous_compare) + + +def _metric_failed(metric: dict[str, Any]) -> bool: + if "passed" in metric: + return metric["passed"] is False + return str(metric.get("status", "")).lower() == "failed" + + +def _failed_metric_names(metrics: dict[str, dict[str, Any]]) -> list[str]: + return sorted(name for name, metric in metrics.items() if _metric_failed(metric)) + + +def _metric_failure_root(failed_metric_names: list[str]) -> tuple[str, str]: + rubric = [name for name in failed_metric_names if "rubric" in name or name.startswith("llm_")] + if rubric: + return "rubric_failed", "rubric metric failed: " + ", ".join(rubric) + knowledge = [ + name + for name in failed_metric_names + if any(token in name.lower() for token in ("knowledge", "retrieval", "recall", "ground")) + ] + if knowledge: + return "knowledge_gap", "knowledge metric failed: " + ", ".join(knowledge) + return "metric_failed", "content metric failed: " + ", ".join(failed_metric_names) + + +def attribute_failure_case( + *, + actual_text: str, + expected_text: str, + error_message: str | None, + metrics: dict[str, dict[str, Any]], +) -> dict[str, Any]: + """Classify one failed case using evaluator output and response shape.""" + + if error_message: + return { + "root_cause": "runtime_error", + "reasons": [f"evaluation runtime error: {error_message}"], + } + + actual = _json_or_none(actual_text) + expected = _json_or_none(expected_text) + if actual is None: + return { + "root_cause": "format_error", + "reasons": ["actual final response is not a valid JSON object"], + } + if expected is None: + return { + "root_cause": "runtime_error", + "reasons": ["expected final response is not a valid JSON object"], + } + + actual_tool = actual.get("tool") + expected_tool = expected.get("tool") + if not isinstance(expected_tool, dict): + return { + "root_cause": "runtime_error", + "reasons": ["expected final response has a non-object tool field"], + } + if str(actual.get("route", "")) != str(expected.get("route", "")): + return { + "root_cause": "final_response_mismatch", + "reasons": [ + "actual route " f"{actual.get('route')!r} did not match expected route {expected.get('route')!r}" + ], + } + if not isinstance(actual_tool, dict): + return { + "root_cause": "tool_call_error", + "reasons": ["actual tool must be a JSON object"], + } + if str(actual_tool.get("name", "")) != str(expected_tool.get("name", "")): + return { + "root_cause": "tool_call_error", + "reasons": [ + "actual tool " f"{actual_tool.get('name')!r} did not match expected tool {expected_tool.get('name')!r}" + ], + } + actual_arguments = actual_tool.get("arguments", _MISSING) + expected_arguments = expected_tool.get("arguments", _MISSING) + if not isinstance(expected_arguments, dict): + return { + "root_cause": "runtime_error", + "reasons": ["expected tool arguments must be a JSON object"], + } + if not isinstance(actual_arguments, dict): + return { + "root_cause": "parameter_error", + "reasons": ["actual tool arguments must be a JSON object"], + } + if actual_arguments != expected_arguments: + return { + "root_cause": "parameter_error", + "reasons": ["tool arguments did not match expected arguments"], + } + + failed_metric_names = _failed_metric_names(metrics) + if failed_metric_names: + root_cause, reason = _metric_failure_root(failed_metric_names) + return {"root_cause": root_cause, "reasons": [reason]} + return { + "root_cause": "metric_failed", + "reasons": ["case failed without a reported failed metric"], + } + + +def _status_name(status: Any) -> str: + return str(getattr(status, "name", status)).lower() + + +def _is_passed_status(status: Any) -> bool: + return _status_name(status) == "passed" + + +def _extract_actual_expected(run: Any, case: dict[str, Any]) -> tuple[str, str]: + actual_text = "" + expected_text = case_expected_text(case) + if run.eval_metric_result_per_invocation: + invocation = run.eval_metric_result_per_invocation[0] + actual_text = final_text_from_content(invocation.actual_invocation.final_response) + if invocation.expected_invocation is not None: + expected_text = final_text_from_content(invocation.expected_invocation.final_response) + return actual_text, expected_text + + +def summarize_evaluate_result(result: Any, evalset_payload: dict[str, Any]) -> dict[str, Any]: + case_by_id = {case["eval_id"]: case for case in evalset_payload["eval_cases"]} + eval_set_id, set_result = next(iter(result.results_by_eval_set_id.items())) + case_results: list[dict[str, Any]] = [] + + for case in evalset_payload["eval_cases"]: + eval_id = case["eval_id"] + runs = set_result.eval_results_by_eval_id.get(eval_id, []) + if not runs: + metrics: dict[str, dict[str, Any]] = {} + expected_text = case_expected_text(case) + error_message = "AgentEvaluator returned no run for case" + attribution = attribute_failure_case( + actual_text="", + expected_text=expected_text, + error_message=error_message, + metrics=metrics, + ) + case_results.append( + { + "case_id": eval_id, + "tags": case_tags(case), + "user": case_user_text(case), + "score": 0.0, + "passed": False, + "metrics": metrics, + "actual_text": "", + "expected_text": expected_text, + "key_trace": { + "invocation_id": str(case_by_id[eval_id]["conversation"][0].get("invocation_id", "")), + "actual_final_response": "", + "expected_final_response": expected_text, + "error_message": error_message, + }, + "root_cause": attribution["root_cause"], + "reasons": attribution["reasons"], + } + ) + continue + + run_scores: list[float] = [] + run_passed = True + metric_run_scores: dict[str, list[float]] = {} + metric_evidence: dict[str, dict[str, Any]] = {} + actual_text, expected_text = _extract_actual_expected(runs[0], case_by_id[eval_id]) + error_message = None + for run in runs: + run_metrics = list(run.overall_eval_metric_results or []) + run_passed = ( + run_passed + and _is_passed_status(run.final_eval_status) + and bool(run_metrics) + and all(_is_passed_status(metric.eval_status) for metric in run_metrics) + ) + if run.error_message and error_message is None: + error_message = sanitize_report_text(run.error_message) + for metric in run_metrics: + score = metric.score + metric_passed = _is_passed_status(metric.eval_status) + details = getattr(metric, "details", None) + reason = getattr(details, "reason", None) if details is not None else None + threshold = float(metric.threshold) + if score is not None: + metric_run_scores.setdefault(metric.metric_name, []).append(float(score)) + metric_evidence[metric.metric_name] = { + "score": None if score is None else float(score), + "threshold": threshold, + "status": _status_name(metric.eval_status), + "passed": metric_passed, + "reason": sanitize_report_text(reason), + } + if metric.metric_name == PRIMARY_METRIC and score is not None: + run_scores.append(float(score)) + + merged_metrics: dict[str, dict[str, Any]] = {} + for metric_name, evidence in metric_evidence.items(): + scores = metric_run_scores.get(metric_name, []) + if scores: + average_score = sum(scores) / len(scores) + threshold = evidence["threshold"] + passed = average_score >= threshold + merged_metrics[metric_name] = { + **evidence, + "score": round(average_score, 6), + "passed": passed, + "status": "passed" if passed else "failed", + } + else: + merged_metrics[metric_name] = evidence + + if not run_scores: + run_scores = [ + float(metric["score"]) for metric in merged_metrics.values() if metric.get("score") is not None + ] + score_value = sum(run_scores) / len(run_scores) if run_scores else (1.0 if run_passed else 0.0) + attribution = {"root_cause": "", "reasons": []} + if not run_passed: + attribution = attribute_failure_case( + actual_text=actual_text, + expected_text=expected_text, + error_message=error_message, + metrics=merged_metrics, + ) + case_results.append( + { + "case_id": eval_id, + "tags": case_tags(case), + "user": case_user_text(case), + "score": round(score_value, 6), + "passed": run_passed, + "metrics": merged_metrics, + "actual_text": actual_text, + "expected_text": expected_text, + "key_trace": { + "invocation_id": str(case_by_id[eval_id]["conversation"][0].get("invocation_id", "")), + "actual_final_response": actual_text, + "expected_final_response": expected_text, + "error_message": error_message, + }, + "root_cause": attribution["root_cause"], + "reasons": attribution["reasons"], + } + ) + + total = len(case_results) + score = sum(item["score"] for item in case_results) / total if total else 0.0 + pass_rate = sum(1 for item in case_results if item["passed"]) / total if total else 0.0 + metric_scores: dict[str, list[float]] = {} + metric_thresholds: dict[str, float] = {} + for case_result in case_results: + for metric_name, metric in case_result["metrics"].items(): + metric_score = _finite_float(metric.get("score")) + metric_threshold = _finite_float(metric.get("threshold")) + if metric_score is not None: + metric_scores.setdefault(metric_name, []).append(metric_score) + if metric_threshold is not None: + metric_thresholds[metric_name] = metric_threshold + metrics_summary: dict[str, dict[str, Any]] = {} + for name, scores in metric_scores.items(): + threshold = metric_thresholds.get(name, 1.0) + avg = sum(scores) / len(scores) if scores else 0.0 + metrics_summary[name] = { + "score": round(avg, 6), + "threshold": threshold, + "passed": avg >= threshold, + "status": "passed" if avg >= threshold else "failed", + } + + return { + "eval_set_id": eval_set_id, + "score": round(score, 6), + "pass_rate": round(pass_rate, 6), + "metrics": metrics_summary, + "case_results": case_results, + "failed_case_ids": [item["case_id"] for item in case_results if not item["passed"]], + "source": "AgentEvaluator", + } + + +def _json_safe(value: Any) -> Any: + if isinstance(value, float): + return value if math.isfinite(value) else None + if isinstance(value, dict): + return {str(key): _json_safe(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_json_safe(item) for item in value] + if isinstance(value, set): + return [_json_safe(item) for item in sorted(value, key=str)] + return value + + +def attribution_for(evaluation: dict[str, Any]) -> dict[str, Any]: + case_results = evaluation.get("case_results") + if not isinstance(case_results, list): + case_results = [] + failed = [case for case in case_results if isinstance(case, dict) and case.get("passed") is not True] + counts = Counter({name: 0 for name in TAXONOMY}) + cases = [] + for case in failed: + root = case.get("root_cause") or "runtime_error" + if root not in TAXONOMY: + root = "runtime_error" + counts[root] += 1 + reasons = case.get("reasons") or ["no failure reason recorded"] + cases.append( + { + "case_id": str(case.get("case_id", "")), + "root_cause": root, + "score": _finite_float(case.get("score")), + "reasons": reasons if isinstance(reasons, list) else [str(reasons)], + } + ) + covered = sum(1 for case in cases if case["reasons"]) + return { + "coverage": round(covered / len(failed), 6) if failed else 1.0, + "taxonomy_counts": {name: counts[name] for name in TAXONOMY}, + "cases": cases, + } + + +def _offline_rubric_score(actual_text: str) -> tuple[float, str]: + parsed = _json_or_none(actual_text) + if parsed is None: + return 0.0, "offline rubric failed: final response is not a JSON object" + if not parsed.get("route"): + return 0.0, "offline rubric failed: route is missing" + if not isinstance(parsed.get("tool"), dict): + return 0.0, "offline rubric failed: tool is missing or not an object" + reason = parsed.get("reason") + if not isinstance(reason, str) or not reason.strip(): + return 0.0, "offline rubric failed: reason is missing" + unsafe_terms = ("guarantee", "guaranteed", "promise", "approved refund", "instant credit") + if any(term in reason.lower() for term in unsafe_terms): + return 0.0, "offline rubric failed: reason makes an unsafe commitment" + return 1.0, "offline rubric passed" + + +def _install_offline_rubric_evaluator(): + from trpc_agent_sdk.evaluation._eval_metrics import EvalMetric + from trpc_agent_sdk.evaluation._eval_metrics import EvalStatus + from trpc_agent_sdk.evaluation._eval_result import EvaluationResult + from trpc_agent_sdk.evaluation._eval_result import PerInvocationResult + from trpc_agent_sdk.evaluation._evaluator_base import Evaluator + from trpc_agent_sdk.evaluation._evaluator_registry import EVALUATOR_REGISTRY + + previous = EVALUATOR_REGISTRY.get_evaluator_class(EvalMetric(metric_name=OFFLINE_RUBRIC_METRIC, threshold=1.0)) + + class OfflineRubricEvaluator(Evaluator): + requires_reference = False + + def __init__(self, eval_metric: Any | None = None) -> None: + self._threshold = float(getattr(eval_metric, "threshold", 1.0) or 1.0) + + async def evaluate_invocations(self, actual_invocations, expected_invocations): + per_invocation_results = [] + scores = [] + for actual in actual_invocations: + score, reason = _offline_rubric_score(final_text_from_content(actual.final_response)) + scores.append(score) + per_invocation_results.append( + PerInvocationResult( + actual_invocation=actual, + expected_invocation=None, + score=score, + eval_status=EvalStatus.PASSED if score >= self._threshold else EvalStatus.FAILED, + reason=reason, + ) + ) + overall = sum(scores) / len(scores) if scores else 0.0 + return EvaluationResult( + overall_score=overall, + overall_eval_status=EvalStatus.PASSED if overall >= self._threshold else EvalStatus.FAILED, + per_invocation_results=per_invocation_results, + ) + + EVALUATOR_REGISTRY.register(OFFLINE_RUBRIC_METRIC, OfflineRubricEvaluator) + return previous + + +async def run_evaluator( + *, + evalset_path: Path, + evalset_payload: dict[str, Any], + metrics_path: Path, + call_agent: Callable[[str], Awaitable[str]] | None = None, + offline_rubric: bool = False, +) -> dict[str, Any]: + from trpc_agent_sdk.evaluation import AgentEvaluator + from trpc_agent_sdk.evaluation._agent_evaluator import _EvaluationCasesFailed + from trpc_agent_sdk.evaluation._evaluator_registry import EVALUATOR_REGISTRY + + old_cwd = os.getcwd() + route_metric_state = _install_route_tool_args_metric() + previous_rubric_evaluator = _install_offline_rubric_evaluator() if offline_rubric else None + os.chdir(evalset_path.parent) + try: + executer = AgentEvaluator.get_executer( + evalset_path.name, + call_agent=call_agent, + eval_metrics_file_path_or_dir=str(metrics_path), + print_detailed_results=False, + print_summary_report=False, + ) + try: + await executer.evaluate() + except _EvaluationCasesFailed: + pass + result = executer.get_result() + if result is None: + raise RuntimeError(f"AgentEvaluator produced no result for {evalset_path}") + return summarize_evaluate_result(result, evalset_payload) + finally: + if previous_rubric_evaluator is not None: + EVALUATOR_REGISTRY.register(OFFLINE_RUBRIC_METRIC, previous_rubric_evaluator) + _restore_route_tool_args_metric(route_metric_state) + os.chdir(old_cwd) + + +def make_fixture_call_agent( + evalset_payload: dict[str, Any], + outputs: dict[str, str], +) -> Callable[[str], Awaitable[str]]: + query_to_output = {case_user_text(case): outputs.get(case["eval_id"], "") for case in evalset_payload["eval_cases"]} + + async def call_agent(query: str) -> str: + return query_to_output.get(query, "") + + return call_agent + + +def materialize_trace_evalset( + *, + source_evalset: Path, + payload: dict[str, Any], + outputs: dict[str, str], + run_dir: Path, + candidate_id: str, + split: str, +) -> tuple[Path, dict[str, Any]]: + _validate_safe_path_component(candidate_id, label="candidate_id") + trace_payload = copy.deepcopy(payload) + trace_payload["eval_set_id"] = f"{payload['eval_set_id']}_{candidate_id}_{split}_trace" + trace_payload["description"] = ( + f"Trace replay for {candidate_id} {split}, generated by eval_optimize_loop/run_pipeline.py" + ) + for case in trace_payload["eval_cases"]: + case["eval_mode"] = "trace" + expected_invocation = copy.deepcopy(case["conversation"][0]) + actual_invocation = copy.deepcopy(expected_invocation) + actual_invocation["final_response"] = { + "parts": [{"text": outputs.get(case["eval_id"], "")}], + "role": "model", + } + case["actual_conversation"] = [actual_invocation] + trace_name = f"{candidate_id}.{split}.trace.evalset.json" + _validate_safe_path_component(trace_name, label="trace artifact name") + path = _resolved_run_descendant( + run_dir, + "evalsets", + trace_name, + label="trace evalset artifact", + ) + write_json(path, trace_payload) + if candidate_id == "candidate_local_patch" and split == "validation": + trace_alias_path = _resolved_run_descendant( + run_dir, + "trace_evalset.json", + label="trace evalset alias artifact", + ) + write_json(trace_alias_path, trace_payload) + return path, trace_payload + + +async def evaluate_fixture_split( + *, + mode: str, + split: str, + candidate_id: str, + evalset_path: Path, + evalset_payload: dict[str, Any], + outputs: dict[str, str], + run_dir: Path, + metrics_path: Path, +) -> tuple[dict[str, Any], dict[str, str]]: + artifacts: dict[str, str] = {} + if mode == "trace": + trace_path, trace_payload = materialize_trace_evalset( + source_evalset=evalset_path, + payload=evalset_payload, + outputs=outputs, + run_dir=run_dir, + candidate_id=candidate_id, + split=split, + ) + artifacts[f"{split}_trace_evalset"] = str(trace_path) + summary = await run_evaluator( + evalset_path=trace_path, + evalset_payload=trace_payload, + metrics_path=metrics_path, + offline_rubric=True, + ) + return summary, artifacts + + call_agent = make_fixture_call_agent(evalset_payload, outputs) + summary = await run_evaluator( + evalset_path=evalset_path, + evalset_payload=evalset_payload, + metrics_path=metrics_path, + call_agent=call_agent, + offline_rubric=True, + ) + return summary, artifacts + + +def classify_case_delta(before: dict[str, Any], after: dict[str, Any]) -> str: + if not bool(before.get("passed")) and bool(after.get("passed")): + return "new_pass" + if bool(before.get("passed")) and not bool(after.get("passed")): + return "new_fail" + before_score = _finite_float(before.get("score")) + after_score = _finite_float(after.get("score")) + if before_score is None or after_score is None: + return "unchanged" + if after_score > before_score: + return "score_improved" + if after_score < before_score: + return "score_regressed" + return "unchanged" + + +def build_case_deltas(baseline_val: dict[str, Any], candidate_val: dict[str, Any]) -> list[dict[str, Any]]: + baseline_by_id, _ = _index_gate_cases(baseline_val) + candidate_by_id, _ = _index_gate_cases(candidate_val) + deltas = [] + for case_id in sorted(set(baseline_by_id) | set(candidate_by_id)): + before = baseline_by_id.get(case_id) + after = candidate_by_id.get(case_id) + baseline_score = None if before is None else _finite_float(before.get("score")) + candidate_score = None if after is None else _finite_float(after.get("score")) + delta = ( + None if baseline_score is None or candidate_score is None else round(candidate_score - baseline_score, 6) + ) + if before is None: + root_cause = "unexpected_candidate" + change_type = "unexpected_candidate" + reasons = ["candidate introduced an unknown validation case"] + elif after is None: + root_cause = "missing_candidate" + change_type = "missing_candidate" + reasons = ["candidate omitted a baseline validation case"] + else: + root_cause = after.get("root_cause", "") + change_type = classify_case_delta(before, after) + reasons = after.get("reasons", []) + deltas.append( + { + "case_id": case_id, + "baseline_score": baseline_score, + "candidate_score": candidate_score, + "baseline_passed": None if before is None else bool(before.get("passed")), + "candidate_passed": None if after is None else bool(after.get("passed")), + "delta": delta, + "change_type": change_type, + "baseline_actual_text": "" if before is None else before.get("actual_text", ""), + "candidate_actual_text": "" if after is None else after.get("actual_text", ""), + "root_cause": root_cause, + "reasons": reasons, + } + ) + return deltas + + +def _finite_float(value: Any) -> float | None: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + try: + parsed = float(value) + except (TypeError, ValueError): + return None + return parsed if math.isfinite(parsed) else None + + +def _score_delta(candidate: Any, baseline: Any) -> float | None: + candidate_score = _finite_float(candidate) + baseline_score = _finite_float(baseline) + if candidate_score is None or baseline_score is None: + return None + delta = candidate_score - baseline_score + return round(delta, 6) if math.isfinite(delta) else None + + +def _index_gate_cases( + evaluation: dict[str, Any], +) -> tuple[dict[str, dict[str, Any]], list[str]]: + cases = evaluation.get("case_results") + if not isinstance(cases, list): + return {}, ["case_results must be an array"] + indexed: dict[str, dict[str, Any]] = {} + issues: list[str] = [] + for position, case in enumerate(cases): + if not isinstance(case, dict): + issues.append(f"case_results[{position}] must be an object") + continue + case_id = case.get("case_id") + if not isinstance(case_id, str) or not case_id.strip(): + issues.append(f"case_results[{position}] case_id must be a non-empty string") + continue + if case_id in indexed: + issues.append(f"duplicate case_id: {case_id}") + continue + indexed[case_id] = case + return indexed, issues + + +def _case_derived_summary_score(evaluation: Mapping[str, Any]) -> float | None: + cases = evaluation.get("case_results") + if not isinstance(cases, list) or not cases: + return None + scores: list[float] = [] + for case in cases: + if not isinstance(case, Mapping): + return None + score = _finite_float(case.get("score")) + if score is None or not 0 <= score <= 1: + return None + scores.append(score) + return round(sum(scores) / len(scores), 6) + + +def _normalized_gate_tags(case: dict[str, Any]) -> set[str]: + tags = case.get("tags", []) + if not isinstance(tags, list): + return set() + return {str(tag).lower() for tag in tags} + + +def _normalize_required_metrics(value: Any, candidate_metrics: dict[str, Any]) -> tuple[list[str], str | None]: + if value == "all": + names = [name for name in candidate_metrics if isinstance(name, str) and name.strip()] + if not names or len(names) != len(candidate_metrics): + return [], "required_metrics='all' requires a non-empty object with string metric names" + return sorted(names), None + if not isinstance(value, list): + return [], "required_metrics must be 'all' or an array of non-empty unique strings" + if any(not isinstance(name, str) or not name.strip() for name in value): + return [], "required_metrics must contain only non-empty strings" + if len(set(value)) != len(value): + return [], "required_metrics must contain unique strings" + return value, None + + +def apply_gate( + *, + candidate_id: str, + baseline_val: Any, + candidate_val: Any, + gate_config: Any, + duration_seconds: float, + cost_usd: float | None, +) -> dict[str, Any]: + reasons: list[str] = [] + if isinstance(baseline_val, Mapping): + normalized_baseline = dict(baseline_val) + else: + normalized_baseline = {} + reasons.append("baseline_val must be a mapping") + if isinstance(candidate_val, Mapping): + normalized_candidate = dict(candidate_val) + else: + normalized_candidate = {} + reasons.append("candidate_val must be a mapping") + if isinstance(gate_config, Mapping): + normalized_gate_config = dict(gate_config) + else: + normalized_gate_config = {} + reasons.append("gate_config must be a mapping") + + unknown_gate_keys = sorted(str(key) for key in set(normalized_gate_config) - GATE_RUNTIME_KEYS) + if unknown_gate_keys: + reasons.append("unknown gate config field(s): " + ", ".join(unknown_gate_keys)) + + baseline_by_id, baseline_issues = _index_gate_cases(normalized_baseline) + candidate_by_id, candidate_issues = _index_gate_cases(normalized_candidate) + reasons.extend(baseline_issues) + reasons.extend(candidate_issues) + accepted = not reasons + + allow_new_hard_fails = normalized_gate_config.get("allow_new_hard_fails", False) + if not isinstance(allow_new_hard_fails, bool): + allow_new_hard_fails = False + accepted = False + reasons.append("allow_new_hard_fails must be a boolean and was treated as false") + allow_critical_regression = normalized_gate_config.get("allow_critical_regression", False) + if not isinstance(allow_critical_regression, bool): + allow_critical_regression = False + accepted = False + reasons.append("allow_critical_regression must be a boolean and was treated as false") + + for label, cases in (("baseline", baseline_by_id), ("candidate", candidate_by_id)): + for case_id, case in cases.items(): + score = _finite_float(case.get("score")) + if score is None or not 0 <= score <= 1: + accepted = False + reasons.append(f"{label} case {case_id} score must be a finite number in [0, 1]") + if not isinstance(case.get("passed"), bool): + accepted = False + reasons.append(f"{label} case {case_id} passed must be a boolean") + if not isinstance(case.get("tags", []), list): + accepted = False + reasons.append(f"{label} case {case_id} tags must be an array") + + for label, summary in (("baseline", normalized_baseline), ("candidate", normalized_candidate)): + expected_summary_score = _case_derived_summary_score(summary) + reported_summary_score = _finite_float(summary.get("score")) + if expected_summary_score is None: + accepted = False + reasons.append(f"{label} summary score could not be derived from non-empty valid case results") + elif reported_summary_score is not None and round(reported_summary_score, 6) != expected_summary_score: + accepted = False + reasons.append( + f"{label} summary score {reported_summary_score:.6f} does not match " + f"case-derived score {expected_summary_score:.6f}" + ) + + baseline_score = _finite_float(normalized_baseline.get("score")) + candidate_score = _finite_float(normalized_candidate.get("score")) + raw_validation_delta = ( + None if baseline_score is None or candidate_score is None else candidate_score - baseline_score + ) + validation_delta = ( + raw_validation_delta if raw_validation_delta is not None and math.isfinite(raw_validation_delta) else None + ) + if baseline_score is not None and not 0 <= baseline_score <= 1: + baseline_score = None + if candidate_score is not None and not 0 <= candidate_score <= 1: + candidate_score = None + if baseline_score is None or candidate_score is None: + validation_delta = None + if validation_delta is None: + accepted = False + reasons.append("baseline and candidate validation scores must be finite numbers") + min_delta = _finite_float(normalized_gate_config.get("min_validation_delta", 0.0)) + if min_delta is None or min_delta < 0: + accepted = False + reasons.append("minimum validation delta must be a finite non-negative number") + if validation_delta is not None and validation_delta <= 0: + accepted = False + reasons.append("validation score did not improve over baseline") + elif validation_delta is not None and min_delta is not None and min_delta >= 0 and validation_delta <= min_delta: + accepted = False + reasons.append( + "validation score improvement " f"{validation_delta:.4f} must be greater than required {min_delta:.4f}" + ) + + baseline_ids = set(baseline_by_id) + candidate_ids = set(candidate_by_id) + missing_case_ids = sorted(baseline_ids - candidate_ids) + unexpected_case_ids = sorted(candidate_ids - baseline_ids) + if missing_case_ids: + accepted = False + reasons.append("candidate omitted validation case(s): " + ", ".join(missing_case_ids)) + if unexpected_case_ids: + accepted = False + reasons.append("candidate introduced unknown validation case(s): " + ", ".join(unexpected_case_ids)) + + common_case_ids = sorted(baseline_ids & candidate_ids) + tag_mismatch_ids = [ + case_id + for case_id in common_case_ids + if _normalized_gate_tags(baseline_by_id[case_id]) != _normalized_gate_tags(candidate_by_id[case_id]) + ] + if tag_mismatch_ids: + accepted = False + reasons.append("baseline/candidate tag mismatch for validation case(s): " + ", ".join(tag_mismatch_ids)) + new_hard_fail_ids = [ + case_id + for case_id in common_case_ids + if baseline_by_id[case_id].get("passed") and not candidate_by_id[case_id].get("passed") + ] + if new_hard_fail_ids and not allow_new_hard_fails: + accepted = False + reasons.append("candidate introduced hard fail(s): " + ", ".join(new_hard_fail_ids)) + + critical_regression_ids = [ + case_id + for case_id in common_case_ids + if "critical" in _normalized_gate_tags(baseline_by_id[case_id]) + and _finite_float(candidate_by_id[case_id].get("score")) is not None + and _finite_float(baseline_by_id[case_id].get("score")) is not None + and _finite_float(candidate_by_id[case_id].get("score")) < _finite_float(baseline_by_id[case_id].get("score")) + ] + if critical_regression_ids and not allow_critical_regression: + accepted = False + reasons.append("candidate regressed critical case(s): " + ", ".join(critical_regression_ids)) + + normalized_cost = None if cost_usd is None else _finite_float(cost_usd) + if cost_usd is not None and (normalized_cost is None or normalized_cost < 0): + normalized_cost = None + accepted = False + reasons.append("run cost must be a finite non-negative number") + + max_cost = normalized_gate_config.get("max_cost_usd") + normalized_max_cost = None if max_cost is None else _finite_float(max_cost) + if max_cost is not None and (normalized_max_cost is None or normalized_max_cost < 0): + normalized_max_cost = None + accepted = False + reasons.append("cost budget must be a finite non-negative number") + elif max_cost is not None and cost_usd is None: + accepted = False + reasons.append("cost budget could not be evaluated because run cost is unknown") + elif max_cost is not None and normalized_cost is not None and normalized_cost > normalized_max_cost: + accepted = False + reasons.append(f"run exceeded cost budget: {normalized_cost:.4f} > {normalized_max_cost:.4f} USD") + + max_seconds = normalized_gate_config.get("max_duration_seconds") + normalized_duration = _finite_float(duration_seconds) + normalized_max_seconds = None if max_seconds is None else _finite_float(max_seconds) + if normalized_duration is None or normalized_duration < 0: + normalized_duration = None + accepted = False + reasons.append("run duration must be a finite non-negative number") + elif max_seconds is not None and (normalized_max_seconds is None or normalized_max_seconds < 0): + normalized_max_seconds = None + accepted = False + reasons.append("duration budget must be a finite non-negative number") + elif max_seconds is not None and normalized_duration > normalized_max_seconds: + accepted = False + reasons.append(f"run exceeded duration budget: {normalized_duration:.2f}s > {normalized_max_seconds:.2f}s") + + candidate_metrics = normalized_candidate.get("metrics") + if not isinstance(candidate_metrics, dict): + candidate_metrics = {} + accepted = False + reasons.append("candidate metrics must be an object") + required, required_issue = _normalize_required_metrics( + normalized_gate_config.get("required_metrics", []), + candidate_metrics, + ) + if required_issue: + accepted = False + reasons.append(required_issue) + missing_or_failed = [] + for name in required: + metric = candidate_metrics.get(name) + metric_consistent = isinstance(metric, dict) and metric.get("passed") is True + if metric_consistent and ("score" in metric or "threshold" in metric): + metric_score = _finite_float(metric.get("score")) + metric_threshold = _finite_float(metric.get("threshold")) + metric_consistent = ( + metric_score is not None + and 0 <= metric_score <= 1 + and metric_threshold is not None + and metric_threshold >= 0 + and metric_score >= metric_threshold + ) + if metric_consistent and "status" in metric: + metric_consistent = str(metric.get("status", "")).lower() == "passed" + if not metric_consistent: + missing_or_failed.append(name) + if missing_or_failed: + accepted = False + reasons.append("required metric(s) missing or failing: " + ", ".join(missing_or_failed)) + + if accepted: + reasons.append("validation improved and all configured gates passed") + + return { + "candidate_id": candidate_id, + "accepted": accepted, + "reasons": reasons, + "new_hard_fail_ids": new_hard_fail_ids, + "critical_regression_ids": critical_regression_ids, + "missing_case_ids": missing_case_ids, + "unexpected_case_ids": unexpected_case_ids, + "validation_delta": None if validation_delta is None else round(validation_delta, 6), + } + + +def gate_candidate( + *, + candidate_id: str, + baseline_val: dict[str, Any], + candidate_val: dict[str, Any], + duration_seconds: float, + max_seconds: float, +) -> dict[str, Any]: + gate_config = copy.deepcopy(DEFAULT_GATE_CONFIG) + gate_config["max_duration_seconds"] = max_seconds + return apply_gate( + candidate_id=candidate_id, + baseline_val=baseline_val, + candidate_val=candidate_val, + gate_config=gate_config, + duration_seconds=duration_seconds, + cost_usd=0.0, + ) + + +def build_candidate_report( + *, + candidate_id: str, + fixture: dict[str, Any], + train: dict[str, Any], + optimizer_dev: dict[str, Any], + validation: dict[str, Any], + baseline_train: dict[str, Any], + baseline_optimizer_dev: dict[str, Any], + baseline_val: dict[str, Any], + gate_config: dict[str, Any], + duration_seconds: float, + gate_duration_seconds: float | None = None, + cost_usd: float | None, + seed: int, + optimizer_config: Path, + prompt_artifacts: list[dict[str, Any]] | None = None, + artifacts: dict[str, str] | None = None, +) -> dict[str, Any]: + observed_gate_duration = round( + duration_seconds if gate_duration_seconds is None else gate_duration_seconds, + 6, + ) + gate = apply_gate( + candidate_id=candidate_id, + baseline_val=baseline_val, + candidate_val=validation, + gate_config=gate_config, + duration_seconds=observed_gate_duration, + cost_usd=cost_usd, + ) + return _json_safe( + { + "id": candidate_id, + "audit": build_candidate_audit( + seed=seed, + duration_seconds=duration_seconds, + cost_usd=cost_usd, + optimizer_config=optimizer_config, + ), + "prompt_patch_summary": fixture.get("prompt_patch_summary", ""), + "prompt_artifacts": prompt_artifacts or [], + "train": _json_safe(train), + "optimizer_dev": _json_safe(optimizer_dev), + "final_validation": _json_safe(validation), + "validation": _json_safe(validation), + "delta": { + "train_score": _score_delta(train.get("score"), baseline_train.get("score")), + "optimizer_dev_score": _score_delta(optimizer_dev.get("score"), baseline_optimizer_dev.get("score")), + "validation_score": _score_delta(validation.get("score"), baseline_val.get("score")), + }, + "case_deltas": build_case_deltas(baseline_val, validation), + "gate": gate, + "failure_attribution": attribution_for(validation), + "artifacts": artifacts or {}, + } + ) + + +def pick_winner(candidates: list[dict[str, Any]]) -> dict[str, Any] | None: + accepted = [candidate for candidate in candidates if candidate["gate"]["accepted"]] + if not accepted: + return None + return max( + accepted, + key=lambda candidate: ( + candidate["validation"]["score"], + candidate["train"]["score"], + candidate["id"], + ), + ) + + +def common_artifacts( + *, + run_dir: Path, + train_evalset: Path, + optimizer_dev_evalset: Path, + val_evalset: Path, + optimizer_config: Path, + fixture_path: Path, + metrics_path: Path, + system_prompt: Path, + router_prompt: Path, +) -> dict[str, str]: + return { + "optimization_report_json": str(run_dir / "optimization_report.json"), + "optimization_report_md": str(run_dir / "optimization_report.md"), + "train_evalset": str(train_evalset), + "optimizer_dev_evalset": str(optimizer_dev_evalset), + "validation_evalset": str(val_evalset), + "final_validation_evalset": str(val_evalset), + "optimizer_config": str(optimizer_config), + "fixtures": str(fixture_path), + "eval_metrics": str(metrics_path), + "system_prompt": str(system_prompt), + "router_prompt": str(router_prompt), + } + + +def existing_artifact(path: Path) -> str: + return str(path) if path.exists() else "" + + +def build_top_level_report( + *, + mode: str, + run_id: str, + run_dir: Path, + seed: int, + baseline_fixture: dict[str, Any], + baseline_train: dict[str, Any], + baseline_optimizer_dev: dict[str, Any], + baseline_val: dict[str, Any], + baseline_prompt_artifacts: list[dict[str, Any]], + candidates: list[dict[str, Any]], + gate_config: dict[str, Any], + artifacts: dict[str, Any], + cost: dict[str, Any], + duration_seconds: float, + config_snapshot: dict[str, Any], + command: str | None = None, + extra: dict[str, Any] | None = None, +) -> dict[str, Any]: + winner = pick_winner(candidates) + if winner is None: + rejection_reasons = ["no candidate passed all gates"] + for candidate in candidates: + for reason in candidate["gate"]["reasons"]: + rejection_reasons.append(f"{candidate['id']}: {reason}") + gate_decision = { + "accepted": False, + "winner": None, + "reasons": rejection_reasons, + } + delta = { + "validation_score": 0.0, + "optimizer_dev_score": 0.0, + "train_score": 0.0, + } + else: + gate_decision = { + "accepted": True, + "winner": winner["id"], + "reasons": winner["gate"]["reasons"], + } + delta = winner["delta"] + + report = { + "run_id": run_id, + "mode": mode, + "seed": seed, + "baseline": { + "prompt_patch_summary": baseline_fixture.get("prompt_patch_summary", ""), + "prompt_artifacts": baseline_prompt_artifacts, + "train": baseline_train, + "optimizer_dev": baseline_optimizer_dev, + "final_validation": baseline_val, + "validation": baseline_val, + }, + "candidates": candidates, + "delta": delta, + "gate_decision": gate_decision, + "failure_attribution": attribution_for(baseline_val), + "cost": cost, + "duration_seconds": round(duration_seconds, 6), + "optimization_rounds": [], + "config_snapshot": config_snapshot, + "environment_snapshot": environment_snapshot( + seed=seed, + command=command, + config_path=str((config_snapshot.get("paths") or {}).get("optimizer_config", "")), + ), + "artifacts": artifacts, + } + if extra: + report.update(extra) + return report + + +def make_report( + *, + mode: str, + run_id: str, + run_dir: Path, + seed: int, + started: float, + extra_artifacts: dict[str, str] | None = None, + command: str | None = None, +) -> dict[str, Any]: + report = asyncio.run( + build_offline_report( + mode=mode, + run_id=run_id, + run_dir=run_dir, + seed=seed, + started=started, + train_evalset=TRAIN_PATH.resolve(), + optimizer_dev_evalset=OPTIMIZER_DEV_PATH.resolve(), + val_evalset=VAL_PATH.resolve(), + optimizer_config=OPTIMIZER_CONFIG_PATH.resolve(), + fixture_path=(TRACE_FIXTURE_PATH if mode == "trace" else FIXTURE_PATH).resolve(), + gate_config=load_gate_config(optimizer_config=OPTIMIZER_CONFIG_PATH.resolve()), + system_prompt=SYSTEM_PROMPT_PATH.resolve(), + router_prompt=ROUTER_PROMPT_PATH.resolve(), + command=command, + ) + ) + if extra_artifacts: + report["artifacts"].update(extra_artifacts) + return report + + +async def build_offline_report( + *, + mode: str, + run_id: str, + run_dir: Path, + seed: int, + started: float, + train_evalset: Path, + optimizer_dev_evalset: Path, + val_evalset: Path, + optimizer_config: Path, + fixture_path: Path, + gate_config: dict[str, Any], + system_prompt: Path, + router_prompt: Path, + command: str | None = None, +) -> dict[str, Any]: + train_payload = load_json(train_evalset) + optimizer_dev_payload = load_json(optimizer_dev_evalset) + val_payload = load_json(val_evalset) + fixtures = load_json(fixture_path) + for candidate_id in fixtures: + _validate_safe_path_component(candidate_id, label="candidate_id") + metrics_path = offline_metrics_path(run_dir) + source_prompts = read_source_prompts(system_prompt, router_prompt) + if mode == "trace": + trace_metrics_path = _resolved_run_descendant( + run_dir, + "trace_metrics.json", + label="trace metrics artifact", + ) + write_json(trace_metrics_path, OFFLINE_METRICS_CONFIG) + + baseline_fixture = fixtures["baseline"] + baseline_train, baseline_train_artifacts = await evaluate_fixture_split( + mode=mode, + split="train", + candidate_id="baseline", + evalset_path=train_evalset, + evalset_payload=train_payload, + outputs=baseline_fixture["outputs"], + run_dir=run_dir, + metrics_path=metrics_path, + ) + baseline_val, baseline_val_artifacts = await evaluate_fixture_split( + mode=mode, + split="validation", + candidate_id="baseline", + evalset_path=val_evalset, + evalset_payload=val_payload, + outputs=baseline_fixture["outputs"], + run_dir=run_dir, + metrics_path=metrics_path, + ) + baseline_optimizer_dev, baseline_optimizer_dev_artifacts = await evaluate_fixture_split( + mode=mode, + split="optimizer_dev", + candidate_id="baseline", + evalset_path=optimizer_dev_evalset, + evalset_payload=optimizer_dev_payload, + outputs=baseline_fixture["outputs"], + run_dir=run_dir, + metrics_path=metrics_path, + ) + baseline_prompt_artifacts, baseline_prompt_paths = write_prompt_artifacts( + run_dir=run_dir, + candidate_id="baseline", + source_prompts=source_prompts, + candidate_prompts=offline_candidate_prompts( + source_prompts, + "baseline", + baseline_fixture.get("prompt_patch_summary", ""), + ), + summary=baseline_fixture.get("prompt_patch_summary", ""), + source_written=False, + ) + + candidates: list[dict[str, Any]] = [] + for candidate_id, fixture in fixtures.items(): + if candidate_id == "baseline": + continue + candidate_started = time.perf_counter() + train, train_artifacts = await evaluate_fixture_split( + mode=mode, + split="train", + candidate_id=candidate_id, + evalset_path=train_evalset, + evalset_payload=train_payload, + outputs=fixture["outputs"], + run_dir=run_dir, + metrics_path=metrics_path, + ) + optimizer_dev, optimizer_dev_artifacts = await evaluate_fixture_split( + mode=mode, + split="optimizer_dev", + candidate_id=candidate_id, + evalset_path=optimizer_dev_evalset, + evalset_payload=optimizer_dev_payload, + outputs=fixture["outputs"], + run_dir=run_dir, + metrics_path=metrics_path, + ) + validation, val_artifacts = await evaluate_fixture_split( + mode=mode, + split="validation", + candidate_id=candidate_id, + evalset_path=val_evalset, + evalset_payload=val_payload, + outputs=fixture["outputs"], + run_dir=run_dir, + metrics_path=metrics_path, + ) + candidate_artifacts = {} + candidate_artifacts.update(train_artifacts) + candidate_artifacts.update(optimizer_dev_artifacts) + candidate_artifacts.update(val_artifacts) + prompt_artifacts, prompt_paths = write_prompt_artifacts( + run_dir=run_dir, + candidate_id=candidate_id, + source_prompts=source_prompts, + candidate_prompts=offline_candidate_prompts( + source_prompts, + candidate_id, + fixture.get("prompt_patch_summary", ""), + ), + summary=fixture.get("prompt_patch_summary", ""), + source_written=False, + ) + candidate_artifacts.update(prompt_paths) + candidate_duration_seconds = time.perf_counter() - candidate_started + candidates.append( + build_candidate_report( + candidate_id=candidate_id, + fixture=fixture, + train=train, + optimizer_dev=optimizer_dev, + validation=validation, + baseline_train=baseline_train, + baseline_optimizer_dev=baseline_optimizer_dev, + baseline_val=baseline_val, + gate_config=gate_config, + duration_seconds=candidate_duration_seconds, + cost_usd=0.0, + seed=seed, + optimizer_config=optimizer_config, + prompt_artifacts=prompt_artifacts, + artifacts=candidate_artifacts, + ) + ) + + artifacts = common_artifacts( + run_dir=run_dir, + train_evalset=train_evalset, + optimizer_dev_evalset=optimizer_dev_evalset, + val_evalset=val_evalset, + optimizer_config=optimizer_config, + fixture_path=fixture_path, + metrics_path=metrics_path, + system_prompt=system_prompt, + router_prompt=router_prompt, + ) + if mode == "trace": + artifacts.update( + { + "trace_evalset": str(run_dir / "trace_evalset.json"), + "trace_metrics": str(run_dir / "trace_metrics.json"), + } + ) + artifacts.update( + { + "baseline_train_trace_evalset": baseline_train_artifacts.get("train_trace_evalset", ""), + "baseline_optimizer_dev_trace_evalset": baseline_optimizer_dev_artifacts.get( + "optimizer_dev_trace_evalset", "" + ), + "baseline_validation_trace_evalset": baseline_val_artifacts.get("validation_trace_evalset", ""), + "baseline_prompt_dir": baseline_prompt_paths.get("prompt_dir", ""), + "baseline_prompt_patch": baseline_prompt_paths.get("prompt_patch", ""), + } + ) + evaluation_snapshot = normalized_evaluation_config(OFFLINE_METRICS_CONFIG) + + return build_top_level_report( + mode=mode, + run_id=run_id, + run_dir=run_dir, + seed=seed, + baseline_fixture=baseline_fixture, + baseline_train=baseline_train, + baseline_optimizer_dev=baseline_optimizer_dev, + baseline_val=baseline_val, + baseline_prompt_artifacts=baseline_prompt_artifacts, + candidates=candidates, + gate_config=gate_config, + artifacts=artifacts, + cost={ + "currency": "USD", + "estimated_total": 0.0, + "cost_source": "deterministic_offline", + "unknown_cost_reason": None, + "model_calls": 0, + "token_usage": { + "prompt": 0, + "completion": 0, + "total": 0, + }, + "token_usage_known": True, + "unknown_token_usage_reason": None, + "optimizer": { + "estimated_cost": 0.0, + "model_calls": 0, + "candidate_evaluation_agent_calls": 0, + "reflection_lm_calls": 0, + "judge_calls_per_candidate_evaluation": 0, + "judge_model_calls": 0, + "native_judge_model_calls": 0, + "derived_judge_model_calls": 0, + "judge_model_call_source": "none", + "token_usage": { + "prompt": 0, + "completion": 0, + "total": 0, + }, + "token_usage_known": True, + "unknown_token_usage_reason": None, + "usage_evidence_valid": True, + "reflection_reported_usage": { + "estimated_cost": 0.0, + "token_usage": { + "prompt": 0, + "completion": 0, + "total": 0, + }, + "token_usage_known": True, + "unknown_token_usage_reason": None, + }, + }, + "final_revalidation": { + "estimated_cost": 0.0, + "agent_calls_per_run": 0, + "agent_calls": 0, + "judge_calls_per_agent_call": 0, + "judge_model_calls": 0, + "model_calls": 0, + "token_usage": { + "prompt": 0, + "completion": 0, + "total": 0, + }, + "token_usage_known": True, + "unknown_token_usage_reason": None, + }, + }, + duration_seconds=time.perf_counter() - started, + config_snapshot={ + "mode": mode, + "seed": seed, + "gate": gate_config, + "evaluation": evaluation_snapshot, + "evaluation_sha256": sha256_json(evaluation_snapshot), + "evaluation_metrics_sha256": sha256_json_file(metrics_path), + "optimizer_config_sha256": sha256_json_file(optimizer_config), + "prompt_targets": build_prompt_target_manifest(source_prompts), + "evalsets": build_evalset_manifests(train_evalset, optimizer_dev_evalset, val_evalset), + "paths": { + "train_evalset": str(train_evalset), + "optimizer_dev_evalset": str(optimizer_dev_evalset), + "validation_evalset": str(val_evalset), + "final_validation_evalset": str(val_evalset), + "optimizer_config": str(optimizer_config), + "evaluation_metrics": str(metrics_path), + "fixture_outputs": str(fixture_path), + "system_prompt": str(system_prompt), + "router_prompt": str(router_prompt), + }, + }, + command=command, + ) + + +def _make_llm_agent_from_prompts(prompt_texts: dict[str, str]): + from trpc_agent_sdk.agents import LlmAgent + from trpc_agent_sdk.models import OpenAIModel + + from agent.config import get_model_config + + api_key, base_url, model_name = get_model_config() + instruction = "\n\n".join( + [ + prompt_texts.get("system_prompt", "").strip(), + prompt_texts.get("router_prompt", "").strip(), + ] + ) + return LlmAgent( + name="support_router_agent", + model=OpenAIModel( + model_name=model_name, + api_key=api_key, + base_url=base_url, + ), + instruction=instruction, + ) + + +def make_online_call_agent( + *, + system_prompt: Path, + router_prompt: Path, + prompt_texts: dict[str, str] | None = None, +) -> Callable[[str], Awaitable[str]]: + async def call_agent(query: str) -> str: + from trpc_agent_sdk.runners import Runner + from trpc_agent_sdk.sessions import InMemorySessionService + from trpc_agent_sdk.types import Content + from trpc_agent_sdk.types import Part + + prompts = prompt_texts or { + "system_prompt": system_prompt.read_text(encoding="utf-8"), + "router_prompt": router_prompt.read_text(encoding="utf-8"), + } + root_agent = _make_llm_agent_from_prompts(prompts) + session_service = InMemorySessionService() + runner = Runner( + app_name="support_router_optimizer", + agent=root_agent, + session_service=session_service, + ) + user_id = "optimizer" + session_id = str(uuid.uuid4()) + try: + await session_service.create_session( + app_name="support_router_optimizer", + user_id=user_id, + session_id=session_id, + state={}, + ) + final = "" + async for event in runner.run_async( + user_id=user_id, + session_id=session_id, + new_message=Content(role="user", parts=[Part.from_text(text=query)]), + ): + if not event.is_final_response() or not event.content: + continue + for part in event.content.parts or []: + if part.thought: + continue + if part.text: + final += part.text + except BaseException: + try: + await runner.close() + except BaseException: + logger.exception("Failed to close online evaluation runner after a primary error.") + raise + else: + await runner.close() + return final.strip() + + return call_agent + + +async def online_call_agent(query: str) -> str: + return await make_online_call_agent( + system_prompt=SYSTEM_PROMPT_PATH, + router_prompt=ROUTER_PROMPT_PATH, + )(query) + + +def _optimizer_fixture(result: Any) -> dict[str, str]: + return {"prompt_patch_summary": "Best prompt returned by AgentOptimizer.optimize(update_source=False)."} + + +def _optimizer_extra(result: Any) -> dict[str, Any]: + return { + "online_result": { + "status": result.status, + "error_message": sanitize_report_text(getattr(result, "error_message", "")), + "baseline_pass_rate": result.baseline_pass_rate, + "best_pass_rate": result.best_pass_rate, + "pass_rate_improvement": result.pass_rate_improvement, + "stop_reason": result.stop_reason, + "baseline_metric_breakdown": getattr(result, "baseline_metric_breakdown", {}), + "best_metric_breakdown": getattr(result, "best_metric_breakdown", {}), + } + } + + +def _count_attr(obj: Any, name: str) -> tuple[int, bool]: + value = getattr(obj, name, 0) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + return 0, True + return value, False + + +def _is_llm_metric(metric: dict[str, Any]) -> bool: + name = str(metric.get("metric_name") or metric.get("metricName") or "") + criterion = metric.get("criterion") or {} + return name.startswith("llm_") or "llm_judge" in criterion or "llmJudge" in criterion + + +def judge_calls_per_agent_call(metrics_config: Mapping[str, Any]) -> int: + from trpc_agent_sdk.evaluation._eval_config import EvalConfig + from trpc_agent_sdk.evaluation._llm_criterion import get_llm_criterion_from_metric + + try: + eval_config = EvalConfig.model_validate(dict(metrics_config)) + except Exception as error: + raise ValueError(f"invalid evaluation metrics config: {error}") from error + + total = 0 + for metric in eval_config.get_eval_metrics(): + criterion = get_llm_criterion_from_metric(metric) + if criterion is not None: + judge_models = criterion.get_judge_models() + if not judge_models: + raise ValueError(f"LLM metric {metric.metric_name} has no judge model") + total += sum(model.get_num_samples() for model in judge_models) + elif str(metric.metric_name).startswith("llm_"): + total += 1 + return total + + +def final_revalidation_call_audit( + summaries: list[dict[str, Any]], + metrics_config: dict[str, Any], + *, + evalset_payloads: list[dict[str, Any]] | None = None, +) -> dict[str, int]: + num_runs, invalid_num_runs = _normalized_round_count(metrics_config.get("num_runs", 1)) + if invalid_num_runs or num_runs <= 0: + raise ValueError("evaluation num_runs must be a positive integer") + if evalset_payloads is None: + agent_calls_per_run = sum(len(summary.get("case_results", [])) for summary in summaries) + else: + if len(evalset_payloads) != len(summaries): + raise ValueError("final revalidation payload count must match summary count") + agent_calls_per_run = 0 + for payload in evalset_payloads: + cases = payload.get("eval_cases") if isinstance(payload, dict) else None + if not isinstance(cases, list): + raise ValueError("final revalidation evalset payload must contain eval_cases") + for case in cases: + conversation = case.get("conversation") if isinstance(case, dict) else None + if not isinstance(conversation, list) or not conversation: + raise ValueError("final revalidation case must contain a non-empty conversation") + agent_calls_per_run += len(conversation) + case_runs = agent_calls_per_run * num_runs + judge_multiplier = judge_calls_per_agent_call(metrics_config) + judge_calls = case_runs * judge_multiplier + return { + "agent_calls_per_run": agent_calls_per_run, + "agent_calls": case_runs, + "judge_calls_per_agent_call": judge_multiplier, + "judge_model_calls": judge_calls, + "model_calls": case_runs + judge_calls, + } + + +def online_cost_audit( + result: Any, + *, + optimizer_candidate_agent_calls: int, + final_revalidation_calls: dict[str, int], + optimizer_judge_calls_per_agent_call: int | None = None, + optimizer_llm_metric_count: int | None = None, +) -> dict[str, Any]: + reflection_calls, invalid_reflection_calls = _count_attr(result, "total_reflection_lm_calls") + native_judge_calls, invalid_judge_calls = _count_attr(result, "total_judge_model_calls") + candidate_calls, invalid_candidate_calls = _normalized_round_count(optimizer_candidate_agent_calls) + legacy_multiplier_conflict = False + if optimizer_judge_calls_per_agent_call is None: + optimizer_judge_calls_per_agent_call = optimizer_llm_metric_count or 0 + elif optimizer_llm_metric_count is not None and optimizer_llm_metric_count != optimizer_judge_calls_per_agent_call: + legacy_multiplier_conflict = True + judge_multiplier, invalid_judge_multiplier = _normalized_round_count(optimizer_judge_calls_per_agent_call) + derived_judge_calls = candidate_calls * judge_multiplier + judge_calls = max(native_judge_calls, derived_judge_calls) + if native_judge_calls and derived_judge_calls: + judge_call_source = ( + "native_and_derived_agree" + if native_judge_calls == derived_judge_calls + else "reconciled_native_and_derived_max" + ) + elif native_judge_calls: + judge_call_source = "native_optimizer_counter" + elif derived_judge_calls: + judge_call_source = "derived_from_candidate_calls_and_llm_metrics" + else: + judge_call_source = "none" + optimizer_calls = candidate_calls + reflection_calls + judge_calls + token_usage, invalid_token_usage = _normalized_token_usage(getattr(result, "total_token_usage", {})) + raw_cost = getattr(result, "total_llm_cost", None) + reflection_cost = _finite_float(raw_cost) + invalid_cost = reflection_cost is None or reflection_cost < 0 + if invalid_cost: + reflection_cost = None + + malformed_native_usage = ( + invalid_reflection_calls + or invalid_judge_calls + or invalid_candidate_calls + or invalid_judge_multiplier + or legacy_multiplier_conflict + or invalid_token_usage + or invalid_cost + ) + phase_usage_known = not malformed_native_usage and candidate_calls == 0 and judge_calls == 0 + optimizer_cost = reflection_cost if phase_usage_known else None + optimizer_token_usage = token_usage if phase_usage_known else None + + total_cost = optimizer_cost + unknown_reasons: list[str] = [] + if candidate_calls > 0: + unknown_reasons.append("optimizer candidate-evaluation calls do not expose token or cost usage") + if judge_calls > 0: + unknown_reasons.append("optimizer judge calls do not expose token or cost usage") + if malformed_native_usage: + unknown_reasons.append("optimizer native usage counters were malformed and normalized fail-closed") + if final_revalidation_calls["model_calls"] > 0: + unknown_reasons.append("final revalidation model calls are not provider-priced by AgentEvaluator") + total_cost = None + + final_tokens_known = final_revalidation_calls["model_calls"] == 0 + final_token_usage = {name: 0 for name in TOKEN_USAGE_KEYS} if final_tokens_known else None + pipeline_tokens_known = phase_usage_known and final_tokens_known + pipeline_token_usage = optimizer_token_usage if pipeline_tokens_known else None + optimizer_token_unknown_reasons = [] + if candidate_calls > 0: + optimizer_token_unknown_reasons.append("optimizer candidate-evaluation token usage is not exposed") + if judge_calls > 0: + optimizer_token_unknown_reasons.append("optimizer judge token usage is not exposed") + if malformed_native_usage: + optimizer_token_unknown_reasons.append("optimizer native usage counters were malformed") + token_unknown_reasons = list(optimizer_token_unknown_reasons) + if not final_tokens_known: + token_unknown_reasons.append("AgentEvaluator does not expose final revalidation token usage") + + return { + "currency": "USD", + "estimated_total": total_cost, + "cost_source": "unknown" if unknown_reasons else "optimizer_result", + "unknown_cost_reason": "; ".join(unknown_reasons) if unknown_reasons else None, + "model_calls": optimizer_calls + final_revalidation_calls["model_calls"], + "token_usage": pipeline_token_usage, + "token_usage_known": pipeline_tokens_known, + "unknown_token_usage_reason": "; ".join(token_unknown_reasons) if token_unknown_reasons else None, + "optimizer": { + "estimated_cost": optimizer_cost, + "model_calls": optimizer_calls, + "candidate_evaluation_agent_calls": candidate_calls, + "reflection_lm_calls": reflection_calls, + "judge_calls_per_candidate_evaluation": judge_multiplier, + "judge_model_calls": judge_calls, + "native_judge_model_calls": native_judge_calls, + "derived_judge_model_calls": derived_judge_calls, + "judge_model_call_source": judge_call_source, + "token_usage": optimizer_token_usage, + "token_usage_known": phase_usage_known, + "unknown_token_usage_reason": (None if phase_usage_known else "; ".join(optimizer_token_unknown_reasons)), + "usage_evidence_valid": not malformed_native_usage, + "reflection_reported_usage": { + "estimated_cost": reflection_cost, + "token_usage": token_usage, + "token_usage_known": not invalid_token_usage, + "unknown_token_usage_reason": ( + "malformed optimizer-reported reflection token usage" if invalid_token_usage else None + ), + }, + }, + "final_revalidation": { + "estimated_cost": 0.0 if final_revalidation_calls["model_calls"] == 0 else None, + **final_revalidation_calls, + "token_usage": final_token_usage, + "token_usage_known": final_tokens_known, + "unknown_token_usage_reason": ( + None if final_tokens_known else "AgentEvaluator does not expose token usage" + ), + }, + } + + +async def run_fake_or_trace( + *, + mode: str, + seed: int, + output_dir: Path | None, + run_id: str | None, + train_evalset: Path | None = None, + optimizer_dev_evalset: Path | None = None, + val_evalset: Path | None = None, + optimizer_config: Path | None = None, + fixture_outputs: Path | None = None, + gate_config_path: Path | None = None, + gate_config: dict[str, Any] | None = None, + system_prompt: Path | None = None, + router_prompt: Path | None = None, + command: str | None = None, +) -> Path: + started = time.perf_counter() + actual_run_id = run_id or datetime.now().strftime("%Y-%m-%dT%H-%M-%S") + run_dir = make_run_dir(output_dir, actual_run_id) + train_path = resolve_path(train_evalset, TRAIN_PATH) + optimizer_dev_path = resolve_path(optimizer_dev_evalset, OPTIMIZER_DEV_PATH) + val_path = resolve_path(val_evalset, VAL_PATH) + optimizer_path = resolve_path(optimizer_config, OPTIMIZER_CONFIG_PATH) + fixture_default = TRACE_FIXTURE_PATH if mode == "trace" else FIXTURE_PATH + fixture_path = resolve_path(fixture_outputs, fixture_default) + system_path = resolve_path(system_prompt, SYSTEM_PROMPT_PATH) + router_path = resolve_path(router_prompt, ROUTER_PROMPT_PATH) + optimizer_evaluate_config = validated_optimizer_evaluate_config(optimizer_path) + validate_inputs( + train_path, + optimizer_dev_path, + val_path, + metrics_config=optimizer_evaluate_config, + ) + gate = load_gate_config(gate_config_path, gate_config, optimizer_config=optimizer_path) + + report = await build_offline_report( + mode=mode, + run_id=actual_run_id, + run_dir=run_dir, + seed=seed, + started=started, + train_evalset=train_path, + optimizer_dev_evalset=optimizer_dev_path, + val_evalset=val_path, + optimizer_config=optimizer_path, + fixture_path=fixture_path, + gate_config=gate, + system_prompt=system_path, + router_prompt=router_path, + command=command, + ) + write_report(run_dir, report) + return run_dir + + +async def run_online( + *, + seed: int, + output_dir: Path | None, + run_id: str | None, + train_evalset: Path | None = None, + optimizer_dev_evalset: Path | None = None, + val_evalset: Path | None = None, + optimizer_config: Path | None = None, + gate_config_path: Path | None = None, + gate_config: dict[str, Any] | None = None, + system_prompt: Path | None = None, + router_prompt: Path | None = None, + command: str | None = None, +) -> Path: + from agent.config import get_model_config + + preflight = require_online_preflight() + print(format_online_preflight(preflight)) + get_model_config() + from trpc_agent_sdk.evaluation import AgentOptimizer + from trpc_agent_sdk.evaluation import TargetPrompt + + started = time.perf_counter() + actual_run_id = run_id or datetime.now().strftime("%Y-%m-%dT%H-%M-%S") + run_dir = make_run_dir(output_dir, actual_run_id) + train_path = resolve_path(train_evalset, TRAIN_PATH) + optimizer_dev_path = resolve_path(optimizer_dev_evalset, OPTIMIZER_DEV_PATH) + val_path = resolve_path(val_evalset, VAL_PATH) + optimizer_path = resolve_path(optimizer_config, OPTIMIZER_CONFIG_PATH) + system_path = resolve_path(system_prompt, SYSTEM_PROMPT_PATH) + router_path = resolve_path(router_prompt, ROUTER_PROMPT_PATH) + source_evaluate_config = validated_optimizer_evaluate_config(optimizer_path) + validate_inputs( + train_path, + optimizer_dev_path, + val_path, + metrics_config=source_evaluate_config, + ) + runtime_optimizer_path = materialize_optimizer_config( + run_dir=run_dir, + source_config=optimizer_path, + seed=seed, + ) + optimizer_evaluate_config = validated_optimizer_evaluate_config(runtime_optimizer_path) + optimizer_judge_multiplier = judge_calls_per_agent_call(optimizer_evaluate_config) + gate = load_gate_config(gate_config_path, gate_config, optimizer_config=runtime_optimizer_path) + source_prompts = read_source_prompts(system_path, router_path) + + online_dir = _resolved_run_descendant( + run_dir, + "online", + label="online optimizer artifact directory", + ) + if online_dir.exists(): + raise ValueError("online optimizer artifact directory must not already exist") + online_dir.mkdir() + target = TargetPrompt().add_path("system_prompt", str(system_path)).add_path("router_prompt", str(router_path)) + optimizer_candidate_agent_calls = 0 + optimizer_call_agent = make_online_call_agent(system_prompt=system_path, router_prompt=router_path) + + async def counted_optimizer_call_agent(query: str) -> str: + nonlocal optimizer_candidate_agent_calls + optimizer_candidate_agent_calls += 1 + return await optimizer_call_agent(query) + + route_metric_state = _install_route_tool_args_metric() + try: + result = await AgentOptimizer.optimize( + config_path=str(runtime_optimizer_path), + call_agent=counted_optimizer_call_agent, + target_prompt=target, + train_dataset_path=str(train_path), + validation_dataset_path=str(optimizer_dev_path), + output_dir=str(online_dir), + update_source=False, + verbose=0, + ) + finally: + _restore_route_tool_args_metric(route_metric_state) + optimization_finished = time.perf_counter() + + train_payload = load_json(train_path) + optimizer_dev_payload = load_json(optimizer_dev_path) + val_payload = load_json(val_path) + metrics_path = online_metrics_path(run_dir, runtime_optimizer_path) + source_prompt_texts = {name: text for name, (_, text) in source_prompts.items()} + best_prompt_texts = { + **source_prompt_texts, + **dict(getattr(result, "best_prompts", {}) or {}), + } + baseline_call_agent = make_online_call_agent(system_prompt=system_path, router_prompt=router_path) + best_call_agent = make_online_call_agent( + system_prompt=system_path, + router_prompt=router_path, + prompt_texts=best_prompt_texts, + ) + baseline_train = await run_evaluator( + evalset_path=train_path, + evalset_payload=train_payload, + metrics_path=metrics_path, + call_agent=baseline_call_agent, + ) + baseline_val = await run_evaluator( + evalset_path=val_path, + evalset_payload=val_payload, + metrics_path=metrics_path, + call_agent=baseline_call_agent, + ) + baseline_optimizer_dev = await run_evaluator( + evalset_path=optimizer_dev_path, + evalset_payload=optimizer_dev_payload, + metrics_path=metrics_path, + call_agent=baseline_call_agent, + ) + baseline_finished = time.perf_counter() + candidate_started = baseline_finished + best_train = await run_evaluator( + evalset_path=train_path, + evalset_payload=train_payload, + metrics_path=metrics_path, + call_agent=best_call_agent, + ) + best_optimizer_dev = await run_evaluator( + evalset_path=optimizer_dev_path, + evalset_payload=optimizer_dev_payload, + metrics_path=metrics_path, + call_agent=best_call_agent, + ) + best_val = await run_evaluator( + evalset_path=val_path, + evalset_payload=val_payload, + metrics_path=metrics_path, + call_agent=best_call_agent, + ) + candidate_finished = time.perf_counter() + candidate_duration_seconds = candidate_finished - candidate_started + gate_elapsed_seconds = candidate_finished - started + + metrics_config = load_json(metrics_path) + cost = online_cost_audit( + result, + optimizer_candidate_agent_calls=optimizer_candidate_agent_calls, + optimizer_judge_calls_per_agent_call=optimizer_judge_multiplier, + final_revalidation_calls=final_revalidation_call_audit( + [ + baseline_train, + baseline_optimizer_dev, + baseline_val, + best_train, + best_optimizer_dev, + best_val, + ], + metrics_config, + evalset_payloads=[ + train_payload, + optimizer_dev_payload, + val_payload, + train_payload, + optimizer_dev_payload, + val_payload, + ], + ), + ) + cost_usd = cost["estimated_total"] + baseline_prompt_artifacts, baseline_prompt_paths = write_prompt_artifacts( + run_dir=run_dir, + candidate_id="baseline", + source_prompts=source_prompts, + candidate_prompts={name: text for name, (_, text) in source_prompts.items()}, + summary="Source prompts before AgentOptimizer.optimize.", + source_written=False, + ) + best_prompt_artifacts, best_prompt_paths = write_prompt_artifacts( + run_dir=run_dir, + candidate_id="optimizer_best", + source_prompts=source_prompts, + candidate_prompts=best_prompt_texts, + summary="Best prompt returned by AgentOptimizer.optimize(update_source=False).", + source_written=False, + ) + optimization_rounds = write_optimizer_round_artifacts( + run_dir=run_dir, + rounds=list(getattr(result, "rounds", []) or []), + ) + candidate = build_candidate_report( + candidate_id="optimizer_best", + fixture=_optimizer_fixture(result), + train=best_train, + optimizer_dev=best_optimizer_dev, + validation=best_val, + baseline_train=baseline_train, + baseline_optimizer_dev=baseline_optimizer_dev, + baseline_val=baseline_val, + gate_config=gate, + duration_seconds=candidate_duration_seconds, + gate_duration_seconds=gate_elapsed_seconds, + cost_usd=cost_usd, + seed=seed, + optimizer_config=runtime_optimizer_path, + prompt_artifacts=best_prompt_artifacts, + artifacts={ + "native_optimizer_dir": existing_artifact(online_dir), + "native_result_json": existing_artifact(online_dir / "result.json"), + "native_summary_txt": existing_artifact(online_dir / "summary.txt"), + "native_rounds_dir": existing_artifact(online_dir / "rounds"), + "native_baseline_prompts_dir": existing_artifact(online_dir / "baseline_prompts"), + "native_best_prompts_dir": existing_artifact(online_dir / "best_prompts"), + "native_best_prompts": existing_artifact(online_dir / "best_prompts"), + "native_config_snapshot_json": existing_artifact(online_dir / "config.snapshot.json"), + **best_prompt_paths, + }, + ) + if result.status != "SUCCEEDED": + candidate["gate"]["accepted"] = False + candidate["gate"]["reasons"].append(f"native optimizer status was {result.status}") + error_message = sanitize_report_text(getattr(result, "error_message", "")) + if error_message: + candidate["gate"]["reasons"].append(error_message) + if not cost["optimizer"]["usage_evidence_valid"]: + candidate["gate"]["accepted"] = False + candidate["gate"]["reasons"].append("optimizer usage evidence was malformed and cannot be audited") + + artifacts = common_artifacts( + run_dir=run_dir, + train_evalset=train_path, + optimizer_dev_evalset=optimizer_dev_path, + val_evalset=val_path, + optimizer_config=runtime_optimizer_path, + fixture_path=FIXTURE_PATH, + metrics_path=metrics_path, + system_prompt=system_path, + router_prompt=router_path, + ) + artifacts.update(candidate["artifacts"]) + artifacts.update( + { + "online_eval_metrics": str(metrics_path), + "optimizer_source_config": str(optimizer_path), + "optimizer_runtime_config": str(runtime_optimizer_path), + "baseline_prompt_dir": baseline_prompt_paths.get("prompt_dir", ""), + "baseline_prompt_patch": baseline_prompt_paths.get("prompt_patch", ""), + } + ) + evaluation_snapshot = normalized_evaluation_config(optimizer_evaluate_config) + report = build_top_level_report( + mode="online", + run_id=actual_run_id, + run_dir=run_dir, + seed=seed, + baseline_fixture={"prompt_patch_summary": "Source prompts before AgentOptimizer.optimize."}, + baseline_train=baseline_train, + baseline_optimizer_dev=baseline_optimizer_dev, + baseline_val=baseline_val, + baseline_prompt_artifacts=baseline_prompt_artifacts, + candidates=[candidate], + gate_config=gate, + artifacts=artifacts, + cost=cost, + duration_seconds=time.perf_counter() - started, + config_snapshot={ + "mode": "online", + "seed": seed, + "gate": gate, + "evaluation": evaluation_snapshot, + "evaluation_sha256": sha256_json(evaluation_snapshot), + "evaluation_metrics_sha256": sha256_json_file(metrics_path), + "optimizer_config_sha256": sha256_json_file(runtime_optimizer_path), + "prompt_targets": build_prompt_target_manifest(source_prompts), + "evalsets": build_evalset_manifests(train_path, optimizer_dev_path, val_path), + "paths": { + "train_evalset": str(train_path), + "optimizer_dev_evalset": str(optimizer_dev_path), + "validation_evalset": str(val_path), + "final_validation_evalset": str(val_path), + "optimizer_config": str(runtime_optimizer_path), + "optimizer_source_config": str(optimizer_path), + "evaluation_metrics": str(metrics_path), + "online_eval_metrics": str(metrics_path), + "system_prompt": str(system_path), + "router_prompt": str(router_path), + }, + }, + command=command, + extra={ + **_optimizer_extra(result), + "online_preflight": preflight, + "online_duration": { + "optimization_seconds": round(optimization_finished - started, 6), + "baseline_revalidation_seconds": round(baseline_finished - optimization_finished, 6), + "candidate_revalidation_seconds": round(candidate_duration_seconds, 6), + "gate_elapsed_seconds": round(gate_elapsed_seconds, 6), + }, + "optimization_rounds": optimization_rounds, + }, + ) + write_report(run_dir, report) + return run_dir + + +def _format_delta_number(value: Any, *, signed: bool = False) -> str: + parsed = _finite_float(value) + if parsed is None: + return "n/a" + return f"{parsed:+.2f}" if signed else f"{parsed:.2f}" + + +def render_markdown(report: dict[str, Any]) -> str: + baseline_train = report["baseline"]["train"]["score"] + baseline_optimizer_dev = report["baseline"]["optimizer_dev"]["score"] + baseline_val = report["baseline"]["validation"]["score"] + winner_id = report["gate_decision"]["winner"] + winner = next((candidate for candidate in report["candidates"] if candidate["id"] == winner_id), None) + winner_score = winner["validation"]["score"] if winner else baseline_val + + lines = [ + "# Optimization Report", + "", + f"- Run: `{report['run_id']}`", + f"- Mode: `{report['mode']}`", + f"- Seed: `{report['seed']}`", + f"- Baseline train score: `{baseline_train:.4f}`", + f"- Baseline optimizer_dev score: `{baseline_optimizer_dev:.4f}`", + f"- Baseline validation score: `{baseline_val:.4f}`", + f"- Winner: `{winner_id or 'none'}`", + f"- Winner validation score: `{winner_score:.4f}`", + f"- Gate accepted: `{str(report['gate_decision']['accepted']).lower()}`", + f"- Cost: `{report['cost'].get('estimated_total')}` `{report['cost'].get('currency')}`", + f"- Duration seconds: `{report['duration_seconds']:.4f}`", + "", + "## Gate Reasons", + "", + ] + for reason in report["gate_decision"]["reasons"]: + lines.append(f"- {reason}") + + lines.extend(["", "## Candidate Summary", ""]) + for candidate in report["candidates"]: + lines.append( + "- `{id}` train `{train:.4f}` optimizer_dev `{optimizer_dev:.4f}` " + "final_validation `{val:.4f}` accepted `{accepted}`".format( + id=candidate["id"], + train=candidate["train"]["score"], + optimizer_dev=candidate["optimizer_dev"]["score"], + val=candidate["validation"]["score"], + accepted=str(candidate["gate"]["accepted"]).lower(), + ) + ) + + for candidate in sorted(report["candidates"], key=lambda item: str(item["id"])): + lines.extend(["", f"## Validation Case Delta: `{candidate['id']}`", ""]) + for item in candidate["case_deltas"]: + lines.append( + "- `{case_id}`: `{baseline_score}` -> `{candidate_score}` " + "delta `{delta}` change_type `{change_type}`".format( + case_id=item["case_id"], + baseline_score=_format_delta_number(item.get("baseline_score")), + candidate_score=_format_delta_number(item.get("candidate_score")), + delta=_format_delta_number(item.get("delta"), signed=True), + change_type=item["change_type"], + ) + ) + + lines.extend(["", "## Failure Attribution", ""]) + attribution = report["failure_attribution"] + for name, count in attribution["taxonomy_counts"].items(): + lines.append(f"- `{name}`: `{count}`") + + lines.extend(["", "## Artifacts", ""]) + for name, path in sorted(report["artifacts"].items()): + if path: + lines.append(f"- `{name}`: `{path}`") + + return "\n".join(lines) + "\n" + + +def write_report(run_dir: Path, report: dict[str, Any]) -> None: + validate_report_schema(report) + json_path = _resolved_run_descendant( + run_dir, + "optimization_report.json", + label="JSON report artifact", + ) + markdown_path = _resolved_run_descendant( + run_dir, + "optimization_report.md", + label="Markdown report artifact", + ) + write_json(json_path, report) + markdown_path.write_text(render_markdown(report), encoding="utf-8") + + +async def amain(argv: list[str] | None = None) -> Path: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--mode", choices=("fake", "trace", "online"), default="fake") + parser.add_argument("--seed", type=int, default=DEFAULT_SEED) + parser.add_argument("--output-dir", type=Path, default=None) + parser.add_argument("--run-id", default=None) + parser.add_argument("--train-evalset", type=Path, default=TRAIN_PATH) + parser.add_argument("--optimizer-dev-evalset", type=Path, default=OPTIMIZER_DEV_PATH) + parser.add_argument("--val-evalset", type=Path, default=VAL_PATH) + parser.add_argument("--optimizer-config", type=Path, default=OPTIMIZER_CONFIG_PATH) + parser.add_argument("--fixture-outputs", type=Path, default=None) + parser.add_argument("--gate-config", type=Path, default=None) + parser.add_argument("--system-prompt", type=Path, default=SYSTEM_PROMPT_PATH) + parser.add_argument("--router-prompt", type=Path, default=ROUTER_PROMPT_PATH) + args = parser.parse_args(argv) + command_args = sys.argv[1:] if argv is None else argv + command = " ".join([sys.executable, str(Path(__file__).resolve()), *command_args]) + + if args.mode in {"fake", "trace"}: + run_dir = await run_fake_or_trace( + mode=args.mode, + seed=args.seed, + output_dir=args.output_dir, + run_id=args.run_id, + train_evalset=args.train_evalset, + optimizer_dev_evalset=args.optimizer_dev_evalset, + val_evalset=args.val_evalset, + optimizer_config=args.optimizer_config, + fixture_outputs=args.fixture_outputs, + gate_config_path=args.gate_config, + system_prompt=args.system_prompt, + router_prompt=args.router_prompt, + command=command, + ) + else: + run_dir = await run_online( + seed=args.seed, + output_dir=args.output_dir, + run_id=args.run_id, + train_evalset=args.train_evalset, + optimizer_dev_evalset=args.optimizer_dev_evalset, + val_evalset=args.val_evalset, + optimizer_config=args.optimizer_config, + gate_config_path=args.gate_config, + system_prompt=args.system_prompt, + router_prompt=args.router_prompt, + command=command, + ) + print(run_dir) + return run_dir + + +def main(argv: list[str] | None = None) -> Path: + return asyncio.run(amain(argv)) + + +if __name__ == "__main__": + main() diff --git a/examples/optimization/eval_optimize_loop/train.evalset.json b/examples/optimization/eval_optimize_loop/train.evalset.json new file mode 100644 index 00000000..5b84b0df --- /dev/null +++ b/examples/optimization/eval_optimize_loop/train.evalset.json @@ -0,0 +1,109 @@ +{ + "eval_set_id": "support_router_train", + "name": "Support router train set", + "description": "Three train cases used by fake and online optimization. They cover successful optimization, escalation, and a no-change FAQ guardrail.", + "eval_cases": [ + { + "eval_id": "train_refund_001", + "conversation": [ + { + "invocation_id": "train-1", + "user_content": { + "parts": [ + { + "text": "The headphones arrived broken yesterday. I want a refund." + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "text": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Broken delivered item needs refund handling.\"}" + } + ], + "role": "model" + } + } + ], + "session_input": { + "app_name": "support_router_optimizer", + "user_id": "trainer", + "state": { + "tags": [ + "refund", + "train" + ] + } + } + }, + { + "eval_id": "train_manual_002", + "conversation": [ + { + "invocation_id": "train-2", + "user_content": { + "parts": [ + { + "text": "My account is locked and I am going to report this unless a person helps me now." + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "text": "{\"route\":\"manual_escalation\",\"tool\":{\"name\":\"create_escalation_case\",\"arguments\":{}},\"reason\":\"Account lock with escalation demand needs a human case.\"}" + } + ], + "role": "model" + } + } + ], + "session_input": { + "app_name": "support_router_optimizer", + "user_id": "trainer", + "state": { + "tags": [ + "manual", + "train" + ] + } + } + }, + { + "eval_id": "train_faq_003", + "conversation": [ + { + "invocation_id": "train-3", + "user_content": { + "parts": [ + { + "text": "Can an expired coupon be reissued?" + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Coupon policy question can be answered by FAQ.\"}" + } + ], + "role": "model" + } + } + ], + "session_input": { + "app_name": "support_router_optimizer", + "user_id": "trainer", + "state": { + "tags": [ + "faq", + "train" + ] + } + } + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/val.evalset.json b/examples/optimization/eval_optimize_loop/val.evalset.json new file mode 100644 index 00000000..7421b4e0 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/val.evalset.json @@ -0,0 +1,110 @@ +{ + "eval_set_id": "support_router_val", + "name": "Support router validation set", + "description": "Three held-out validation cases. The shipping-delay case is critical because over-routing angry but ordinary logistics questions to manual support is a regression.", + "eval_cases": [ + { + "eval_id": "val_refund_window_101", + "conversation": [ + { + "invocation_id": "val-1", + "user_content": { + "parts": [ + { + "text": "The blender arrived missing parts and I need a refund even though it has been 10 days." + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "text": "{\"route\":\"refund\",\"tool\":{\"name\":\"create_refund_ticket\",\"arguments\":{}},\"reason\":\"Missing parts with refund request should open refund handling.\"}" + } + ], + "role": "model" + } + } + ], + "session_input": { + "app_name": "support_router_optimizer", + "user_id": "validator", + "state": { + "tags": [ + "refund", + "validation" + ] + } + } + }, + { + "eval_id": "val_address_change_102", + "conversation": [ + { + "invocation_id": "val-2", + "user_content": { + "parts": [ + { + "text": "Can I change the delivery address for an order that has not shipped?" + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Address-change policy question can be answered by FAQ.\"}" + } + ], + "role": "model" + } + } + ], + "session_input": { + "app_name": "support_router_optimizer", + "user_id": "validator", + "state": { + "tags": [ + "faq", + "validation" + ] + } + } + }, + { + "eval_id": "val_shipping_delay_103", + "conversation": [ + { + "invocation_id": "val-3", + "user_content": { + "parts": [ + { + "text": "I am angry that shipping is late, but I only want to know the delivery policy." + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "text": "{\"route\":\"faq\",\"tool\":{\"name\":\"none\",\"arguments\":{}},\"reason\":\"Angry wording with a policy question should stay in FAQ.\"}" + } + ], + "role": "model" + } + } + ], + "session_input": { + "app_name": "support_router_optimizer", + "user_id": "validator", + "state": { + "tags": [ + "faq", + "validation", + "critical" + ] + } + } + } + ] +} diff --git a/pyproject.toml b/pyproject.toml index 6c47d4d5..897d2d57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,6 +89,7 @@ agent-claude = [ eval = [ "pytest", "pytest-asyncio", + "jsonschema>=4.0.0", "rouge-score", "pandas", "tabulate", @@ -123,6 +124,7 @@ dev = [ "numpy>=2.2.5", "pytest", "pytest-asyncio", + "jsonschema>=4.0.0", "black", "flake8", "yapf", @@ -144,6 +146,7 @@ all = [ "claude-agent-sdk>=0.1.3,<0.1.64", "pytest", "pytest-asyncio", + "jsonschema>=4.0.0", "rouge-score", "pandas", "tabulate", diff --git a/tests/agents/core/test_llm_processor.py b/tests/agents/core/test_llm_processor.py index 5779bea6..b226b2e6 100644 --- a/tests/agents/core/test_llm_processor.py +++ b/tests/agents/core/test_llm_processor.py @@ -12,6 +12,7 @@ from unittest.mock import Mock import pytest +from unittest.mock import patch from trpc_agent_sdk.agents._base_agent import BaseAgent from trpc_agent_sdk.agents.core._llm_processor import LlmProcessor @@ -23,6 +24,8 @@ class _StubAgent(BaseAgent): + instruction: str = "" + async def _run_async_impl(self, ctx): yield @@ -159,6 +162,34 @@ def test_event_without_content_skips_planning(self, model, invocation_context): class TestCallLlmAsync: + + @pytest.mark.asyncio + async def test_non_streaming_drains_model_before_yielding_event(self, invocation_context): + """A non-streaming event is exposed only after the model generator is closed.""" + + class ClosingModel(MockLLMModel): + + def __init__(self): + super().__init__(model_name="test-llmproc-model") + self.finished = False + + async def _generate_async_impl(self, request, stream=False, ctx=None): + try: + yield LlmResponse(content=Content(parts=[Part(text="complete")])) + finally: + self.finished = True + + model = ClosingModel() + processor = LlmProcessor(model) + event_stream = processor.call_llm_async(LlmRequest(), invocation_context, stream=False) + + with patch("trpc_agent_sdk.agents.core._llm_processor.trace_call_llm"): + event = await anext(event_stream) + + assert event.content.parts[0].text == "complete" + assert model.finished is True + await event_stream.aclose() + def test_yields_events_for_responses(self, model, invocation_context): proc = LlmProcessor(model) request = LlmRequest() diff --git a/tests/agents/test_llm_agent_ext.py b/tests/agents/test_llm_agent_ext.py index 5dc3ad32..e2c3f5ba 100644 --- a/tests/agents/test_llm_agent_ext.py +++ b/tests/agents/test_llm_agent_ext.py @@ -409,5 +409,37 @@ async def run(): assert events[0].error_code == "build_error" +class TestRunAsyncImplStreaming: + + def test_disables_model_stream_when_run_config_disables_streaming(self, invocation_context): + """A non-streaming run must not create a provider SSE response.""" + + class RecordingModel(MockLLMModel): + + def __init__(self): + super().__init__(model_name="test-llm-ext-model") + self.stream_args: list[bool] = [] + + async def _generate_async_impl(self, request, stream=False, ctx=None): + self.stream_args.append(stream) + yield LlmResponse(content=Content(parts=[Part(text="recorded")])) + + model = RecordingModel() + agent = _agent(model=model) + invocation_context.agent = agent + invocation_context.run_config = RunConfig(streaming=False) + invocation_context.override_messages = [ + Content(role="user", parts=[Part(text="evaluate this response")]), + ] + + async def run(): + async for _ in agent._run_async_impl(invocation_context): + pass + + asyncio.run(run()) + + assert model.stream_args == [False] + + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/tests/evaluation/test_agent_evaluator.py b/tests/evaluation/test_agent_evaluator.py index 8609ccc5..11c329b2 100644 --- a/tests/evaluation/test_agent_evaluator.py +++ b/tests/evaluation/test_agent_evaluator.py @@ -5,12 +5,15 @@ # tRPC-Agent-Python is licensed under Apache-2.0. """Unit tests for agent evaluator (agent_evaluator).""" +import json + import pytest import trpc_agent_sdk.runners # noqa: F401 from trpc_agent_sdk.evaluation import EvalStatus from trpc_agent_sdk.evaluation import EvalCaseResult +from trpc_agent_sdk.evaluation import EvalConfig from trpc_agent_sdk.evaluation import EvalMetricResult from trpc_agent_sdk.evaluation import EvalSetAggregateResult from trpc_agent_sdk.evaluation import EvaluateResult @@ -97,3 +100,55 @@ def test_pass_at_k_delegates(self): def test_pass_hat_k_delegates(self): """Test AgentEvaluator.pass_hat_k delegates to _eval_pass.""" assert AgentEvaluator.pass_hat_k(10, 5, 2) == 0.25 + + +class TestAgentEvaluatorLoadEvalSet: + """Test suite for loading eval sets from ADK-style path selectors.""" + + def test_loads_selected_case_from_json_path_suffix(self, tmp_path): + eval_set_path = tmp_path / "cases.evalset.json" + eval_set_path.write_text( + json.dumps({ + "eval_set_id": "selector_set", + "name": "Selector set", + "eval_cases": [ + { + "eval_id": "case_1", + "conversation": [{ + "invocation_id": "invocation_1", + "user_content": { + "role": "user", + "parts": [{"text": "first"}], + }, + "final_response": { + "role": "model", + "parts": [{"text": "first response"}], + }, + }], + }, + { + "eval_id": "case_2", + "conversation": [{ + "invocation_id": "invocation_2", + "user_content": { + "role": "user", + "parts": [{"text": "second"}], + }, + "final_response": { + "role": "model", + "parts": [{"text": "second response"}], + }, + }], + }, + ], + }), + encoding="utf-8", + ) + + eval_set = AgentEvaluator._load_eval_set_from_file( + f"{eval_set_path}:case_2", + EvalConfig(), + ) + + assert eval_set.eval_set_id == "selector_set_case_2" + assert [case.eval_id for case in eval_set.eval_cases] == ["case_2"] diff --git a/tests/evaluation/test_eval_optimize_loop_example.py b/tests/evaluation/test_eval_optimize_loop_example.py new file mode 100644 index 00000000..cfa0c04b --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop_example.py @@ -0,0 +1,4695 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Tests for examples/optimization/eval_optimize_loop.""" + +from __future__ import annotations + +import copy +import importlib.util +import inspect +import json +import os +import re +import shutil +import subprocess +import sys +import warnings +from pathlib import Path +from types import SimpleNamespace +from typing import Any +from unittest.mock import patch + +import pytest +from jsonschema import ValidationError + +EXAMPLE_DIR = Path(__file__).resolve().parents[2] / "examples" / "optimization" / "eval_optimize_loop" +RUN_PIPELINE = EXAMPLE_DIR / "run_pipeline.py" +REPORT_SCHEMA = EXAMPLE_DIR / "optimization_report.schema.json" +ROUTE_TOOL_ARGS_METRIC = "route_tool_args_score" + + +def _gate_summary( + score: float, + cases: list[dict[str, Any]], + *, + metric_passed: bool = True, +) -> dict[str, Any]: + return { + "score": score, + "metrics": {ROUTE_TOOL_ARGS_METRIC: {"passed": metric_passed}}, + "case_results": cases, + } + + +def _complete_summary(score: float, *, passed: bool, tags: list[str] | None = None) -> dict[str, Any]: + return { + "eval_set_id": "stubbed_online", + "score": score, + "pass_rate": 1.0 if passed else 0.0, + "metrics": { + ROUTE_TOOL_ARGS_METRIC: { + "score": score, + "threshold": 1.0, + "passed": passed, + "status": "passed" if passed else "failed", + } + }, + "case_results": [ + { + "case_id": "case_1", + "tags": tags or [], + "user": "stubbed user", + "score": score, + "passed": passed, + "metrics": { + ROUTE_TOOL_ARGS_METRIC: { + "score": score, + "threshold": 1.0, + "passed": passed, + "status": "passed" if passed else "failed", + } + }, + "actual_text": "", + "expected_text": "", + "key_trace": { + "invocation_id": "case_1", + "actual_final_response": "", + "expected_final_response": "", + "error_message": None, + }, + "root_cause": "" if passed else "final_response_mismatch", + "reasons": [] if passed else ["stubbed mismatch"], + } + ], + "failed_case_ids": [] if passed else ["case_1"], + "source": "AgentEvaluator", + } + + +def _optimizer_result(token_usage: Any) -> SimpleNamespace: + return SimpleNamespace( + status="SUCCEEDED", + error_message="", + baseline_pass_rate=0.0, + best_pass_rate=1.0, + pass_rate_improvement=1.0, + stop_reason="completed", + total_llm_cost=0.01, + total_reflection_lm_calls=1, + total_judge_model_calls=0, + total_token_usage=token_usage, + best_prompts={"system_prompt": "better system", "router_prompt": "better router"}, + baseline_prompts={"system_prompt": "baseline system", "router_prompt": "baseline router"}, + baseline_metric_breakdown={ROUTE_TOOL_ARGS_METRIC: 0.0}, + best_metric_breakdown={ROUTE_TOOL_ARGS_METRIC: 1.0}, + rounds=[], + ) + + +async def _run_stubbed_online( + *, + module: Any, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + result: SimpleNamespace, + run_id: str, + gate_config: dict[str, Any] | None = None, + optimizer_agent_calls: int = 0, + optimizer_config: Path | None = None, +) -> dict[str, Any]: + monkeypatch.setenv("TRPC_AGENT_API_KEY", "fake-key") + monkeypatch.setenv("TRPC_AGENT_BASE_URL", "http://localhost/fake") + monkeypatch.setenv("TRPC_AGENT_MODEL_NAME", "fake-model") + + async def fake_optimize(**kwargs: Any) -> SimpleNamespace: + for _ in range(optimizer_agent_calls): + await kwargs["call_agent"]("optimizer candidate") + output_dir = Path(kwargs["output_dir"]) + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "result.json").write_text("{}", encoding="utf-8") + return result + + summaries = iter( + [ + _complete_summary(0.0, passed=False), + _complete_summary(0.0, passed=False), + _complete_summary(0.0, passed=False), + _complete_summary(1.0, passed=True), + _complete_summary(1.0, passed=True), + _complete_summary(1.0, passed=True), + ] + ) + + async def fake_run_evaluator(**_: Any) -> dict[str, Any]: + return next(summaries) + + import trpc_agent_sdk.evaluation as evaluation_pkg + + if optimizer_agent_calls: + + async def fake_call_agent(_: str) -> str: + return "{}" + + monkeypatch.setattr(module, "make_online_call_agent", lambda **_: fake_call_agent) + monkeypatch.setattr(evaluation_pkg.AgentOptimizer, "optimize", staticmethod(fake_optimize)) + monkeypatch.setattr(module, "run_evaluator", fake_run_evaluator) + run_dir = await module.run_online( + seed=7, + output_dir=tmp_path, + run_id=run_id, + gate_config=gate_config, + optimizer_config=optimizer_config, + ) + return load_report(run_dir / "optimization_report.json") + + +def _write_optimizer_config(tmp_path: Path, *, metrics: list[dict[str, Any]]) -> Path: + payload = load_report(EXAMPLE_DIR / "optimizer.json") + payload["evaluate"]["metrics"] = metrics + path = tmp_path / "optimizer.json" + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + +def load_pipeline_module() -> Any: + spec = importlib.util.spec_from_file_location("eval_optimize_loop_run_pipeline", RUN_PIPELINE) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def test_gate_fails_closed_for_boundary_and_invalid_evidence(): + module = load_pipeline_module() + baseline = _gate_summary( + 0.25, + [ + {"case_id": "a", "score": 0.0, "passed": False, "tags": []}, + {"case_id": "b", "score": 0.5, "passed": True, "tags": ["critical"]}, + ], + ) + valid_candidate = _gate_summary( + 0.75, + [ + {"case_id": "a", "score": 1.0, "passed": True, "tags": []}, + {"case_id": "b", "score": 0.5, "passed": True, "tags": ["critical"]}, + ], + ) + + exact_boundary = module.apply_gate( + candidate_id="boundary", + baseline_val=baseline, + candidate_val=valid_candidate, + gate_config={ + "min_validation_delta": 0.5, + "required_metrics": [ROUTE_TOOL_ARGS_METRIC], + }, + duration_seconds=1.0, + cost_usd=0.0, + ) + assert exact_boundary["accepted"] is False + + missing_case = copy.deepcopy(valid_candidate) + missing_case["case_results"] = missing_case["case_results"][:1] + missing = module.apply_gate( + candidate_id="missing", + baseline_val=baseline, + candidate_val=missing_case, + gate_config={"required_metrics": [ROUTE_TOOL_ARGS_METRIC]}, + duration_seconds=1.0, + cost_usd=0.0, + ) + assert missing["accepted"] is False + assert missing["missing_case_ids"] == ["b"] + + extra_case = copy.deepcopy(valid_candidate) + extra_case["case_results"].append({"case_id": "c", "score": 1.0, "passed": True, "tags": []}) + extra = module.apply_gate( + candidate_id="extra", + baseline_val=baseline, + candidate_val=extra_case, + gate_config={"required_metrics": [ROUTE_TOOL_ARGS_METRIC]}, + duration_seconds=1.0, + cost_usd=0.0, + ) + assert extra["accepted"] is False + assert extra["unexpected_case_ids"] == ["c"] + + non_finite = copy.deepcopy(valid_candidate) + non_finite["score"] = float("nan") + invalid = module.apply_gate( + candidate_id="nan", + baseline_val=baseline, + candidate_val=non_finite, + gate_config={"required_metrics": [ROUTE_TOOL_ARGS_METRIC]}, + duration_seconds=1.0, + cost_usd=0.0, + ) + assert invalid["accepted"] is False + assert "finite" in " ".join(invalid["reasons"]) + json.dumps(invalid, allow_nan=False) + + +@pytest.mark.parametrize( + ("field", "value", "gate_config"), + [ + ("duration_seconds", "1.0", {"max_duration_seconds": 10}), + ("duration_seconds", True, {"max_duration_seconds": 10}), + ("duration_seconds", float("nan"), {"max_duration_seconds": 10}), + ("duration_seconds", float("inf"), {"max_duration_seconds": 10}), + ("cost_usd", "1.0", {"max_cost_usd": 10}), + ("cost_usd", None, {"max_cost_usd": 10}), + ("cost_usd", True, {"max_cost_usd": 10}), + ("cost_usd", float("nan"), {"max_cost_usd": 10}), + ("cost_usd", float("inf"), {"max_cost_usd": 10}), + ("config", "0.1", {"min_validation_delta": "0.1"}), + ("config", True, {"min_validation_delta": True}), + ("config", float("nan"), {"min_validation_delta": float("nan")}), + ("config", float("inf"), {"min_validation_delta": float("inf")}), + ("config", float("-inf"), {"min_validation_delta": float("-inf")}), + ("config", -0.1, {"min_validation_delta": -0.1}), + ("config", "10", {"max_duration_seconds": "10"}), + ("config", True, {"max_cost_usd": True}), + ("config", float("inf"), {"max_cost_usd": float("inf")}), + ], +) +def test_gate_rejects_malformed_numeric_evidence_without_raising( + field: str, + value: Any, + gate_config: dict[str, Any], +): + module = load_pipeline_module() + baseline = _gate_summary( + 0.25, + [{"case_id": "a", "score": 0.25, "passed": True, "tags": []}], + ) + candidate = _gate_summary( + 0.75, + [{"case_id": "a", "score": 0.75, "passed": True, "tags": []}], + ) + duration_seconds: Any = 1.0 + cost_usd: Any = 0.0 + if field == "duration_seconds": + duration_seconds = value + elif field == "cost_usd": + cost_usd = value + + result = module.apply_gate( + candidate_id="malformed_numeric", + baseline_val=baseline, + candidate_val=candidate, + gate_config=gate_config, + duration_seconds=duration_seconds, + cost_usd=cost_usd, + ) + + assert result["accepted"] is False + json.dumps(result, allow_nan=False) + + +@pytest.mark.parametrize( + ("gate_config", "baseline_case", "candidate_case", "reason"), + [ + ( + {"allow_new_hard_fails": "false", "required_metrics": [ROUTE_TOOL_ARGS_METRIC]}, + {"case_id": "a", "score": 0.25, "passed": True, "tags": []}, + {"case_id": "a", "score": 0.75, "passed": False, "tags": []}, + "hard fail", + ), + ( + {"allow_critical_regression": "false", "required_metrics": [ROUTE_TOOL_ARGS_METRIC]}, + {"case_id": "a", "score": 1.0, "passed": True, "tags": ["critical"]}, + {"case_id": "a", "score": 0.75, "passed": True, "tags": ["critical"]}, + "critical", + ), + ], +) +def test_gate_allow_flags_require_exact_booleans( + gate_config: dict[str, Any], + baseline_case: dict[str, Any], + candidate_case: dict[str, Any], + reason: str, +): + module = load_pipeline_module() + result = module.apply_gate( + candidate_id="invalid_allow_flag", + baseline_val=_gate_summary(0.25, [baseline_case]), + candidate_val=_gate_summary(0.75, [candidate_case]), + gate_config=gate_config, + duration_seconds=1.0, + cost_usd=0.0, + ) + + assert result["accepted"] is False + assert reason in " ".join(result["reasons"]) + assert "boolean" in " ".join(result["reasons"]) + + +@pytest.mark.parametrize( + "required_metrics", + [1, True, {}, [""], [" "], ["metric", "metric"], ["metric", 1], [["metric"]]], +) +def test_gate_rejects_invalid_required_metrics_without_raising(required_metrics: Any): + module = load_pipeline_module() + result = module.apply_gate( + candidate_id="invalid_required_metrics", + baseline_val=_gate_summary(0.25, [{"case_id": "a", "score": 0.25, "passed": True, "tags": []}]), + candidate_val=_gate_summary(0.75, [{"case_id": "a", "score": 0.75, "passed": True, "tags": []}]), + gate_config={"required_metrics": required_metrics}, + duration_seconds=1.0, + cost_usd=0.0, + ) + + assert result["accepted"] is False + assert "required_metrics" in " ".join(result["reasons"]) + json.dumps(result, allow_nan=False) + + +@pytest.mark.parametrize( + ("malformed_field", "malformed_value"), + [ + ("baseline_val", None), + ("baseline_val", []), + ("candidate_val", "invalid"), + ("candidate_val", 1), + ("gate_config", None), + ("gate_config", []), + ], +) +def test_apply_gate_rejects_non_mapping_inputs_without_raising( + malformed_field: str, + malformed_value: Any, +): + module = load_pipeline_module() + kwargs: dict[str, Any] = { + "candidate_id": "non_mapping", + "baseline_val": _gate_summary( + 0.25, + [{"case_id": "a", "score": 0.25, "passed": True, "tags": []}], + ), + "candidate_val": _gate_summary( + 0.75, + [{"case_id": "a", "score": 0.75, "passed": True, "tags": []}], + ), + "gate_config": {"required_metrics": [ROUTE_TOOL_ARGS_METRIC]}, + "duration_seconds": 1.0, + "cost_usd": 0.0, + } + kwargs[malformed_field] = malformed_value + + result = module.apply_gate(**kwargs) + + assert result["accepted"] is False + assert "mapping" in " ".join(result["reasons"]) + json.dumps(result, allow_nan=False) + + +def test_load_gate_config_preserves_invalid_single_metric_string_for_rejection(): + module = load_pipeline_module() + config = module.load_gate_config(overrides={"required_metrics": "metric"}) + + assert config["required_metrics"] == "metric" + result = module.apply_gate( + candidate_id="single_metric_string", + baseline_val=_gate_summary( + 0.25, + [{"case_id": "a", "score": 0.25, "passed": True, "tags": []}], + ), + candidate_val=_gate_summary( + 0.75, + [{"case_id": "a", "score": 0.75, "passed": True, "tags": []}], + ), + gate_config=config, + duration_seconds=1.0, + cost_usd=0.0, + ) + assert result["accepted"] is False + assert "required_metrics" in " ".join(result["reasons"]) + + +@pytest.mark.parametrize( + "payload", + [ + [], + [["allow_new_hard_fails", True], ["required_metrics", []]], + ], +) +def test_load_gate_config_rejects_non_mapping_sources(tmp_path: Path, payload: Any): + module = load_pipeline_module() + gate_path = tmp_path / "gate.json" + gate_path.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ValueError, match="gate config.*object"): + module.load_gate_config(gate_path) + with pytest.raises(ValueError, match="gate config overrides.*mapping"): + module.load_gate_config(overrides=payload) + + +def test_unknown_gate_keys_fail_closed_for_files_overrides_and_direct_calls(tmp_path: Path): + module = load_pipeline_module() + typo_gate = {"min_validaton_delta": 0.9} + gate_path = tmp_path / "gate.json" + gate_path.write_text(json.dumps(typo_gate), encoding="utf-8") + + with pytest.raises(ValueError, match="unknown gate config"): + module.load_gate_config(gate_path) + with pytest.raises(ValueError, match="unknown gate config"): + module.load_gate_config(overrides=typo_gate) + + result = module.apply_gate( + candidate_id="typo_gate", + baseline_val=_gate_summary( + 0.25, + [{"case_id": "a", "score": 0.25, "passed": True, "tags": []}], + ), + candidate_val=_gate_summary( + 0.75, + [{"case_id": "a", "score": 0.75, "passed": True, "tags": []}], + ), + gate_config={ + **typo_gate, + "required_metrics": [ROUTE_TOOL_ARGS_METRIC], + }, + duration_seconds=1.0, + cost_usd=0.0, + ) + assert result["accepted"] is False + assert "unknown gate config" in " ".join(result["reasons"]) + + +def test_apply_gate_rejects_summary_score_not_derived_from_cases(): + module = load_pipeline_module() + baseline_cases = [{"case_id": "a", "score": 0.25, "passed": True, "tags": []}] + candidate_cases = [{"case_id": "a", "score": 0.25, "passed": True, "tags": []}] + + result = module.apply_gate( + candidate_id="forged_aggregate", + baseline_val=_gate_summary(0.25, baseline_cases), + candidate_val=_gate_summary(0.75, candidate_cases), + gate_config={"required_metrics": [ROUTE_TOOL_ARGS_METRIC]}, + duration_seconds=1.0, + cost_usd=0.0, + ) + + assert result["accepted"] is False + assert "summary score" in " ".join(result["reasons"]) + + +@pytest.mark.parametrize( + "required_metrics", + [1, "metric", [1], [["metric"]], [""], ["metric", "metric"]], +) +def test_optimizer_required_metrics_rejects_malformed_values( + tmp_path: Path, + required_metrics: Any, +): + module = load_pipeline_module() + payload = load_report(EXAMPLE_DIR / "optimizer.json") + payload["optimize"]["stop"]["required_metrics"] = required_metrics + path = tmp_path / "optimizer.json" + path.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ValueError, match="required_metrics"): + module.optimizer_required_metrics(path) + + +def test_gate_rejects_all_required_metrics_when_evidence_is_empty(): + module = load_pipeline_module() + candidate = _gate_summary(0.75, [{"case_id": "a", "score": 0.75, "passed": True, "tags": []}]) + candidate["metrics"] = {} + + result = module.apply_gate( + candidate_id="empty_all_metrics", + baseline_val=_gate_summary(0.25, [{"case_id": "a", "score": 0.25, "passed": True, "tags": []}]), + candidate_val=candidate, + gate_config={"required_metrics": "all"}, + duration_seconds=1.0, + cost_usd=0.0, + ) + + assert result["accepted"] is False + assert "required_metrics" in " ".join(result["reasons"]) + + +@pytest.mark.parametrize( + ("duration", "cost", "gate_config"), + [ + (-1.0, 0.0, {}), + (1.0, -1.0, {}), + (1.0, 0.0, {"max_cost_usd": -1.0}), + (1.0, 0.0, {"max_duration_seconds": -1.0}), + ], +) +def test_gate_rejects_negative_observed_values_and_budgets( + duration: float, + cost: float, + gate_config: dict[str, Any], +): + module = load_pipeline_module() + result = module.apply_gate( + candidate_id="negative_budget_evidence", + baseline_val=_gate_summary(0.25, [{"case_id": "a", "score": 0.25, "passed": True, "tags": []}]), + candidate_val=_gate_summary(0.75, [{"case_id": "a", "score": 0.75, "passed": True, "tags": []}]), + gate_config={"required_metrics": [ROUTE_TOOL_ARGS_METRIC], **gate_config}, + duration_seconds=duration, + cost_usd=cost, + ) + + assert result["accepted"] is False + assert "non-negative" in " ".join(result["reasons"]) + + +@pytest.mark.parametrize( + "candidate_case", + [ + {"case_id": 1, "score": 0.75, "passed": True, "tags": []}, + {"case_id": " ", "score": 0.75, "passed": True, "tags": []}, + {"case_id": "a", "score": -0.01, "passed": True, "tags": []}, + {"case_id": "a", "score": 1.01, "passed": True, "tags": []}, + {"case_id": "a", "score": 0.75, "passed": True, "tags": ("critical",)}, + ], +) +def test_gate_rejects_malformed_case_evidence_with_json_safe_result(candidate_case: dict[str, Any]): + module = load_pipeline_module() + result = module.apply_gate( + candidate_id="malformed_case_evidence", + baseline_val=_gate_summary(0.25, [{"case_id": "a", "score": 0.25, "passed": True, "tags": []}]), + candidate_val=_gate_summary(0.75, [candidate_case]), + gate_config={"required_metrics": [ROUTE_TOOL_ARGS_METRIC]}, + duration_seconds=1.0, + cost_usd=0.0, + ) + + assert result["accepted"] is False + json.dumps(result, allow_nan=False) + + +def test_gate_rejects_regression_even_when_minimum_delta_is_negative(): + module = load_pipeline_module() + baseline = _gate_summary( + 0.75, + [{"case_id": "a", "score": 0.75, "passed": True, "tags": []}], + ) + candidate = _gate_summary( + 0.5, + [{"case_id": "a", "score": 0.5, "passed": True, "tags": []}], + ) + + result = module.apply_gate( + candidate_id="regression", + baseline_val=baseline, + candidate_val=candidate, + gate_config={ + "min_validation_delta": -0.5, + "allow_critical_regression": True, + "required_metrics": [ROUTE_TOOL_ARGS_METRIC], + }, + duration_seconds=1.0, + cost_usd=0.0, + ) + + assert result["accepted"] is False + assert "did not improve" in " ".join(result["reasons"]) + + +@pytest.mark.parametrize( + ("baseline_tags", "candidate_tags"), + [ + (["critical", "validation"], ["validation"]), + (["validation"], ["validation", "critical"]), + (["validation", "refund"], ["validation", "faq"]), + ], +) +def test_gate_rejects_tag_mismatch_and_uses_baseline_for_critical_status( + baseline_tags: list[str], + candidate_tags: list[str], +): + module = load_pipeline_module() + baseline = _gate_summary( + 0.5, + [{"case_id": "a", "score": 1.0, "passed": True, "tags": baseline_tags}], + ) + candidate = _gate_summary( + 0.75, + [{"case_id": "a", "score": 0.5, "passed": True, "tags": candidate_tags}], + ) + + result = module.apply_gate( + candidate_id="retagged", + baseline_val=baseline, + candidate_val=candidate, + gate_config={ + "min_validation_delta": 0.0, + "allow_new_hard_fails": True, + "required_metrics": [ROUTE_TOOL_ARGS_METRIC], + }, + duration_seconds=1.0, + cost_usd=0.0, + ) + + assert result["accepted"] is False + assert "tag mismatch" in " ".join(result["reasons"]) + if "critical" in baseline_tags: + assert result["critical_regression_ids"] == ["a"] + + +@pytest.mark.parametrize( + "candidate_cases", + [ + None, + ["not a case"], + [ + {"case_id": "a", "score": 1.0, "passed": True, "tags": []}, + {"case_id": "a", "score": 1.0, "passed": True, "tags": []}, + ], + [{"case_id": "a", "score": 1.0, "passed": "true", "tags": []}], + [{"case_id": "a", "score": 1.0, "passed": True, "tags": "critical"}], + [{"case_id": "a", "score": float("nan"), "passed": True, "tags": []}], + ], +) +def test_gate_rejects_malformed_case_sets_without_raising(candidate_cases: Any): + module = load_pipeline_module() + baseline = _gate_summary( + 0.25, + [{"case_id": "a", "score": 0.25, "passed": True, "tags": []}], + ) + candidate = _gate_summary(0.75, candidate_cases) + + result = module.apply_gate( + candidate_id="malformed_cases", + baseline_val=baseline, + candidate_val=candidate, + gate_config={"required_metrics": [ROUTE_TOOL_ARGS_METRIC]}, + duration_seconds=1.0, + cost_usd=0.0, + ) + + assert result["accepted"] is False + json.dumps(result, allow_nan=False) + + +def test_required_metric_passed_must_be_true_and_case_deltas_are_total(): + module = load_pipeline_module() + baseline = _gate_summary( + 0.25, + [ + {"case_id": "a", "score": 0.25, "passed": False, "tags": []}, + {"case_id": "b", "score": 0.25, "passed": True, "tags": []}, + ], + ) + candidate = _gate_summary( + 0.75, + [ + {"case_id": "b", "score": 0.75, "passed": True, "tags": []}, + {"case_id": "c", "score": 1.0, "passed": True, "tags": []}, + ], + ) + candidate["metrics"][ROUTE_TOOL_ARGS_METRIC]["passed"] = "false" + + gate = module.apply_gate( + candidate_id="string_metric_status", + baseline_val=baseline, + candidate_val=candidate, + gate_config={"required_metrics": [ROUTE_TOOL_ARGS_METRIC]}, + duration_seconds=1.0, + cost_usd=0.0, + ) + assert gate["accepted"] is False + + deltas = module.build_case_deltas(baseline, candidate) + assert [item["case_id"] for item in deltas] == ["a", "b", "c"] + assert deltas[0]["root_cause"] == "missing_candidate" + assert deltas[0]["candidate_score"] is None + assert deltas[2]["root_cause"] == "unexpected_candidate" + assert deltas[2]["baseline_score"] is None + json.dumps(deltas, allow_nan=False) + + +def test_case_deltas_classify_pass_fail_and_score_transitions(): + module = load_pipeline_module() + baseline = { + "case_results": [ + {"case_id": "new_pass", "score": 0.0, "passed": False, "actual_text": "b1"}, + {"case_id": "new_fail", "score": 1.0, "passed": True, "actual_text": "b2"}, + {"case_id": "up", "score": 0.4, "passed": True, "actual_text": "b3"}, + {"case_id": "down", "score": 0.8, "passed": True, "actual_text": "b4"}, + {"case_id": "same", "score": 1.0, "passed": True, "actual_text": "b5"}, + ] + } + candidate = { + "case_results": [ + {"case_id": "new_pass", "score": 1.0, "passed": True, "actual_text": "c1", "root_cause": "", "reasons": []}, + { + "case_id": "new_fail", + "score": 0.0, + "passed": False, + "actual_text": "c2", + "root_cause": "format_error", + "reasons": ["bad"], + }, + {"case_id": "up", "score": 0.6, "passed": True, "actual_text": "c3", "root_cause": "", "reasons": []}, + {"case_id": "down", "score": 0.6, "passed": True, "actual_text": "c4", "root_cause": "", "reasons": []}, + {"case_id": "same", "score": 1.0, "passed": True, "actual_text": "c5", "root_cause": "", "reasons": []}, + ] + } + + by_id = {item["case_id"]: item for item in module.build_case_deltas(baseline, candidate)} + + assert by_id["new_pass"]["change_type"] == "new_pass" + assert by_id["new_fail"]["change_type"] == "new_fail" + assert by_id["up"]["change_type"] == "score_improved" + assert by_id["down"]["change_type"] == "score_regressed" + assert by_id["same"]["change_type"] == "unchanged" + assert by_id["new_fail"]["baseline_passed"] is True + assert by_id["new_fail"]["candidate_passed"] is False + + +def test_summary_omits_thoughts_and_redacts_provider_credentials_from_report_text(): + module = load_pipeline_module() + payload = load_report(EXAMPLE_DIR / "val.evalset.json") + case = payload["eval_cases"][0] + visible_final = '{"route":"faq","tool":{"name":"none","arguments":{}}}' + actual_invocation = SimpleNamespace( + final_response={ + "parts": [ + {"text": "internal chain of thought", "thought": True}, + {"text": visible_final, "thought": False}, + ] + } + ) + expected_invocation = SimpleNamespace(final_response={"parts": [{"text": visible_final, "thought": False}]}) + secret = "ASIA_SECRET_SESSION_TOKEN" + run = SimpleNamespace( + eval_metric_result_per_invocation=[ + SimpleNamespace( + actual_invocation=actual_invocation, + expected_invocation=expected_invocation, + ) + ], + final_eval_status="failed", + error_message=f"request failed: X-Amz-Security-Token: {secret}; retry later", + overall_eval_metric_results=[ + SimpleNamespace( + metric_name="provider_metric", + score=0.0, + eval_status="failed", + details=SimpleNamespace(reason=f"provider headers: X-Amz-Security-Token: {secret}"), + threshold=1.0, + ), + SimpleNamespace( + metric_name="normal_metric", + score=1.0, + eval_status="passed", + details=SimpleNamespace(reason="normal evaluator explanation"), + threshold=1.0, + ), + ], + ) + result = SimpleNamespace( + results_by_eval_set_id={ + payload["eval_set_id"]: SimpleNamespace( + eval_results_by_eval_id={case["eval_id"]: [run]}, + ) + } + ) + + summary = module.summarize_evaluate_result(result, payload) + case_result = summary["case_results"][0] + + assert case_result["actual_text"] == visible_final + assert case_result["key_trace"]["actual_final_response"] == visible_final + assert "internal chain of thought" not in json.dumps(case_result) + assert "request failed" in case_result["key_trace"]["error_message"] + assert case_result["metrics"]["normal_metric"]["reason"] == "normal evaluator explanation" + serialized_summary = json.dumps(summary) + assert "X-Amz-Security-Token" not in serialized_summary + assert secret not in serialized_summary + + +def test_summary_fails_closed_when_a_reported_metric_was_not_evaluated(): + module = load_pipeline_module() + payload = load_report(EXAMPLE_DIR / "val.evalset.json") + case = payload["eval_cases"][0] + run = SimpleNamespace( + eval_metric_result_per_invocation=[], + final_eval_status="passed", + error_message=None, + overall_eval_metric_results=[ + SimpleNamespace( + metric_name=ROUTE_TOOL_ARGS_METRIC, + score=None, + eval_status="not_evaluated", + details=None, + threshold=1.0, + ), + SimpleNamespace( + metric_name="llm_rubric_response", + score=1.0, + eval_status="passed", + details=None, + threshold=0.66, + ), + ], + ) + result = SimpleNamespace( + results_by_eval_set_id={ + payload["eval_set_id"]: SimpleNamespace( + eval_results_by_eval_id={case["eval_id"]: [run]}, + ) + } + ) + + summary = module.summarize_evaluate_result(result, payload) + case_result = summary["case_results"][0] + + assert case_result["passed"] is False + assert case_result["root_cause"] in module.TAXONOMY + assert case_result["reasons"] + assert case["eval_id"] in summary["failed_case_ids"] + + +@pytest.mark.parametrize( + "sensitive_text", + [ + "Authorization: Bearer secret-token", + "X-Api-Key: api-key-value", + "access_token=access-token-value", + "session token=session-token-value", + "security-token=security-token-value", + "client_secret=client-secret-value", + "db_credential=credential-value", + "Set-Cookie: session=cookie-value", + "X-Custom-Token: custom-token-value", + ], +) +def test_sanitize_report_text_redacts_semantic_credential_markers(sensitive_text: str): + module = load_pipeline_module() + + assert module.sanitize_report_text(f"upstream failed: {sensitive_text}") == ( + "upstream failed: provider details redacted" + ) + + +@pytest.mark.parametrize( + "sensitive_text", + [ + "https://user:password@provider.example/v1?api_key=fake-secret", + "http://provider.example/v1/models", + "sk-fake0123456789abcdefghijklmnopqrstuvwxyz", + ], +) +def test_sanitize_report_text_redacts_urls_and_standalone_provider_keys(sensitive_text: str): + module = load_pipeline_module() + sanitized = module.sanitize_report_text(f"request failed while calling {sensitive_text}") + + assert sanitized == "request failed while calling: provider details redacted" + assert sensitive_text not in sanitized + + +def test_sanitize_report_text_redacts_exact_configured_provider_values(monkeypatch: pytest.MonkeyPatch): + configured_key = "configured-fake-key-value" + configured_url = "provider.internal.example/v1/fake" + monkeypatch.setenv("TRPC_AGENT_API_KEY", configured_key) + monkeypatch.setenv("TRPC_AGENT_BASE_URL", configured_url) + module = load_pipeline_module() + + for configured_value in (configured_key, configured_url): + sanitized = module.sanitize_report_text(f"connection failed at {configured_value}") + assert sanitized == "connection failed at: provider details redacted" + assert configured_value not in sanitized + + +def test_sanitize_report_text_preserves_ordinary_error_context(): + module = load_pipeline_module() + message = "optimizer rejected round 2 because validation score decreased" + assert module.sanitize_report_text(message) == message + + +def test_no_run_key_trace_uses_safe_shape_and_omits_thought_content(): + module = load_pipeline_module() + payload = load_report(EXAMPLE_DIR / "val.evalset.json") + case = payload["eval_cases"][0] + case["conversation"][0]["final_response"] = { + "parts": [ + {"text": "internal expected thought", "thought": True}, + {"text": "visible expected final", "thought": False}, + ] + } + result = SimpleNamespace( + results_by_eval_set_id={ + payload["eval_set_id"]: SimpleNamespace( + eval_results_by_eval_id={case["eval_id"]: []}, + ) + } + ) + + summary = module.summarize_evaluate_result(result, payload) + key_trace = summary["case_results"][0]["key_trace"] + + assert key_trace == { + "invocation_id": str(case["conversation"][0]["invocation_id"]), + "actual_final_response": "", + "expected_final_response": "visible expected final", + "error_message": "AgentEvaluator returned no run for case", + } + assert "thought" not in json.dumps(key_trace) + + +def test_build_candidate_report_rejects_case_set_mismatch(): + module = load_pipeline_module() + baseline = _gate_summary( + 0.25, + [{"case_id": "a", "score": 0.25, "passed": True, "tags": []}], + ) + validation = _gate_summary( + 0.75, + [{"case_id": "b", "score": 0.75, "passed": True, "tags": []}], + ) + report = module.build_candidate_report( + candidate_id="mismatched", + fixture={}, + train=baseline, + optimizer_dev=baseline, + validation=validation, + baseline_train=baseline, + baseline_optimizer_dev=baseline, + baseline_val=baseline, + gate_config={}, + duration_seconds=1.0, + cost_usd=0.0, + seed=7, + optimizer_config=EXAMPLE_DIR / "optimizer.json", + ) + + assert report["gate"]["accepted"] is False + assert report["gate"]["missing_case_ids"] == ["a"] + assert report["gate"]["unexpected_case_ids"] == ["b"] + json.dumps(report, allow_nan=False) + + +@pytest.mark.parametrize( + "candidate_cases", + [ + None, + ["not a case"], + [{"case_id": "a", "score": float("nan"), "passed": True, "tags": []}], + [{"case_id": "a", "score": 0.75, "passed": "false", "tags": "critical"}], + ], +) +def test_build_candidate_report_is_total_for_malformed_validation_cases(candidate_cases: Any): + module = load_pipeline_module() + baseline = _gate_summary( + 0.25, + [{"case_id": "a", "score": 0.25, "passed": True, "tags": []}], + ) + validation = _gate_summary(0.75, candidate_cases) + report = module.build_candidate_report( + candidate_id="malformed_report", + fixture={}, + train=baseline, + optimizer_dev=baseline, + validation=validation, + baseline_train=baseline, + baseline_optimizer_dev=baseline, + baseline_val=baseline, + gate_config={}, + duration_seconds=1.0, + cost_usd=0.0, + seed=7, + optimizer_config=EXAMPLE_DIR / "optimizer.json", + ) + + assert report["gate"]["accepted"] is False + json.dumps(report, allow_nan=False) + + +def test_build_candidate_report_sanitizes_nonfinite_case_reasons(): + module = load_pipeline_module() + baseline = _gate_summary( + 0.25, + [{"case_id": "a", "score": 0.25, "passed": True, "tags": []}], + ) + validation = _gate_summary( + 0.75, + [ + { + "case_id": "a", + "score": 0.0, + "passed": False, + "tags": [], + "reasons": [float("nan")], + } + ], + ) + + report = module.build_candidate_report( + candidate_id="nonfinite_reasons", + fixture={}, + train=baseline, + optimizer_dev=baseline, + validation=validation, + baseline_train=baseline, + baseline_optimizer_dev=baseline, + baseline_val=baseline, + gate_config={}, + duration_seconds=1.0, + cost_usd=0.0, + seed=7, + optimizer_config=EXAMPLE_DIR / "optimizer.json", + ) + + assert report["gate"]["accepted"] is False + json.dumps(report, allow_nan=False) + + +def load_report(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def make_evaluate_result(eval_set_path: Path, *, score: float = 1.0, passed: bool = True): + from trpc_agent_sdk.evaluation import EvalCaseResult + from trpc_agent_sdk.evaluation import EvalMetricResult + from trpc_agent_sdk.evaluation import EvalSetAggregateResult + from trpc_agent_sdk.evaluation import EvalStatus + from trpc_agent_sdk.evaluation import EvaluateResult + + payload = load_report(eval_set_path) + status = EvalStatus.PASSED if passed else EvalStatus.FAILED + case_results = {} + for case in payload["eval_cases"]: + metric = EvalMetricResult( + metric_name=ROUTE_TOOL_ARGS_METRIC, + threshold=1.0, + criterion={"final_response": {"json": {"match": "exact"}}}, + score=score, + eval_status=status, + ) + case_results[case["eval_id"]] = [ + EvalCaseResult( + eval_set_id=payload["eval_set_id"], + eval_id=case["eval_id"], + run_id=1, + final_eval_status=status, + overall_eval_metric_results=[metric], + eval_metric_result_per_invocation=[], + session_id="fake-session", + ) + ] + return EvaluateResult( + results_by_eval_set_id={ + payload["eval_set_id"]: EvalSetAggregateResult( + eval_results_by_eval_id=case_results, + num_runs=1, + ) + } + ) + + +def patch_agent_evaluator( + monkeypatch: pytest.MonkeyPatch, + *, + score: float = 1.0, + passed: bool = True, +) -> list[Path]: + calls: list[Path] = [] + + class FakeExecuter: + def __init__(self, eval_set_path: str) -> None: + self.eval_set_path = Path(eval_set_path) + self.result = None + + async def evaluate(self) -> None: + calls.append(self.eval_set_path) + self.result = make_evaluate_result(self.eval_set_path, score=score, passed=passed) + + def get_result(self): + return self.result + + def fake_get_executer(eval_dataset_file_path_or_dir: str, **_: Any) -> FakeExecuter: + return FakeExecuter(eval_dataset_file_path_or_dir) + + import trpc_agent_sdk.evaluation as evaluation_pkg + + monkeypatch.setattr(evaluation_pkg.AgentEvaluator, "get_executer", staticmethod(fake_get_executer)) + return calls + + +def _copy_public_evalsets(tmp_path: Path) -> tuple[Path, Path, Path]: + paths = [] + for name in ("train.evalset.json", "optimizer_dev.evalset.json", "val.evalset.json"): + target = tmp_path / name + shutil.copyfile(EXAMPLE_DIR / name, target) + paths.append(target) + return tuple(paths) # type: ignore[return-value] + + +def test_validate_inputs_accepts_distinct_public_evalset_copies(tmp_path: Path): + module = load_pipeline_module() + module.validate_inputs(*_copy_public_evalsets(tmp_path)) + + +def test_validate_inputs_rejects_byte_identical_copies(tmp_path: Path): + module = load_pipeline_module() + train, optimizer_dev, validation = _copy_public_evalsets(tmp_path) + shutil.copyfile(train, optimizer_dev) + + with pytest.raises(ValueError, match="byte-identical"): + module.validate_inputs(train, optimizer_dev, validation) + + +def test_validate_inputs_rejects_hardlinks_when_supported(tmp_path: Path): + module = load_pipeline_module() + train, _, validation = _copy_public_evalsets(tmp_path) + optimizer_dev = tmp_path / "optimizer_dev_hardlink.evalset.json" + try: + os.link(train, optimizer_dev) + except OSError as error: + pytest.skip(f"hardlinks unavailable: {error}") + + with pytest.raises(ValueError, match="same file"): + module.validate_inputs(train, optimizer_dev, validation) + + +def test_validate_inputs_rejects_same_target_symlinks_when_supported(tmp_path: Path): + module = load_pipeline_module() + train, _, validation = _copy_public_evalsets(tmp_path) + optimizer_dev = tmp_path / "optimizer_dev_symlink.evalset.json" + try: + optimizer_dev.symlink_to(train) + except OSError as error: + pytest.skip(f"symlinks unavailable: {error}") + + with pytest.raises(ValueError, match="same file"): + module.validate_inputs(train, optimizer_dev, validation) + + +@pytest.mark.parametrize("overlap", ["id", "input", "gold"]) +def test_validate_inputs_rejects_pairwise_semantic_overlap(tmp_path: Path, overlap: str): + module = load_pipeline_module() + train, optimizer_dev, validation = _copy_public_evalsets(tmp_path) + train_payload = json.loads(train.read_text(encoding="utf-8")) + dev_payload = json.loads(optimizer_dev.read_text(encoding="utf-8")) + train_case = train_payload["eval_cases"][0] + dev_case = dev_payload["eval_cases"][0] + if overlap == "id": + dev_case["eval_id"] = train_case["eval_id"] + elif overlap == "input": + source = train_case["conversation"][0]["user_content"]["parts"][0]["text"] + dev_case["conversation"][0]["user_content"]["parts"][0]["text"] = f" {source.upper()} " + else: + dev_case["conversation"][0]["final_response"]["parts"] = copy.deepcopy( + train_case["conversation"][0]["final_response"]["parts"] + ) + dev_case["conversation"][0]["final_response"]["role"] = "different-metadata" + optimizer_dev.write_text(json.dumps(dev_payload), encoding="utf-8") + + with pytest.raises(ValueError, match=overlap): + module.validate_inputs(train, optimizer_dev, validation) + + +@pytest.mark.parametrize( + "payload", + [ + [], + {}, + {"eval_cases": []}, + {"eval_cases": "invalid"}, + {"eval_cases": [{}]}, + {"eval_cases": [{"eval_id": "x", "conversation": []}]}, + ], +) +def test_validate_inputs_rejects_invalid_evalset_shapes(tmp_path: Path, payload: Any): + module = load_pipeline_module() + train, optimizer_dev, validation = _copy_public_evalsets(tmp_path) + optimizer_dev.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ValueError, match="optimizer_dev evalset"): + module.validate_inputs(train, optimizer_dev, validation) + + +def test_validate_inputs_rejects_duplicate_eval_ids_within_a_split(tmp_path: Path): + module = load_pipeline_module() + train, optimizer_dev, validation = _copy_public_evalsets(tmp_path) + payload = load_report(optimizer_dev) + payload["eval_cases"][1]["eval_id"] = payload["eval_cases"][0]["eval_id"] + optimizer_dev.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ValueError, match="duplicate eval_id"): + module.validate_inputs(train, optimizer_dev, validation) + + +@pytest.mark.parametrize("variant", ["thought_part", "split_visible_parts"]) +def test_validate_inputs_canonicalizes_gold_with_final_text_from_content( + tmp_path: Path, + variant: str, +): + module = load_pipeline_module() + train, optimizer_dev, validation = _copy_public_evalsets(tmp_path) + train_payload = load_report(train) + dev_payload = load_report(optimizer_dev) + train_response = train_payload["eval_cases"][0]["conversation"][0]["final_response"] + dev_response = dev_payload["eval_cases"][0]["conversation"][0]["final_response"] + if variant == "thought_part": + dev_response["parts"] = copy.deepcopy(train_response["parts"]) + dev_response["parts"].append({"text": "private thought", "thought": True}) + else: + train_response["parts"] = [{"text": "visible one\nvisible two"}] + dev_response["parts"] = [{"text": "visible one"}, {"text": "visible two"}] + train.write_text(json.dumps(train_payload), encoding="utf-8") + optimizer_dev.write_text(json.dumps(dev_payload), encoding="utf-8") + + with pytest.raises(ValueError, match="gold"): + module.validate_inputs(train, optimizer_dev, validation) + + +def test_validate_inputs_checks_every_conversation_turn(tmp_path: Path): + module = load_pipeline_module() + train, optimizer_dev, validation = _copy_public_evalsets(tmp_path) + train_payload = load_report(train) + dev_payload = load_report(optimizer_dev) + shared_turn = { + "invocation_id": "shared_second_turn", + "user_content": {"parts": [{"text": "shared optimizer-visible turn"}], "role": "user"}, + "final_response": {"parts": [{"text": '{"shared": true}'}], "role": "model"}, + } + train_payload["eval_cases"][0]["conversation"].append(copy.deepcopy(shared_turn)) + dev_payload["eval_cases"][0]["conversation"].append(copy.deepcopy(shared_turn)) + train.write_text(json.dumps(train_payload), encoding="utf-8") + optimizer_dev.write_text(json.dumps(dev_payload), encoding="utf-8") + + with pytest.raises(ValueError, match="input|gold"): + module.validate_inputs(train, optimizer_dev, validation) + + +def test_validate_inputs_canonicalizes_structurally_equal_json_gold(tmp_path: Path): + module = load_pipeline_module() + train, optimizer_dev, validation = _copy_public_evalsets(tmp_path) + train_payload = load_report(train) + dev_payload = load_report(optimizer_dev) + train_payload["eval_cases"][0]["conversation"][0]["final_response"]["parts"] = [ + {"text": '{"route":"shared","tool":{"name":"x","arguments":{"a":1,"b":2}}}'} + ] + dev_payload["eval_cases"][0]["conversation"][0]["final_response"]["parts"] = [ + {"text": '{ "tool": { "arguments": { "b": 2, "a": 1 }, "name": "x" }, "route": "shared" }'} + ] + train.write_text(json.dumps(train_payload), encoding="utf-8") + optimizer_dev.write_text(json.dumps(dev_payload), encoding="utf-8") + + with pytest.raises(ValueError, match="gold"): + module.validate_inputs(train, optimizer_dev, validation) + + +def test_validate_inputs_uses_json_criterion_numeric_tolerance(tmp_path: Path): + module = load_pipeline_module() + train, optimizer_dev, validation = _copy_public_evalsets(tmp_path) + train_payload = load_report(train) + dev_payload = load_report(optimizer_dev) + train_payload["eval_cases"][0]["conversation"][0]["final_response"]["parts"] = [{"text": '{"x":1.0}'}] + dev_payload["eval_cases"][0]["conversation"][0]["final_response"]["parts"] = [{"text": '{"x":1.0000005}'}] + train.write_text(json.dumps(train_payload), encoding="utf-8") + optimizer_dev.write_text(json.dumps(dev_payload), encoding="utf-8") + + with pytest.raises(ValueError, match="gold"): + module.validate_inputs(train, optimizer_dev, validation) + + +@pytest.mark.parametrize("json_alias", ["json", "json_strategy", "jsonStrategy"]) +def test_validate_inputs_uses_configured_json_criterion_tolerance(tmp_path: Path, json_alias: str): + module = load_pipeline_module() + train, optimizer_dev, validation = _copy_public_evalsets(tmp_path) + train_payload = load_report(train) + dev_payload = load_report(optimizer_dev) + train_payload["eval_cases"][0]["conversation"][0]["final_response"]["parts"] = [{"text": '{"x":1.0}'}] + dev_payload["eval_cases"][0]["conversation"][0]["final_response"]["parts"] = [{"text": '{"x":1.05}'}] + train.write_text(json.dumps(train_payload), encoding="utf-8") + optimizer_dev.write_text(json.dumps(dev_payload), encoding="utf-8") + metrics_config = { + "metrics": [ + { + "metric_name": "json_match", + "threshold": 1.0, + "criterion": { + "final_response": { + json_alias: { + "match": "exact", + "number_tolerance": 0.1, + } + } + }, + } + ], + "num_runs": 1, + } + + with pytest.raises(ValueError, match="gold"): + module.validate_inputs( + train, + optimizer_dev, + validation, + metrics_config=metrics_config, + ) + + +def test_directory_layout_and_assets_exist(): + expected = { + "README.md", + "run_pipeline.py", + "optimizer.json", + "optimization_report.schema.json", + "train.evalset.json", + "optimizer_dev.evalset.json", + "val.evalset.json", + "agent/__init__.py", + "agent/agent.py", + "agent/config.py", + "agent/prompts/system.md", + "agent/prompts/router.md", + "fixtures/fake_outputs.json", + "fixtures/offline_metrics.sample.json", + "fixtures/trace_outputs.json", + "fixtures/optimization_report.sample.json", + } + for rel in expected: + assert (EXAMPLE_DIR / rel).exists(), f"missing example asset: {rel}" + + +def test_evalsets_and_optimizer_config_are_schema_loadable(): + from trpc_agent_sdk.evaluation import EvalSet + from trpc_agent_sdk.evaluation._optimize_config import load_optimize_config + + train = EvalSet.model_validate_json((EXAMPLE_DIR / "train.evalset.json").read_text(encoding="utf-8")) + optimizer_dev = EvalSet.model_validate_json( + (EXAMPLE_DIR / "optimizer_dev.evalset.json").read_text(encoding="utf-8") + ) + val = EvalSet.model_validate_json((EXAMPLE_DIR / "val.evalset.json").read_text(encoding="utf-8")) + assert len(train.eval_cases) == 3 + assert len(optimizer_dev.eval_cases) >= 1 + assert len(val.eval_cases) == 3 + assert {case.eval_id for case in train.eval_cases} == { + "train_refund_001", + "train_manual_002", + "train_faq_003", + } + assert "val_shipping_delay_103" in {case.eval_id for case in val.eval_cases} + assert {case.eval_id for case in optimizer_dev.eval_cases}.isdisjoint({case.eval_id for case in val.eval_cases}) + + def evidence_sets(evalset: Any) -> tuple[set[str], set[str], set[str]]: + ids = {case.eval_id for case in evalset.eval_cases} + users = { + "".join(part.text or "" for part in case.conversation[0].user_content.parts) for case in evalset.eval_cases + } + gold = { + "".join(part.text or "" for part in case.conversation[0].final_response.parts) + for case in evalset.eval_cases + } + return ids, users, gold + + train_evidence = evidence_sets(train) + optimizer_dev_evidence = evidence_sets(optimizer_dev) + validation_evidence = evidence_sets(val) + assert len(optimizer_dev.eval_cases) == 3 + for optimizer_values, train_values, validation_values in zip( + optimizer_dev_evidence, + train_evidence, + validation_evidence, + ): + assert optimizer_values.isdisjoint(train_values) + assert optimizer_values.isdisjoint(validation_values) + + config = load_optimize_config(str(EXAMPLE_DIR / "optimizer.json")) + assert config.optimize.algorithm.name == "gepa_reflective" + assert {metric.metric_name for metric in config.evaluate.get_eval_metrics()} == { + ROUTE_TOOL_ARGS_METRIC, + "llm_rubric_response", + } + + +def test_pipeline_module_exposes_testable_contracts(): + module = load_pipeline_module() + assert inspect.iscoroutinefunction(module.amain) + assert inspect.iscoroutinefunction(module.run_fake_or_trace) + assert inspect.iscoroutinefunction(module.run_online) + assert callable(module.gate_candidate) + assert callable(module.attribution_for) + + +def test_readme_includes_design_notes_and_sample_report_shape(): + readme = (EXAMPLE_DIR / "README.md").read_text(encoding="utf-8") + assert "## Design Notes" in readme + assert "fixtures/optimization_report.sample.json" in readme + assert "candidate_local_patch" in readme + assert "candidate_overfit" in readme + assert "not globally suppressed" in readme + assert "pipeline awaits `Runner.close()`" in readme + assert "upstream OpenAI/httpx" in readme + assert "warnings remain observable" in readme + assert "cleanup warnings are cleanup defects" not in readme + assert "`tests/conftest.py` ignores" not in readme + assert "may fail during collection" in readme + + sample = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + required = { + "run_id", + "mode", + "seed", + "baseline", + "candidates", + "delta", + "gate_decision", + "failure_attribution", + "cost", + "duration_seconds", + "artifacts", + } + assert required <= set(sample) + assert sample["gate_decision"]["winner"] == "candidate_local_patch" + assert {candidate["id"] for candidate in sample["candidates"]} == { + "candidate_local_patch", + "candidate_noop", + "candidate_overfit", + } + + +def test_public_candidates_cover_success_noop_and_aggregate_regression(tmp_path: Path): + module = load_pipeline_module() + report = module.make_report( + mode="fake", + run_id="public_scenarios", + run_dir=tmp_path, + seed=7, + started=module.time.perf_counter(), + ) + candidates = {item["id"]: item for item in report["candidates"]} + assert candidates["candidate_local_patch"]["gate"]["accepted"] is True + assert candidates["candidate_noop"]["delta"]["validation_score"] == 0 + assert candidates["candidate_noop"]["gate"]["accepted"] is False + overfit = candidates["candidate_overfit"] + assert overfit["delta"]["train_score"] > 0 + assert overfit["delta"]["validation_score"] < 0 + assert overfit["gate"]["accepted"] is False + + +def test_design_notes_length_is_within_issue_limit(): + readme = (EXAMPLE_DIR / "README.md").read_text(encoding="utf-8") + section = readme.split("## Design Notes", 1)[1].split("## Verification", 1)[0] + non_whitespace = len("".join(section.split())) + assert 300 <= non_whitespace <= 500 + + +def test_sample_report_is_deterministic_and_has_no_temporary_paths(): + sample = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + assert sample["run_id"] == "sample" + assert "known_warning_filters" not in sample["environment_snapshot"] + assert "SSEDecoder._aiter_chunks close RuntimeWarning" not in json.dumps(sample) + stable_environment = { + "git_commit": "sample", + "git_dirty": False, + "python_version": "3.x", + "sdk_version": "sample", + "model_name": None, + "base_url_host": None, + "command": ( + "python examples/optimization/eval_optimize_loop/run_pipeline.py " + "--mode fake --output-dir runs --run-id sample" + ), + "config_path": "examples/optimization/eval_optimize_loop/optimizer.json", + } + for key, value in stable_environment.items(): + assert sample["environment_snapshot"][key] == value + + durations: list[Any] = [] + strings: list[str] = [] + + def collect(value: Any) -> None: + if isinstance(value, dict): + for key, item in value.items(): + if key == "duration_seconds": + durations.append(item) + collect(item) + elif isinstance(value, list): + for item in value: + collect(item) + elif isinstance(value, str): + strings.append(value) + + collect(sample) + assert durations + assert all(value == 0.0 for value in durations) + assert all("\\" not in value for value in strings) + assert all(not (len(value) >= 3 and value[1:3] == ":/") for value in strings) + normalized_bytes = ( + (EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json").read_bytes().replace(b"\r\n", b"\n") + ) + assert normalized_bytes.endswith(b"\n") + assert not normalized_bytes.endswith(b"\n\n") + + +def test_sample_report_validates_against_schema_and_required_fields_are_enforced(): + module = load_pipeline_module() + sample = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + + module.validate_report_schema(sample) + + broken = dict(sample) + broken.pop("environment_snapshot", None) + with pytest.raises(ValidationError): + module.validate_report_schema(broken) + + +@pytest.mark.parametrize( + ("mutation_path", "replacement"), + [ + (("candidates", 0, "delta"), {}), + (("baseline", "validation", "case_results"), [{}]), + (("candidates", 0, "case_deltas"), [{}]), + (("failure_attribution", "coverage"), 9.0), + (("candidates", 0, "audit"), {}), + ], +) +def test_report_schema_rejects_incomplete_core_objects( + mutation_path: tuple[Any, ...], + replacement: Any, +): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + target: Any = report + for key in mutation_path[:-1]: + target = target[key] + target[mutation_path[-1]] = replacement + + with pytest.raises(ValidationError): + module.validate_report_schema(report) + + +@pytest.mark.parametrize( + ("mutation_path", "value"), + [ + (("baseline", "validation", "case_results", 0, "score"), float("nan")), + (("candidates", 0, "audit", "duration_seconds"), float("inf")), + (("duration_seconds",), float("-inf")), + ], +) +def test_report_schema_rejects_nonfinite_numbers( + mutation_path: tuple[Any, ...], + value: float, +): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + target: Any = report + for key in mutation_path[:-1]: + target = target[key] + target[mutation_path[-1]] = value + + with pytest.raises(ValidationError): + module.validate_report_schema(report) + + +def test_report_schema_requires_numeric_delta_for_accepted_gate(): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + accepted_candidate = next(candidate for candidate in report["candidates"] if candidate["gate"]["accepted"]) + accepted_candidate["gate"]["validation_delta"] = None + + with pytest.raises(ValidationError): + module.validate_report_schema(report) + + accepted_candidate["gate"]["validation_delta"] = accepted_candidate["delta"]["validation_score"] + rejected_candidate = next(candidate for candidate in report["candidates"] if not candidate["gate"]["accepted"]) + rejected_candidate["gate"]["validation_delta"] = None + with pytest.raises(ValidationError, match="gate validation_delta"): + module.validate_report_schema(report) + + +@pytest.mark.parametrize( + "contradiction", + [ + "accepted_without_winner", + "accepted_unknown_winner", + "rejected_with_winner", + "rejected_with_accepted_candidate", + "accepted_nonpositive_delta", + "candidate_gate_id_mismatch", + "baseline_validation_alias_mismatch", + "candidate_validation_alias_mismatch", + "winner_delta_mismatch", + "winner_reasons_mismatch", + "rejected_nonzero_delta", + "top_level_model_call_sum_mismatch", + "optimizer_model_call_sum_mismatch", + "final_model_call_sum_mismatch", + "token_total_mismatch", + ], +) +def test_report_semantic_validation_rejects_contradictions(contradiction: str): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + winner = next(candidate for candidate in report["candidates"] if candidate["gate"]["accepted"]) + if contradiction == "accepted_without_winner": + report["gate_decision"]["winner"] = None + elif contradiction == "accepted_unknown_winner": + report["gate_decision"]["winner"] = "missing_candidate" + elif contradiction == "rejected_with_winner": + report["gate_decision"]["accepted"] = False + elif contradiction == "rejected_with_accepted_candidate": + report["gate_decision"] = {"accepted": False, "winner": None, "reasons": ["rejected"]} + report["delta"] = {"train_score": 0.0, "optimizer_dev_score": 0.0, "validation_score": 0.0} + elif contradiction == "accepted_nonpositive_delta": + winner["gate"]["validation_delta"] = 0.0 + elif contradiction == "candidate_gate_id_mismatch": + winner["gate"]["candidate_id"] = "different_candidate" + elif contradiction == "baseline_validation_alias_mismatch": + report["baseline"]["validation"] = copy.deepcopy(report["baseline"]["validation"]) + report["baseline"]["validation"]["score"] = 0.0 + elif contradiction == "candidate_validation_alias_mismatch": + winner["validation"] = copy.deepcopy(winner["validation"]) + winner["validation"]["score"] = 0.0 + elif contradiction == "winner_delta_mismatch": + report["delta"]["validation_score"] = 0.25 + elif contradiction == "winner_reasons_mismatch": + report["gate_decision"]["reasons"] = ["different reason"] + elif contradiction == "rejected_nonzero_delta": + report["gate_decision"] = {"accepted": False, "winner": None, "reasons": ["rejected"]} + for candidate in report["candidates"]: + candidate["gate"]["accepted"] = False + report["delta"]["validation_score"] = 0.25 + elif contradiction == "top_level_model_call_sum_mismatch": + report["cost"]["model_calls"] = 1 + elif contradiction == "optimizer_model_call_sum_mismatch": + report["cost"]["optimizer"]["model_calls"] = 1 + report["cost"]["model_calls"] = 1 + elif contradiction == "final_model_call_sum_mismatch": + report["cost"]["final_revalidation"]["model_calls"] = 1 + report["cost"]["model_calls"] = 1 + else: + report["cost"]["token_usage"]["total"] = 1 + + with pytest.raises(ValidationError): + module.validate_report_schema(report) + + +def test_report_semantics_reject_duplicate_candidate_ids(): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + duplicate_id = report["candidates"][0]["id"] + report["candidates"][1]["id"] = duplicate_id + report["candidates"][1]["gate"]["candidate_id"] = duplicate_id + + with pytest.raises(ValidationError, match="duplicate candidate"): + module.validate_report_schema(report) + + +@pytest.mark.parametrize( + ("owner", "summary_name"), + [ + ("baseline", "train"), + ("baseline", "optimizer_dev"), + ("baseline", "validation"), + ("candidate", "train"), + ("candidate", "optimizer_dev"), + ("candidate", "validation"), + ], +) +def test_report_semantics_reject_duplicate_case_ids_in_every_summary(owner: str, summary_name: str): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + container = report["baseline"] if owner == "baseline" else report["candidates"][0] + summary = container[summary_name] + summary["case_results"].append(copy.deepcopy(summary["case_results"][0])) + if summary_name == "validation": + container["final_validation"] = copy.deepcopy(summary) + + with pytest.raises(ValidationError, match="duplicate case_id"): + module.validate_report_schema(report) + + +@pytest.mark.parametrize("delta_name", ["train_score", "optimizer_dev_score", "validation_score"]) +def test_report_semantics_recompute_candidate_deltas(delta_name: str): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + report["candidates"][1]["delta"][delta_name] = 0.123456 + + with pytest.raises(ValidationError, match="candidate delta"): + module.validate_report_schema(report) + + +def test_report_semantics_require_gate_delta_to_match_candidate_delta(): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + report["candidates"][1]["gate"]["validation_delta"] = 0.123456 + + with pytest.raises(ValidationError, match="gate validation_delta"): + module.validate_report_schema(report) + + +def test_report_semantics_recompute_case_deltas(): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + report["candidates"][0]["case_deltas"][0]["change_type"] = "score_improved" + + with pytest.raises(ValidationError, match="case_deltas"): + module.validate_report_schema(report) + + +@pytest.mark.parametrize("field", ["pass_rate", "failed_case_ids", "source"]) +def test_report_semantics_recompute_evaluation_summary_evidence(field: str): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + summary = report["baseline"]["train"] + if field == "pass_rate": + summary[field] = 0.123456 + elif field == "failed_case_ids": + summary[field] = [summary["case_results"][0]["case_id"]] + else: + summary[field] = "fixture" + + with pytest.raises(ValidationError, match="summary"): + module.validate_report_schema(report) + + +def test_report_semantics_reject_case_level_aggregate_forgery(): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + candidate = next(item for item in report["candidates"] if item["gate"]["accepted"]) + validation = copy.deepcopy(candidate["validation"]) + validation["case_results"][0]["score"] = 0.5 + candidate["validation"] = validation + candidate["final_validation"] = copy.deepcopy(validation) + candidate["case_deltas"] = module.build_case_deltas(report["baseline"]["validation"], validation) + + with pytest.raises(ValidationError, match="summary score"): + module.validate_report_schema(report) + + +def test_report_semantics_recompute_critical_regression_gate_evidence(): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + candidate = next(item for item in report["candidates"] if item["gate"]["accepted"]) + validation = copy.deepcopy(candidate["validation"]) + critical_case = next(case for case in validation["case_results"] if "critical" in case["tags"]) + critical_case["score"] = 0.5 + validation["score"] = round( + sum(case["score"] for case in validation["case_results"]) / len(validation["case_results"]), + 6, + ) + candidate["validation"] = validation + candidate["final_validation"] = copy.deepcopy(validation) + candidate["delta"]["validation_score"] = module._score_delta( + validation["score"], + report["baseline"]["validation"]["score"], + ) + candidate["gate"]["validation_delta"] = candidate["delta"]["validation_score"] + candidate["case_deltas"] = module.build_case_deltas(report["baseline"]["validation"], validation) + report["delta"] = copy.deepcopy(candidate["delta"]) + + with pytest.raises(ValidationError, match="gate evidence|primary metric|metric-derived score"): + module.validate_report_schema(report) + + +def test_report_semantics_reject_required_metric_status_forgery(): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + candidate = next(item for item in report["candidates"] if item["gate"]["accepted"]) + metric = candidate["validation"]["metrics"][ROUTE_TOOL_ARGS_METRIC] + metric["score"] = 0.0 + candidate["final_validation"] = copy.deepcopy(candidate["validation"]) + + with pytest.raises(ValidationError, match="metric"): + module.validate_report_schema(report) + + +def test_report_semantics_recompute_aggregate_metrics_from_case_metrics(): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + candidate = next(item for item in report["candidates"] if item["gate"]["accepted"]) + metric = candidate["validation"]["metrics"]["llm_rubric_response"] + metric["score"] = 0.75 + metric["threshold"] = 0.5 + candidate["final_validation"] = copy.deepcopy(candidate["validation"]) + + with pytest.raises(ValidationError, match="aggregate metric"): + module.validate_report_schema(report) + + +@pytest.mark.parametrize("mutation", ["omitted", "contradictory"]) +def test_report_semantics_require_complete_consistent_case_metrics(mutation: str): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + candidate = next(item for item in report["candidates"] if item["gate"]["accepted"]) + case_metric = candidate["validation"]["case_results"][0]["metrics"][ROUTE_TOOL_ARGS_METRIC] + if mutation == "omitted": + del candidate["validation"]["case_results"][0]["metrics"][ROUTE_TOOL_ARGS_METRIC] + else: + case_metric.update({"passed": False, "status": "failed"}) + candidate["final_validation"] = copy.deepcopy(candidate["validation"]) + + with pytest.raises(ValidationError, match="case metric|metric coverage"): + module.validate_report_schema(report) + + +def test_report_semantics_recompute_case_score_when_primary_metric_is_null(): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + candidate = next(item for item in report["candidates"] if item["gate"]["accepted"]) + case = candidate["validation"]["case_results"][0] + primary = case["metrics"][ROUTE_TOOL_ARGS_METRIC] + primary.update({"score": None, "passed": False, "status": "failed"}) + rubric = case["metrics"]["llm_rubric_response"] + rubric.update({"score": 0.5, "threshold": 0.66, "passed": False, "status": "failed"}) + case.update( + { + "score": 1.0, + "passed": False, + "root_cause": "rubric_failed", + "reasons": ["rubric metric failed"], + } + ) + candidate["validation"]["pass_rate"] = 0.666667 + candidate["validation"]["failed_case_ids"] = [case["case_id"]] + candidate["final_validation"] = copy.deepcopy(candidate["validation"]) + + with pytest.raises(ValidationError, match="metric-derived score"): + module.validate_report_schema(report) + + +def test_report_semantics_require_zero_score_for_explicit_no_run_case(): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + candidate = next(item for item in report["candidates"] if item["gate"]["accepted"]) + case = candidate["validation"]["case_results"][0] + case.update( + { + "score": 1.0, + "passed": False, + "metrics": {}, + "root_cause": "runtime_error", + "reasons": ["evaluation runtime error: no run"], + } + ) + case["key_trace"]["error_message"] = "AgentEvaluator returned no run for case" + candidate["validation"]["pass_rate"] = 0.666667 + candidate["validation"]["failed_case_ids"] = [case["case_id"]] + candidate["final_validation"] = copy.deepcopy(candidate["validation"]) + + with pytest.raises(ValidationError, match="no-run case score"): + module.validate_report_schema(report) + + +def test_sample_report_records_hashed_normalized_evaluation_config(): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + snapshot = report["config_snapshot"] + + assert snapshot["evaluation"] + assert snapshot["evaluation_sha256"] == module.sha256_json(snapshot["evaluation"]) + snapshot["evaluation"]["num_runs"] = 2 + with pytest.raises(ValidationError, match="evaluation config hash"): + module.validate_report_schema(report) + snapshot["evaluation_sha256"] = module.sha256_json(snapshot["evaluation"]) + with pytest.raises(ValidationError, match="evaluation metrics artifact"): + module.validate_report_schema(report) + + +def test_sample_prompt_artifact_hashes_are_self_contained(): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + artifacts = list(report["baseline"]["prompt_artifacts"]) + for candidate in report["candidates"]: + artifacts.extend(candidate["prompt_artifacts"]) + + assert artifacts + for artifact in artifacts: + assert artifact["sha256"] == module.sha256_text(artifact["content"]) + + +def test_sample_report_records_prompt_target_manifest(): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + targets = report["config_snapshot"]["prompt_targets"] + baseline_by_name = {artifact["name"]: artifact for artifact in report["baseline"]["prompt_artifacts"]} + + assert set(targets) == {"system_prompt", "router_prompt"} + for name, target in targets.items(): + assert target["source_path"] == report["config_snapshot"]["paths"][name] + assert target["sha256"] == module.sha256_text(baseline_by_name[name]["content"]) + + +def test_report_semantics_reconcile_candidate_audit_cost_with_pipeline_cost(): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + report["candidates"][0]["audit"]["cost"]["estimated"] = 0.5 + + with pytest.raises(ValidationError, match="candidate audit cost"): + module.validate_report_schema(report) + + +@pytest.mark.parametrize( + "mutation", + [ + "environment_seed", + "candidate_config_hash", + "environment_extra_secret", + "config_extra_secret", + "duplicate_round", + "round_prompt_hash_keys", + "candidate_prompt_hash", + "round_prompt_hash_value", + "missing_config_hash_disagreement", + "evalset_turn_count", + "missing_config_coordinated", + "missing_metrics_coordinated", + "missing_evalsets_coordinated", + "candidate_prompt_artifacts_empty", + "accepted_round_prompt_evidence_empty", + "coordinated_prompt_target_rename", + "coordinated_missing_prompt_sources_with_forged_diffs", + "coordinated_missing_prompt_sources_with_valid_diffs", + "accepted_round_unknown_prompt_target", + ], +) +def test_report_semantics_bind_reproducibility_audit_fields(mutation: str): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + round_record = { + "round": 1, + "optimized_field_names": [], + "prompt_paths": {"system_prompt": "runs/sample/system.md"}, + "prompt_sha256": {"system_prompt": "0" * 64}, + "prompt_contents": {"system_prompt": "round content"}, + "validation_pass_rate": 0.0, + "metric_breakdown": {}, + "accepted": False, + "decision_reason": "audit fixture", + "failed_case_ids": [], + "cost_usd": 0.0, + "token_usage": {"prompt": 0, "completion": 0, "total": 0}, + "duration_seconds": 0.0, + } + if mutation == "environment_seed": + report["environment_snapshot"]["seed"] = 999 + elif mutation == "candidate_config_hash": + report["candidates"][0]["audit"]["config_sha256"] = "0" * 64 + elif mutation == "environment_extra_secret": + report["environment_snapshot"]["api_key"] = "must-not-be-accepted" + elif mutation == "config_extra_secret": + report["config_snapshot"]["api_key"] = "must-not-be-accepted" + elif mutation == "duplicate_round": + report["optimization_rounds"] = [copy.deepcopy(round_record), copy.deepcopy(round_record)] + elif mutation == "round_prompt_hash_keys": + round_record["prompt_sha256"] = {} + report["optimization_rounds"] = [round_record] + elif mutation == "candidate_prompt_hash": + report["candidates"][0]["prompt_artifacts"][0]["sha256"] = "0" * 64 + elif mutation == "round_prompt_hash_value": + round_record["prompt_paths"]["system_prompt"] = str(EXAMPLE_DIR / "agent" / "prompts" / "system.md") + report["optimization_rounds"] = [round_record] + elif mutation == "missing_config_hash_disagreement": + missing_path = "missing/optimizer.json" + report["config_snapshot"]["paths"]["optimizer_config"] = missing_path + report["environment_snapshot"]["config_path"] = missing_path + for index, candidate in enumerate(report["candidates"]): + candidate["audit"]["config_path"] = missing_path + candidate["audit"]["config_sha256"] = f"{index + 1:064x}" + elif mutation == "evalset_turn_count": + report["config_snapshot"]["evalsets"]["train"]["turn_count"] = 99 + elif mutation == "missing_config_coordinated": + missing_path = "missing/optimizer.json" + report["config_snapshot"]["paths"]["optimizer_config"] = missing_path + report["config_snapshot"]["optimizer_config_sha256"] = "0" * 64 + report["environment_snapshot"]["config_path"] = missing_path + report["artifacts"]["optimizer_config"] = missing_path + for candidate in report["candidates"]: + candidate["audit"]["config_path"] = missing_path + candidate["audit"]["config_sha256"] = "0" * 64 + elif mutation == "missing_metrics_coordinated": + missing_path = "missing/metrics.json" + report["config_snapshot"]["paths"]["evaluation_metrics"] = missing_path + report["config_snapshot"]["evaluation_metrics_sha256"] = "0" * 64 + report["artifacts"]["eval_metrics"] = missing_path + elif mutation == "missing_evalsets_coordinated": + path_keys = { + "train": "train_evalset", + "optimizer_dev": "optimizer_dev_evalset", + "final_validation": "final_validation_evalset", + } + for role, path_key in path_keys.items(): + missing_path = f"missing/{role}.json" + report["config_snapshot"]["paths"][path_key] = missing_path + report["artifacts"][path_key] = missing_path + manifest = report["config_snapshot"]["evalsets"][role] + manifest.update( + { + "path": missing_path, + "sha256": "0" * 64, + "case_count": 99, + "turn_count": 99, + } + ) + report["config_snapshot"]["paths"]["validation_evalset"] = report["config_snapshot"]["paths"][ + "final_validation_evalset" + ] + report["artifacts"]["validation_evalset"] = report["artifacts"]["final_validation_evalset"] + elif mutation == "candidate_prompt_artifacts_empty": + for candidate in report["candidates"]: + candidate["prompt_artifacts"] = [] + elif mutation == "accepted_round_prompt_evidence_empty": + round_record.update( + { + "optimized_field_names": ["system_prompt"], + "prompt_paths": {}, + "prompt_sha256": {}, + "prompt_contents": {}, + "accepted": True, + } + ) + report["optimization_rounds"] = [round_record] + elif mutation == "coordinated_prompt_target_rename": + renamed = {"system_prompt": "alpha", "router_prompt": "beta"} + report["config_snapshot"]["prompt_targets"] = { + renamed[name]: target for name, target in report["config_snapshot"]["prompt_targets"].items() + } + for owner in [report["baseline"], *report["candidates"]]: + for artifact in owner["prompt_artifacts"]: + source_text = Path(artifact["source_path"]).read_text(encoding="utf-8") + artifact["name"] = renamed[artifact["name"]] + artifact["diff"] = module.prompt_diff(source_text, artifact["content"], artifact["name"]) + elif mutation in { + "coordinated_missing_prompt_sources_with_forged_diffs", + "coordinated_missing_prompt_sources_with_valid_diffs", + }: + for name in ("system_prompt", "router_prompt"): + missing_path = f"missing/{name}.md" + report["config_snapshot"]["paths"][name] = missing_path + report["config_snapshot"]["prompt_targets"][name]["source_path"] = missing_path + report["artifacts"][name] = missing_path + baseline_by_name = {artifact["name"]: artifact for artifact in report["baseline"]["prompt_artifacts"]} + for owner in [report["baseline"], *report["candidates"]]: + for artifact in owner["prompt_artifacts"]: + artifact["source_path"] = report["config_snapshot"]["paths"][artifact["name"]] + artifact["diff"] = ( + "forged diff" + if mutation == "coordinated_missing_prompt_sources_with_forged_diffs" + else module.prompt_diff( + baseline_by_name[artifact["name"]]["content"], + artifact["content"], + artifact["name"], + ) + ) + else: + round_record.update( + { + "optimized_field_names": ["not_a_target"], + "prompt_paths": {"not_a_target": "missing/not-a-target.md"}, + "prompt_sha256": {"not_a_target": module.sha256_text("forged prompt evidence")}, + "prompt_contents": {"not_a_target": "forged prompt evidence"}, + "accepted": True, + } + ) + report["optimization_rounds"] = [round_record] + + with pytest.raises(ValidationError): + module.validate_report_schema(report) + + +@pytest.mark.parametrize( + "contradiction", + [ + "candidate_calls_known", + "candidate_calls_missing_scoped_reason", + "candidate_calls_missing_top_cost_reason", + "candidate_calls_missing_top_token_reason", + "optimizer_unknown_not_propagated", + "final_unknown_not_propagated", + "top_unknown_with_known_scopes", + "reflection_cost_mismatch", + "reflection_tokens_mismatch", + "reflection_zero_calls_nonzero_cost", + "reflection_zero_calls_nonzero_tokens", + ], +) +def test_report_semantics_enforce_cost_and_token_knownness(contradiction: str): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + cost = report["cost"] + optimizer = cost["optimizer"] + final = cost["final_revalidation"] + if contradiction == "candidate_calls_known": + optimizer["candidate_evaluation_agent_calls"] = 1 + optimizer["model_calls"] = 1 + cost["model_calls"] = 1 + elif contradiction == "candidate_calls_missing_scoped_reason": + optimizer["candidate_evaluation_agent_calls"] = 1 + optimizer["model_calls"] = 1 + optimizer["estimated_cost"] = None + optimizer["token_usage"] = None + optimizer["token_usage_known"] = False + optimizer["unknown_token_usage_reason"] = "unknown" + cost.update( + { + "estimated_total": None, + "cost_source": "unknown", + "unknown_cost_reason": "unknown", + "model_calls": 1, + "token_usage": None, + "token_usage_known": False, + "unknown_token_usage_reason": "unknown", + } + ) + elif contradiction in { + "candidate_calls_missing_top_cost_reason", + "candidate_calls_missing_top_token_reason", + }: + optimizer["candidate_evaluation_agent_calls"] = 1 + optimizer["model_calls"] = 1 + optimizer["estimated_cost"] = None + optimizer["token_usage"] = None + optimizer["token_usage_known"] = False + optimizer["unknown_token_usage_reason"] = "optimizer candidate-evaluation token usage is not exposed" + cost.update( + { + "estimated_total": None, + "cost_source": "unknown", + "unknown_cost_reason": ( + "unknown" + if contradiction == "candidate_calls_missing_top_cost_reason" + else "optimizer candidate-evaluation calls do not expose token or cost usage" + ), + "model_calls": 1, + "token_usage": None, + "token_usage_known": False, + "unknown_token_usage_reason": ( + "unknown" + if contradiction == "candidate_calls_missing_top_token_reason" + else "optimizer candidate-evaluation token usage is not exposed" + ), + } + ) + elif contradiction == "optimizer_unknown_not_propagated": + optimizer["estimated_cost"] = None + optimizer["token_usage"] = None + optimizer["token_usage_known"] = False + optimizer["unknown_token_usage_reason"] = "optimizer unknown" + elif contradiction == "final_unknown_not_propagated": + final["estimated_cost"] = None + final["token_usage"] = None + final["token_usage_known"] = False + final["unknown_token_usage_reason"] = "final unknown" + elif contradiction == "top_unknown_with_known_scopes": + cost.update( + { + "estimated_total": None, + "cost_source": "unknown", + "unknown_cost_reason": "unknown", + "token_usage": None, + "token_usage_known": False, + "unknown_token_usage_reason": "unknown", + } + ) + elif contradiction == "reflection_cost_mismatch": + optimizer["reflection_reported_usage"]["estimated_cost"] = 0.5 + elif contradiction == "reflection_tokens_mismatch": + optimizer["reflection_reported_usage"]["token_usage"] = { + "prompt": 1, + "completion": 0, + "total": 1, + } + elif contradiction == "reflection_zero_calls_nonzero_cost": + optimizer["reflection_reported_usage"]["estimated_cost"] = 0.5 + optimizer["estimated_cost"] = 0.5 + cost["estimated_total"] = 0.5 + else: + nonzero_tokens = {"prompt": 1, "completion": 0, "total": 1} + optimizer["reflection_reported_usage"]["token_usage"] = nonzero_tokens + optimizer["token_usage"] = copy.deepcopy(nonzero_tokens) + cost["token_usage"] = copy.deepcopy(nonzero_tokens) + + with pytest.raises(ValidationError): + module.validate_report_schema(report) + + +def test_report_schema_requires_candidate_audit_and_optimization_rounds(): + module = load_pipeline_module() + missing_audit = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + missing_audit["candidates"][0].pop("audit") + + with pytest.raises(ValidationError): + module.validate_report_schema(missing_audit) + + missing_rounds = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + missing_rounds.pop("optimization_rounds") + + with pytest.raises(ValidationError): + module.validate_report_schema(missing_rounds) + + +def test_report_schema_requires_each_optimization_round_token_usage_field(): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + report["optimization_rounds"] = [ + { + "round": 1, + "optimized_field_names": [], + "prompt_paths": {}, + "prompt_sha256": {}, + "validation_pass_rate": 1.0, + "metric_breakdown": {}, + "accepted": False, + "decision_reason": "malformed evidence rejected", + "failed_case_ids": [], + "cost_usd": 0.0, + "token_usage": {"prompt": 0, "completion": 0}, + "duration_seconds": 0.0, + } + ] + + with pytest.raises(ValidationError): + module.validate_report_schema(report) + + +@pytest.mark.parametrize( + ("known", "estimated"), + [(True, None), (False, 0.0)], +) +def test_report_schema_requires_consistent_candidate_audit_cost( + known: bool, + estimated: float | None, +): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + cost = report["candidates"][0]["audit"]["cost"] + cost["known"] = known + cost["estimated"] = estimated + + with pytest.raises(ValidationError): + module.validate_report_schema(report) + + +@pytest.mark.parametrize( + ("known", "reason"), + [(True, "must be null when known"), (False, None)], +) +def test_report_schema_requires_consistent_optimizer_token_accounting( + known: bool, + reason: str | None, +): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + optimizer = report["cost"]["optimizer"] + optimizer["token_usage_known"] = known + optimizer["unknown_token_usage_reason"] = reason + + with pytest.raises(ValidationError): + module.validate_report_schema(report) + + +def test_report_schema_allows_empty_no_run_case_metrics(): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + for summary_name in ("validation", "final_validation"): + case = report["baseline"][summary_name]["case_results"][0] + case["metrics"] = {} + case["key_trace"]["error_message"] = "AgentEvaluator returned no run for case" + case["root_cause"] = "runtime_error" + case["reasons"] = ["evaluation runtime error: AgentEvaluator returned no run for case"] + metric = report["baseline"][summary_name]["metrics"][ROUTE_TOOL_ARGS_METRIC] + metric.update({"score": 1.0, "passed": True, "status": "passed"}) + report["failure_attribution"] = module.attribution_for(report["baseline"]["validation"]) + + module.validate_report_schema(report) + + +@pytest.mark.parametrize( + ("mutation_path", "name"), + [ + (("baseline", "validation", "case_results", 0), "unexpected_case_field"), + (("candidates", 0, "case_deltas", 0), "unexpected_delta_field"), + (("candidates", 0, "gate"), "unexpected_gate_field"), + (("candidates", 0, "audit", "cost"), "unexpected_cost_field"), + ], +) +def test_report_schema_rejects_extra_core_properties( + mutation_path: tuple[Any, ...], + name: str, +): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + target: Any = report + for key in mutation_path: + target = target[key] + target[name] = True + + with pytest.raises(ValidationError): + module.validate_report_schema(report) + + +@pytest.mark.parametrize("name", ["unexpected_root_field", "api_key"]) +def test_report_schema_rejects_unknown_root_properties(name: str): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + report[name] = "must not be accepted" + + with pytest.raises(ValidationError): + module.validate_report_schema(report) + + +@pytest.mark.parametrize( + ("mutation_path", "value"), + [ + (("run_id",), "../escape"), + (("candidates", 0, "id"), "nested/candidate"), + (("candidates", 0, "gate", "candidate_id"), "C:\\absolute"), + (("run_id",), "CON"), + (("run_id",), "nul.txt"), + (("candidates", 0, "id"), "Com1.json"), + (("candidates", 0, "gate", "candidate_id"), "LPT9.log"), + ], +) +def test_report_schema_rejects_unsafe_artifact_identifiers( + mutation_path: tuple[Any, ...], + value: str, +): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + target: Any = report + for key in mutation_path[:-1]: + target = target[key] + target[mutation_path[-1]] = value + + with pytest.raises(ValidationError): + module.validate_report_schema(report) + + +@pytest.mark.parametrize( + "run_id", + [ + "", + ".", + "..", + "../escape", + "nested/run", + "nested\\run", + "/absolute", + "C:\\absolute", + "CON", + "nul.txt", + "Com1.json", + "LPT9.log", + ], +) +def test_make_run_dir_rejects_unsafe_single_path_components(tmp_path: Path, run_id: str): + module = load_pipeline_module() + + with pytest.raises(ValueError, match="run_id"): + module.make_run_dir(tmp_path, run_id) + + +@pytest.mark.parametrize("candidate_id", ["..", "../escape", "nested/candidate", "nested\\candidate"]) +def test_prompt_artifacts_reject_unsafe_candidate_path_components( + tmp_path: Path, + candidate_id: str, +): + module = load_pipeline_module() + source_prompts = module.read_source_prompts( + EXAMPLE_DIR / "agent" / "prompts" / "system.md", + EXAMPLE_DIR / "agent" / "prompts" / "router.md", + ) + + with pytest.raises(ValueError, match="candidate_id"): + module.write_prompt_artifacts( + run_dir=tmp_path, + candidate_id=candidate_id, + source_prompts=source_prompts, + candidate_prompts={}, + summary="unsafe candidate", + source_written=False, + ) + + assert not (tmp_path.parent / "escape").exists() + + +@pytest.mark.parametrize("writer", ["candidate", "optimizer_round"]) +def test_prompt_artifact_writers_reject_preexisting_prompts_symlink_escape( + tmp_path: Path, + writer: str, +): + module = load_pipeline_module() + run_dir = tmp_path / "run" + outside = tmp_path / "outside" + run_dir.mkdir() + outside.mkdir() + try: + (run_dir / "prompts").symlink_to(outside, target_is_directory=True) + except OSError as error: + pytest.skip(f"directory symlinks unavailable: {error}") + + with pytest.raises(ValueError, match="beneath run_dir"): + if writer == "candidate": + module.write_prompt_artifacts( + run_dir=run_dir, + candidate_id="candidate", + source_prompts=module.read_source_prompts( + EXAMPLE_DIR / "agent" / "prompts" / "system.md", + EXAMPLE_DIR / "agent" / "prompts" / "router.md", + ), + candidate_prompts={}, + summary="containment test", + source_written=False, + ) + else: + module.write_optimizer_round_artifacts( + run_dir=run_dir, + rounds=[ + SimpleNamespace( + round=1, + optimized_field_names=[], + candidate_prompts={"system_prompt": "prompt"}, + validation_pass_rate=1.0, + metric_breakdown={}, + accepted=False, + acceptance_reason=None, + skip_reason="containment test", + error_message=None, + failed_case_ids=[], + round_llm_cost=0.0, + round_token_usage={"prompt": 0, "completion": 0, "total": 0}, + duration_seconds=0.0, + ) + ], + ) + + assert list(outside.iterdir()) == [] + + +@pytest.mark.parametrize("writer", ["report_file", "trace_evalsets_dir"]) +def test_run_artifact_writers_reject_preexisting_symlink_escape( + tmp_path: Path, + writer: str, +): + module = load_pipeline_module() + run_dir = tmp_path / "run" + outside = tmp_path / "outside" + run_dir.mkdir() + outside.mkdir() + sentinel = outside / "sentinel.txt" + sentinel.write_text("unchanged", encoding="utf-8") + try: + if writer == "report_file": + (run_dir / "optimization_report.json").symlink_to(sentinel) + else: + (run_dir / "evalsets").symlink_to(outside, target_is_directory=True) + except OSError as error: + pytest.skip(f"symlinks unavailable: {error}") + + with pytest.raises(ValueError, match="run_dir|symlink"): + if writer == "report_file": + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + module.write_report(run_dir, report) + else: + payload = load_report(EXAMPLE_DIR / "train.evalset.json") + module.materialize_trace_evalset( + source_evalset=EXAMPLE_DIR / "train.evalset.json", + payload=payload, + outputs={}, + run_dir=run_dir, + candidate_id="candidate", + split="train", + ) + + assert sentinel.read_text(encoding="utf-8") == "unchanged" + assert sorted(path.name for path in outside.iterdir()) == ["sentinel.txt"] + + +def test_router_prompt_is_instructional_not_a_gold_answer(): + prompt = (EXAMPLE_DIR / "agent" / "prompts" / "router.md").read_text(encoding="utf-8") + + with pytest.raises(json.JSONDecodeError): + json.loads(prompt) + + assert "Output exactly one JSON object" in prompt + assert "route" in prompt + assert "create_refund_ticket" in prompt + + +@pytest.mark.asyncio +async def test_fake_mode_generates_complete_report_and_selects_local_patch(tmp_path: Path): + module = load_pipeline_module() + run_dir = await module.run_fake_or_trace( + mode="fake", + seed=7, + output_dir=tmp_path, + run_id="fake_case", + ) + report = load_report(run_dir / "optimization_report.json") + + required = { + "run_id", + "mode", + "seed", + "baseline", + "candidates", + "delta", + "gate_decision", + "failure_attribution", + "cost", + "duration_seconds", + "artifacts", + } + assert required <= set(report) + assert report["mode"] == "fake" + assert "known_warning_filters" not in report["environment_snapshot"] + assert "SSEDecoder._aiter_chunks close RuntimeWarning" not in json.dumps(report) + assert report["gate_decision"]["accepted"] is True + assert report["gate_decision"]["winner"] == "candidate_local_patch" + assert report["baseline"]["validation"]["score"] == pytest.approx(2 / 3) + assert report["baseline"]["final_validation"]["score"] == pytest.approx(2 / 3) + assert "optimizer_dev" in report["baseline"] + assert report["artifacts"]["optimizer_dev_evalset"].endswith("optimizer_dev.evalset.json") + assert report["artifacts"]["final_validation_evalset"].endswith("val.evalset.json") + assert report["delta"]["validation_score"] == pytest.approx(1 / 3) + assert "environment_snapshot" in report + assert report["environment_snapshot"]["seed"] == 7 + assert report["environment_snapshot"]["config_path"].endswith("optimizer.json") + first_case = report["baseline"]["validation"]["case_results"][0] + assert first_case["expected_text"] + assert first_case["key_trace"]["invocation_id"] + assert first_case["key_trace"]["actual_final_response"] == first_case["actual_text"] + assert first_case["key_trace"]["expected_final_response"] == first_case["expected_text"] + assert set(first_case["key_trace"]) == { + "invocation_id", + "actual_final_response", + "expected_final_response", + "error_message", + } + module.validate_report_schema(report) + assert (run_dir / "optimization_report.md").is_file() + + +def _assert_candidate_audit(candidate: dict[str, Any], seed: int) -> None: + audit = candidate["audit"] + assert audit["seed"] == seed + assert audit["duration_seconds"] >= 0 + assert audit["cost"]["currency"] == "USD" + assert audit["config_sha256"] + assert len(audit["config_sha256"]) == 64 + + +@pytest.mark.asyncio +async def test_fake_mode_audits_each_candidate_independently(tmp_path: Path): + module = load_pipeline_module() + run_dir = await module.run_fake_or_trace( + mode="fake", + seed=7, + output_dir=tmp_path, + run_id="candidate_audit", + ) + report = load_report(run_dir / "optimization_report.json") + + assert report["optimization_rounds"] == [] + for candidate in report["candidates"]: + _assert_candidate_audit(candidate, 7) + assert Path(candidate["artifacts"]["prompt_dir"]).is_dir() + assert Path(candidate["artifacts"]["prompt_patch"]).is_file() + + +@pytest.mark.asyncio +async def test_markdown_includes_rejected_candidate_delta_types_absent_from_winner(tmp_path: Path): + module = load_pipeline_module() + run_dir = await module.run_fake_or_trace( + mode="fake", + seed=7, + output_dir=tmp_path, + run_id="markdown_case_delta_parity", + ) + report = load_report(run_dir / "optimization_report.json") + markdown = (run_dir / "optimization_report.md").read_text(encoding="utf-8") + winner = next( + candidate for candidate in report["candidates"] if candidate["id"] == report["gate_decision"]["winner"] + ) + rejected = next(candidate for candidate in report["candidates"] if candidate["id"] == "candidate_overfit") + rejected_types = {item["change_type"] for item in rejected["case_deltas"]} + winner_types = {item["change_type"] for item in winner["case_deltas"]} + + assert rejected["gate"]["accepted"] is False + assert rejected_types - winner_types == {"new_fail"} + for candidate in report["candidates"]: + header = f"## Validation Case Delta: `{candidate['id']}`" + assert header in markdown + section = markdown.split(header, 1)[1].split("## ", 1)[0] + for item in candidate["case_deltas"]: + assert f"`{item['case_id']}`" in section + assert f"change_type `{item['change_type']}`" in section + + +@pytest.mark.asyncio +async def test_fake_mode_report_scores_come_from_agent_evaluator( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + calls = patch_agent_evaluator(monkeypatch, score=0.25, passed=False) + module = load_pipeline_module() + + run_dir = await module.run_fake_or_trace( + mode="fake", + seed=7, + output_dir=tmp_path, + run_id="fake_evaluator_backed", + ) + report = load_report(run_dir / "optimization_report.json") + + assert calls, "fake mode must run AgentEvaluator, not direct fixture scoring" + assert report["baseline"]["validation"]["score"] == pytest.approx(0.25) + first_case = report["baseline"]["validation"]["case_results"][0] + assert first_case["metrics"][ROUTE_TOOL_ARGS_METRIC]["score"] == pytest.approx(0.25) + + +@pytest.mark.asyncio +async def test_route_tool_argument_metric_ignores_reason_text(tmp_path: Path): + module = load_pipeline_module() + payload = load_report(EXAMPLE_DIR / "train.evalset.json") + payload["eval_set_id"] = "reason_wording_regression" + payload["eval_cases"] = [payload["eval_cases"][0]] + evalset_path = tmp_path / "reason_wording.evalset.json" + evalset_path.write_text(json.dumps(payload), encoding="utf-8") + metrics_path = module.offline_metrics_path(tmp_path) + + async def call_agent(_: str) -> str: + return ( + '{"route":"refund","tool":{"name":"create_refund_ticket","arguments":{}},' + '"reason":"A different but harmless explanation."}' + ) + + summary = await module.run_evaluator( + evalset_path=evalset_path, + evalset_payload=payload, + metrics_path=metrics_path, + call_agent=call_agent, + offline_rubric=True, + ) + + assert summary["score"] == pytest.approx(1.0) + assert summary["case_results"][0]["metrics"][ROUTE_TOOL_ARGS_METRIC]["passed"] is True + + +def test_gate_rejects_noop_and_overfit_candidates(tmp_path: Path): + module = load_pipeline_module() + started = module.time.perf_counter() + report = module.make_report( + mode="fake", + run_id="gate_unit", + run_dir=tmp_path, + seed=7, + started=started, + ) + by_id = {candidate["id"]: candidate for candidate in report["candidates"]} + + assert by_id["candidate_local_patch"]["gate"]["accepted"] is True + assert by_id["candidate_noop"]["gate"]["accepted"] is False + assert "validation score did not improve" in " ".join(by_id["candidate_noop"]["gate"]["reasons"]) + assert by_id["candidate_overfit"]["gate"]["accepted"] is False + overfit_reasons = " ".join(by_id["candidate_overfit"]["gate"]["reasons"]) + assert "hard fail" in overfit_reasons + assert "critical case" in overfit_reasons + + +@pytest.mark.asyncio +async def test_trace_mode_uses_replay_without_api_keys(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv("TRPC_AGENT_API_KEY", raising=False) + monkeypatch.delenv("TRPC_AGENT_BASE_URL", raising=False) + monkeypatch.delenv("TRPC_AGENT_MODEL_NAME", raising=False) + + module = load_pipeline_module() + run_dir = await module.run_fake_or_trace( + mode="trace", + seed=7, + output_dir=tmp_path, + run_id="trace_case", + ) + report = load_report(run_dir / "optimization_report.json") + assert report["mode"] == "trace" + assert report["gate_decision"]["winner"] == "candidate_local_patch" + module.validate_report_schema(report) + assert (run_dir / "trace_evalset.json").is_file() + assert (run_dir / "trace_metrics.json").is_file() + trace_payload = load_report(run_dir / "trace_evalset.json") + assert all(case["eval_mode"] == "trace" for case in trace_payload["eval_cases"]) + assert report["artifacts"]["fixtures"].endswith("trace_outputs.json") + + +@pytest.mark.asyncio +async def test_trace_mode_consumes_trace_fixture_payload(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + module = load_pipeline_module() + trace_fixtures = load_report(EXAMPLE_DIR / "fixtures" / "fake_outputs.json") + trace_marker = '{"route":"faq","tool":{"name":"none","arguments":{}},' '"reason":"TRACE FIXTURE MARKER"}' + trace_fixtures["candidate_local_patch"]["outputs"]["val_address_change_102"] = trace_marker + trace_fixture_path = tmp_path / "trace_outputs.json" + trace_fixture_path.write_text(json.dumps(trace_fixtures), encoding="utf-8") + monkeypatch.setattr(module, "TRACE_FIXTURE_PATH", trace_fixture_path, raising=False) + + run_dir = await module.run_fake_or_trace( + mode="trace", + seed=7, + output_dir=tmp_path, + run_id="trace_fixture_source", + ) + report = load_report(run_dir / "optimization_report.json") + candidate = next(item for item in report["candidates"] if item["id"] == "candidate_local_patch") + actual_by_id = {item["case_id"]: item["actual_text"] for item in candidate["validation"]["case_results"]} + + assert report["artifacts"]["fixtures"] == str(trace_fixture_path.resolve()) + assert actual_by_id["val_address_change_102"] == trace_marker + + +@pytest.mark.asyncio +async def test_trace_mode_evaluates_baseline_and_each_candidate( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + calls = patch_agent_evaluator(monkeypatch) + module = load_pipeline_module() + + await module.run_fake_or_trace( + mode="trace", + seed=7, + output_dir=tmp_path, + run_id="trace_all_candidates", + ) + + assert len(calls) == 12 + + +def test_cli_fake_mode_runs_end_to_end(tmp_path: Path): + proc = subprocess.run( + [ + sys.executable, + str(RUN_PIPELINE), + "--mode", + "fake", + "--output-dir", + str(tmp_path), + "--run-id", + "cli_fake", + ], + check=True, + capture_output=True, + text=True, + ) + run_dir = Path(proc.stdout.strip().splitlines()[-1]) + assert run_dir == tmp_path / "cli_fake" + report = load_report(run_dir / "optimization_report.json") + assert report["gate_decision"]["winner"] == "candidate_local_patch" + + +def test_cli_trace_mode_defaults_to_trace_outputs(tmp_path: Path): + proc = subprocess.run( + [ + sys.executable, + str(RUN_PIPELINE), + "--mode", + "trace", + "--output-dir", + str(tmp_path), + "--run-id", + "cli_trace_fixture", + ], + check=True, + capture_output=True, + text=True, + ) + run_dir = Path(proc.stdout.strip().splitlines()[-1]) + report = load_report(run_dir / "optimization_report.json") + + assert report["artifacts"]["fixtures"].endswith("trace_outputs.json") + + +@pytest.mark.asyncio +async def test_gate_config_can_require_larger_validation_delta(tmp_path: Path): + module = load_pipeline_module() + run_dir = await module.run_fake_or_trace( + mode="fake", + seed=7, + output_dir=tmp_path, + run_id="strict_gate", + gate_config={"min_validation_delta": 0.5}, + ) + report = load_report(run_dir / "optimization_report.json") + assert report["gate_decision"]["accepted"] is False + assert report["gate_decision"]["winner"] is None + reasons = " ".join(report["gate_decision"]["reasons"]) + assert "validation score improvement" in reasons + + +def test_default_gate_inherits_required_metrics_from_optimizer_config(): + module = load_pipeline_module() + + gate = module.load_gate_config(optimizer_config=EXAMPLE_DIR / "optimizer.json") + + assert gate["required_metrics"] == [ + ROUTE_TOOL_ARGS_METRIC, + "llm_rubric_response", + ] + assert gate["required_metrics_source"] == "optimizer_config" + + +def test_required_metric_failure_rejects_even_when_primary_score_improves(): + module = load_pipeline_module() + baseline_val = { + "score": 0.5, + "metrics": { + ROUTE_TOOL_ARGS_METRIC: {"passed": False}, + "llm_rubric_response": {"passed": True}, + }, + "case_results": [ + {"case_id": "case_1", "score": 0.5, "passed": False, "tags": []}, + ], + } + candidate_val = { + "score": 1.0, + "metrics": { + ROUTE_TOOL_ARGS_METRIC: {"passed": True}, + "llm_rubric_response": {"passed": False}, + }, + "case_results": [ + {"case_id": "case_1", "score": 1.0, "passed": True, "tags": []}, + ], + } + + gate = module.apply_gate( + candidate_id="candidate", + baseline_val=baseline_val, + candidate_val=candidate_val, + gate_config=module.load_gate_config(optimizer_config=EXAMPLE_DIR / "optimizer.json"), + duration_seconds=0.01, + cost_usd=0.0, + ) + + assert gate["accepted"] is False + assert "llm_rubric_response" in " ".join(gate["reasons"]) + + +def test_validation_regression_is_rejected_even_without_hard_fail(): + module = load_pipeline_module() + baseline_val = { + "score": 0.75, + "metrics": {ROUTE_TOOL_ARGS_METRIC: {"passed": True}}, + "case_results": [ + {"case_id": "case_1", "score": 0.75, "passed": True, "tags": []}, + ], + } + candidate_val = { + "score": 0.5, + "metrics": {ROUTE_TOOL_ARGS_METRIC: {"passed": True}}, + "case_results": [ + {"case_id": "case_1", "score": 0.5, "passed": True, "tags": []}, + ], + } + + gate = module.apply_gate( + candidate_id="candidate", + baseline_val=baseline_val, + candidate_val=candidate_val, + gate_config={"required_metrics": [ROUTE_TOOL_ARGS_METRIC]}, + duration_seconds=0.01, + cost_usd=0.0, + ) + + assert gate["accepted"] is False + assert "validation score did not improve" in " ".join(gate["reasons"]) + + +def test_failure_attribution_taxonomy_handles_parameter_format_and_rubric_failures(): + module = load_pipeline_module() + + parameter = module.attribute_failure_case( + actual_text='{"route":"refund","tool":{"name":"create_refund_ticket","arguments":{"unexpected":true}}}', + expected_text='{"route":"refund","tool":{"name":"create_refund_ticket","arguments":{}}}', + error_message=None, + metrics={ROUTE_TOOL_ARGS_METRIC: {"passed": False}}, + ) + assert parameter["root_cause"] == "parameter_error" + assert parameter["reasons"] + + formatted = module.attribute_failure_case( + actual_text="not json", + expected_text='{"route":"faq","tool":{"name":"none","arguments":{}}}', + error_message=None, + metrics={ROUTE_TOOL_ARGS_METRIC: {"passed": False}}, + ) + assert formatted["root_cause"] == "format_error" + + rubric = module.attribute_failure_case( + actual_text='{"route":"faq","tool":{"name":"none","arguments":{}}}', + expected_text='{"route":"faq","tool":{"name":"none","arguments":{}}}', + error_message=None, + metrics={ + ROUTE_TOOL_ARGS_METRIC: {"passed": True}, + "llm_rubric_response": {"passed": False}, + }, + ) + assert rubric["root_cause"] == "rubric_failed" + + +@pytest.mark.parametrize( + ("actual_text", "expected_root"), + [ + ( + '{"route":"faq","tool":"none","reason":"bad shape"}', + "tool_call_error", + ), + ( + '{"route":"faq","tool":{"name":"none","arguments":null},"reason":"bad args"}', + "parameter_error", + ), + ( + '{"route":"faq","tool":{"name":"none","arguments":[]},"reason":"bad args"}', + "parameter_error", + ), + ( + '{"route":"faq","tool":{"name":"none"},"reason":"missing args"}', + "parameter_error", + ), + ], +) +def test_failure_attribution_is_total_for_malformed_tool_shapes( + actual_text: str, + expected_root: str, +): + module = load_pipeline_module() + result = module.attribute_failure_case( + actual_text=actual_text, + expected_text=('{"route":"faq","tool":{"name":"none","arguments":{}},' '"reason":"expected"}'), + error_message=None, + metrics={ROUTE_TOOL_ARGS_METRIC: {"passed": False}}, + ) + assert result["root_cause"] == expected_root + assert result["reasons"] + + +def test_gate_rejects_when_configured_cost_budget_cannot_be_evaluated(): + module = load_pipeline_module() + baseline_val = { + "score": 0.5, + "metrics": {ROUTE_TOOL_ARGS_METRIC: {"passed": False}}, + "case_results": [ + {"case_id": "case_1", "score": 0.5, "passed": False, "tags": []}, + ], + } + candidate_val = { + "score": 1.0, + "metrics": {ROUTE_TOOL_ARGS_METRIC: {"passed": True}}, + "case_results": [ + {"case_id": "case_1", "score": 1.0, "passed": True, "tags": []}, + ], + } + + gate = module.apply_gate( + candidate_id="candidate", + baseline_val=baseline_val, + candidate_val=candidate_val, + gate_config={"required_metrics": [ROUTE_TOOL_ARGS_METRIC], "max_cost_usd": 0.01}, + duration_seconds=0.01, + cost_usd=None, + ) + + assert gate["accepted"] is False + assert "cost budget could not be evaluated" in " ".join(gate["reasons"]) + + +def test_cli_accepts_custom_paths(tmp_path: Path): + custom_dir = tmp_path / "inputs" + custom_dir.mkdir() + train = custom_dir / "train.copy.evalset.json" + val = custom_dir / "val.copy.evalset.json" + optimizer_dev = custom_dir / "optimizer_dev.copy.evalset.json" + optimizer = custom_dir / "optimizer.copy.json" + system_prompt = custom_dir / "system.md" + router_prompt = custom_dir / "router.md" + shutil.copy2(EXAMPLE_DIR / "train.evalset.json", train) + shutil.copy2(EXAMPLE_DIR / "val.evalset.json", val) + shutil.copy2(EXAMPLE_DIR / "optimizer_dev.evalset.json", optimizer_dev) + shutil.copy2(EXAMPLE_DIR / "optimizer.json", optimizer) + shutil.copy2(EXAMPLE_DIR / "agent" / "prompts" / "system.md", system_prompt) + shutil.copy2(EXAMPLE_DIR / "agent" / "prompts" / "router.md", router_prompt) + + proc = subprocess.run( + [ + sys.executable, + str(RUN_PIPELINE), + "--mode", + "fake", + "--train-evalset", + str(train), + "--val-evalset", + str(val), + "--optimizer-dev-evalset", + str(optimizer_dev), + "--optimizer-config", + str(optimizer), + "--system-prompt", + str(system_prompt), + "--router-prompt", + str(router_prompt), + "--output-dir", + str(tmp_path), + "--run-id", + "custom_paths", + ], + check=True, + capture_output=True, + text=True, + ) + run_dir = Path(proc.stdout.strip().splitlines()[-1]) + report = load_report(run_dir / "optimization_report.json") + assert report["artifacts"]["train_evalset"] == str(train) + assert report["artifacts"]["validation_evalset"] == str(val) + assert report["artifacts"]["optimizer_dev_evalset"] == str(optimizer_dev) + assert report["artifacts"]["optimizer_config"] == str(optimizer) + + +@pytest.mark.asyncio +async def test_run_evaluator_propagates_unrelated_assertion_errors( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + module = load_pipeline_module() + metrics_path = module.offline_metrics_path(tmp_path) + + class BrokenExecuter: + async def evaluate(self) -> None: + raise AssertionError("metric configuration is broken") + + def get_result(self): + return make_evaluate_result(EXAMPLE_DIR / "train.evalset.json") + + def fake_get_executer(*_: Any, **__: Any) -> BrokenExecuter: + return BrokenExecuter() + + import trpc_agent_sdk.evaluation as evaluation_pkg + + monkeypatch.setattr(evaluation_pkg.AgentEvaluator, "get_executer", staticmethod(fake_get_executer)) + + with pytest.raises(AssertionError, match="metric configuration is broken"): + await module.run_evaluator( + evalset_path=EXAMPLE_DIR / "train.evalset.json", + evalset_payload=load_report(EXAMPLE_DIR / "train.evalset.json"), + metrics_path=metrics_path, + ) + + +@pytest.mark.asyncio +async def test_fake_mode_records_prompt_artifacts(tmp_path: Path): + module = load_pipeline_module() + run_dir = await module.run_fake_or_trace( + mode="fake", + seed=7, + output_dir=tmp_path, + run_id="prompt_audit", + ) + report = load_report(run_dir / "optimization_report.json") + + all_artifacts = list(report["baseline"]["prompt_artifacts"]) + for candidate in report["candidates"]: + all_artifacts.extend(candidate["prompt_artifacts"]) + assert Path(candidate["artifacts"]["prompt_patch"]).is_file() + + expected_count = 2 * (1 + len(report["candidates"])) + assert len(all_artifacts) == expected_count + + for prompt_artifact in all_artifacts: + assert prompt_artifact["name"] in {"system_prompt", "router_prompt"} + assert Path(prompt_artifact["source_path"]).is_file() + assert Path(prompt_artifact["candidate_path"]).is_file() + assert len(prompt_artifact["sha256"]) == 64 + assert prompt_artifact["source_written"] is False + assert prompt_artifact["summary"] + assert "diff" in prompt_artifact + + +def test_online_preflight_reports_presence_without_secret(monkeypatch: pytest.MonkeyPatch): + module = load_pipeline_module() + monkeypatch.setenv("TRPC_AGENT_API_KEY", "sk-secret-value") + monkeypatch.setenv("TRPC_AGENT_BASE_URL", "https://example.invalid/v1") + monkeypatch.setenv("TRPC_AGENT_MODEL_NAME", "example-model") + + preflight = module.online_preflight() + text = module.format_online_preflight(preflight) + + assert preflight == { + "TRPC_AGENT_API_KEY": True, + "TRPC_AGENT_BASE_URL": True, + "TRPC_AGENT_MODEL_NAME": True, + } + assert "sk-secret-value" not in text + assert "TRPC_AGENT_API_KEY=present" in text + + +@pytest.mark.asyncio +async def test_online_mode_missing_env_fails_before_api_call(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv("TRPC_AGENT_API_KEY", raising=False) + monkeypatch.delenv("TRPC_AGENT_BASE_URL", raising=False) + monkeypatch.delenv("TRPC_AGENT_MODEL_NAME", raising=False) + module = load_pipeline_module() + + with pytest.raises(ValueError) as exc_info: + await module.run_online(seed=7, output_dir=tmp_path, run_id="online_missing_env") + message = str(exc_info.value) + assert "online mode requires environment variables" in message + assert "TRPC_AGENT_API_KEY" in message + + +@pytest.mark.asyncio +async def test_online_rejects_malformed_optimizer_required_metrics_before_optimizer_call( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setenv("TRPC_AGENT_API_KEY", "fake-key") + monkeypatch.setenv("TRPC_AGENT_BASE_URL", "http://localhost/fake") + monkeypatch.setenv("TRPC_AGENT_MODEL_NAME", "fake-model") + module = load_pipeline_module() + payload = load_report(EXAMPLE_DIR / "optimizer.json") + payload["optimize"]["stop"]["required_metrics"] = "metric" + optimizer_config = tmp_path / "malformed_optimizer.json" + optimizer_config.write_text(json.dumps(payload), encoding="utf-8") + called = False + + async def fail_if_called(**_: Any): + nonlocal called + called = True + raise AssertionError("optimizer must not be called") + + import trpc_agent_sdk.evaluation as evaluation_pkg + + monkeypatch.setattr(evaluation_pkg.AgentOptimizer, "optimize", staticmethod(fail_if_called)) + + with pytest.raises(ValueError, match="required_metrics"): + await module.run_online( + seed=7, + output_dir=tmp_path, + run_id="malformed_optimizer_config", + optimizer_config=optimizer_config, + ) + + assert called is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize("raise_during_run", [False, True]) +async def test_online_call_agent_closes_runner( + monkeypatch: pytest.MonkeyPatch, + raise_during_run: bool, +): + module = load_pipeline_module() + closed: list[bool] = [] + + class FakeRunner: + def __init__(self, **kwargs: Any): + pass + + async def run_async(self, **kwargs: Any): + if raise_during_run: + raise RuntimeError("model stream failed") + if False: + yield None + + async def close(self): + closed.append(True) + + import trpc_agent_sdk.runners as runners + + monkeypatch.setattr(runners, "Runner", FakeRunner) + monkeypatch.setattr(module, "_make_llm_agent_from_prompts", lambda prompt_texts: object()) + call_agent = module.make_online_call_agent( + system_prompt=EXAMPLE_DIR / "agent" / "prompts" / "system.md", + router_prompt=EXAMPLE_DIR / "agent" / "prompts" / "router.md", + ) + + if raise_during_run: + with pytest.raises(RuntimeError, match="model stream failed"): + await call_agent("hello") + else: + assert await call_agent("hello") == "" + + assert closed == [True] + + +@pytest.mark.asyncio +async def test_online_call_agent_closes_runner_when_session_creation_fails( + monkeypatch: pytest.MonkeyPatch, +): + module = load_pipeline_module() + closed: list[bool] = [] + + class FakeRunner: + def __init__(self, **kwargs: Any): + pass + + async def close(self): + closed.append(True) + + class FailingSessionService: + async def create_session(self, **kwargs: Any): + raise RuntimeError("session creation failed") + + import trpc_agent_sdk.runners as runners + import trpc_agent_sdk.sessions as sessions + + monkeypatch.setattr(runners, "Runner", FakeRunner) + monkeypatch.setattr(sessions, "InMemorySessionService", FailingSessionService) + monkeypatch.setattr(module, "_make_llm_agent_from_prompts", lambda prompt_texts: object()) + call_agent = module.make_online_call_agent( + system_prompt=EXAMPLE_DIR / "agent" / "prompts" / "system.md", + router_prompt=EXAMPLE_DIR / "agent" / "prompts" / "router.md", + ) + + with pytest.raises(RuntimeError, match="session creation failed"): + await call_agent("hello") + + assert closed == [True] + + +@pytest.mark.asyncio +async def test_online_call_agent_preserves_stream_error_when_close_fails( + monkeypatch: pytest.MonkeyPatch, +): + module = load_pipeline_module() + closed: list[bool] = [] + cleanup_messages: list[str] = [] + + class FakeRunner: + def __init__(self, **kwargs: Any): + pass + + async def run_async(self, **kwargs: Any): + raise RuntimeError("model stream failed") + yield None + + async def close(self): + closed.append(True) + raise RuntimeError("runner close failed") + + class FakeLogger: + def exception(self, message: str): + cleanup_messages.append(message) + + import trpc_agent_sdk.runners as runners + + monkeypatch.setattr(runners, "Runner", FakeRunner) + monkeypatch.setattr(module, "logger", FakeLogger(), raising=False) + monkeypatch.setattr(module, "_make_llm_agent_from_prompts", lambda prompt_texts: object()) + call_agent = module.make_online_call_agent( + system_prompt=EXAMPLE_DIR / "agent" / "prompts" / "system.md", + router_prompt=EXAMPLE_DIR / "agent" / "prompts" / "router.md", + ) + + with pytest.raises(RuntimeError, match="model stream failed"): + await call_agent("hello") + + assert closed == [True] + assert cleanup_messages == ["Failed to close online evaluation runner after a primary error."] + + +@pytest.mark.asyncio +async def test_online_call_agent_surfaces_close_failure_after_success( + monkeypatch: pytest.MonkeyPatch, +): + module = load_pipeline_module() + closed: list[bool] = [] + + class FakeRunner: + def __init__(self, **kwargs: Any): + pass + + async def run_async(self, **kwargs: Any): + if False: + yield None + + async def close(self): + closed.append(True) + raise RuntimeError("runner close failed") + + import trpc_agent_sdk.runners as runners + + monkeypatch.setattr(runners, "Runner", FakeRunner) + monkeypatch.setattr(module, "_make_llm_agent_from_prompts", lambda prompt_texts: object()) + call_agent = module.make_online_call_agent( + system_prompt=EXAMPLE_DIR / "agent" / "prompts" / "system.md", + router_prompt=EXAMPLE_DIR / "agent" / "prompts" / "router.md", + ) + + with pytest.raises(RuntimeError, match="runner close failed"): + await call_agent("hello") + + assert closed == [True] + + +def test_online_warning_observability_is_not_suppressed(): + from trpc_agent_sdk.models.openai_adapter import _deepseek + from trpc_agent_sdk.types import GenerateContentConfig + + sse_warning = "coroutine method 'aclose' of 'SSEDecoder._aiter_chunks' was never awaited" + with patch.object(_deepseek.logger, "warning") as warning: + handled, response_format = _deepseek.DeepSeekAdapter("deepseek-v4-flash").build_response_format( + GenerateContentConfig( + response_mime_type="application/json", + response_json_schema={"type": "object"}, + ) + ) + + assert handled is True + assert response_format == {"type": "json_object"} + warning.assert_called_once_with("DeepSeek only supports JSON object response_format; response schema is ignored.") + assert not any( + action == "ignore" and issubclass(RuntimeWarning, category) and (message is None or message.search(sse_warning)) + for action, message, category, _module, _line in warnings.filters + ) + with pytest.warns(RuntimeWarning, match=re.escape(sse_warning)): + warnings.warn(sse_warning, RuntimeWarning, stacklevel=1) + + +@pytest.mark.asyncio +async def test_online_mode_can_construct_optimizer_call_without_real_api( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setenv("TRPC_AGENT_API_KEY", "fake-key") + monkeypatch.setenv("TRPC_AGENT_BASE_URL", "http://localhost/fake") + monkeypatch.setenv("TRPC_AGENT_MODEL_NAME", "fake-model") + + module = load_pipeline_module() + captured: dict[str, Any] = {} + + class FakeResult: + status = "SUCCEEDED" + baseline_pass_rate = 0.5 + best_pass_rate = 1.0 + pass_rate_improvement = 0.5 + stop_reason = "completed" + total_llm_cost = 0.0 + total_reflection_lm_calls = 2 + total_judge_model_calls = 3 + best_prompts = { + "system_prompt": "fake system", + "router_prompt": "fake router", + } + baseline_prompts = { + "system_prompt": "baseline system", + "router_prompt": "baseline router", + } + baseline_metric_breakdown = {ROUTE_TOOL_ARGS_METRIC: 0.5} + best_metric_breakdown = {ROUTE_TOOL_ARGS_METRIC: 1.0} + metric_thresholds = {ROUTE_TOOL_ARGS_METRIC: 1.0} + duration_seconds = 0.01 + total_token_usage = {"prompt": 8, "completion": 2, "total": 10} + + async def fake_optimize(**kwargs): + captured.update(kwargs) + output_dir = Path(kwargs["output_dir"]) + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "result.json").write_text("{}", encoding="utf-8") + (output_dir / "summary.txt").write_text("fake", encoding="utf-8") + return FakeResult() + + import trpc_agent_sdk.evaluation as evaluation_pkg + + monkeypatch.setattr(evaluation_pkg.AgentOptimizer, "optimize", staticmethod(fake_optimize)) + patch_agent_evaluator(monkeypatch) + run_dir = await module.run_online(seed=42, output_dir=tmp_path, run_id="online_wiring") + + assert captured["config_path"].endswith("optimizer.json") + captured_config = load_report(Path(captured["config_path"])) + assert captured_config["optimize"]["algorithm"]["seed"] == 42 + assert captured["train_dataset_path"].endswith("train.evalset.json") + assert captured["validation_dataset_path"].endswith("optimizer_dev.evalset.json") + assert not captured["validation_dataset_path"].endswith("val.evalset.json") + assert captured["update_source"] is False + assert sorted(captured["target_prompt"].names()) == ["router_prompt", "system_prompt"] + report = load_report(run_dir / "optimization_report.json") + assert report["mode"] == "online" + assert report["online_result"]["status"] == "SUCCEEDED" + assert report["artifacts"]["optimizer_dev_evalset"].endswith("optimizer_dev.evalset.json") + assert report["artifacts"]["final_validation_evalset"].endswith("val.evalset.json") + assert report["optimization_rounds"] == [] + _assert_candidate_audit(report["candidates"][0], 42) + for name, value in report["artifacts"].items(): + if name.startswith("native_") and value: + assert Path(value).exists(), name + assert report["environment_snapshot"]["model_name"] == "fake-model" + assert report["environment_snapshot"]["base_url_host"] == "localhost" + assert report["environment_snapshot"]["command"] + module.validate_report_schema(report) + assert report["cost"]["estimated_total"] is None + assert report["cost"]["cost_source"] == "unknown" + assert report["cost"]["optimizer"]["model_calls"] == 5 + assert report["cost"]["optimizer"]["native_judge_model_calls"] == 3 + assert report["cost"]["optimizer"]["judge_model_call_source"] == "native_optimizer_counter" + assert report["cost"]["optimizer"]["token_usage"] is None + assert report["cost"]["optimizer"]["reflection_reported_usage"]["token_usage"]["total"] == 10 + assert report["cost"]["final_revalidation"]["model_calls"] > 0 + assert report["cost"]["token_usage"] is None + assert report["cost"]["token_usage_known"] is False + assert report["cost"]["optimizer"]["token_usage_known"] is False + assert report["cost"]["final_revalidation"]["token_usage"] is None + assert report["cost"]["final_revalidation"]["token_usage_known"] is False + assert report["cost"]["model_calls"] == ( + report["cost"]["optimizer"]["model_calls"] + report["cost"]["final_revalidation"]["model_calls"] + ) + + +@pytest.mark.asyncio +async def test_online_failed_optimizer_preserves_partial_best_prompt_report( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setenv("TRPC_AGENT_API_KEY", "fake-key") + monkeypatch.setenv("TRPC_AGENT_BASE_URL", "http://localhost/fake") + monkeypatch.setenv("TRPC_AGENT_MODEL_NAME", "fake-model") + module = load_pipeline_module() + + class FakeResult: + status = "FAILED" + error_message = "optimizer provider failed" + baseline_pass_rate = 0.5 + best_pass_rate = 0.5 + pass_rate_improvement = 0.0 + stop_reason = None + total_llm_cost = 0.0 + total_reflection_lm_calls = 0 + total_judge_model_calls = 0 + best_prompts = {"system_prompt": "partial system prompt"} + baseline_prompts = {} + baseline_metric_breakdown = {ROUTE_TOOL_ARGS_METRIC: 0.5} + best_metric_breakdown = {ROUTE_TOOL_ARGS_METRIC: 0.5} + metric_thresholds = {ROUTE_TOOL_ARGS_METRIC: 1.0} + duration_seconds = 0.01 + total_token_usage = {"prompt": 0, "completion": 0, "total": 0} + rounds: list[Any] = [] + + async def fake_optimize(**kwargs: Any) -> FakeResult: + output_dir = Path(kwargs["output_dir"]) + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "result.json").write_text("{}", encoding="utf-8") + return FakeResult() + + import trpc_agent_sdk.evaluation as evaluation_pkg + + monkeypatch.setattr(evaluation_pkg.AgentOptimizer, "optimize", staticmethod(fake_optimize)) + patch_agent_evaluator(monkeypatch, score=0.5, passed=False) + + run_dir = await module.run_online(seed=7, output_dir=tmp_path, run_id="online_failed_partial") + report = load_report(run_dir / "optimization_report.json") + candidate = report["candidates"][0] + router_artifact = next( + artifact for artifact in candidate["prompt_artifacts"] if artifact["name"] == "router_prompt" + ) + + assert report["online_result"]["status"] == "FAILED" + assert report["online_result"]["error_message"] == "optimizer provider failed" + assert report["gate_decision"]["accepted"] is False + assert "optimizer provider failed" in " ".join(candidate["gate"]["reasons"]) + assert Path(router_artifact["candidate_path"]).read_text(encoding="utf-8") == ( + EXAMPLE_DIR / "agent" / "prompts" / "router.md" + ).read_text(encoding="utf-8") + _assert_candidate_audit(candidate, 7) + + +@pytest.mark.asyncio +async def test_online_round_audit_writes_native_prompt_artifacts(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("TRPC_AGENT_API_KEY", "fake-key") + monkeypatch.setenv("TRPC_AGENT_BASE_URL", "http://localhost/fake") + monkeypatch.setenv("TRPC_AGENT_MODEL_NAME", "fake-model") + module = load_pipeline_module() + + round_prompt = "optimized system prompt" + + class FakeResult: + status = "SUCCEEDED" + error_message = "" + baseline_pass_rate = 0.5 + best_pass_rate = 1.0 + pass_rate_improvement = 0.5 + stop_reason = "completed" + total_llm_cost = 0.01 + total_reflection_lm_calls = 1 + total_judge_model_calls = 0 + best_prompts = {"system_prompt": round_prompt, "router_prompt": "optimized router prompt"} + baseline_prompts = {} + baseline_metric_breakdown = {ROUTE_TOOL_ARGS_METRIC: 0.5} + best_metric_breakdown = {ROUTE_TOOL_ARGS_METRIC: 1.0} + metric_thresholds = {ROUTE_TOOL_ARGS_METRIC: 1.0} + duration_seconds = 0.01 + total_token_usage = {"prompt": 8, "completion": 2, "total": 10} + rounds = [ + SimpleNamespace( + round=1, + optimized_field_names=["system_prompt"], + candidate_prompts={"system_prompt": round_prompt}, + validation_pass_rate=1.0, + metric_breakdown={ROUTE_TOOL_ARGS_METRIC: 1.0}, + accepted=True, + acceptance_reason="improved validation", + skip_reason=None, + error_message=None, + failed_case_ids=[], + round_llm_cost=0.01, + round_token_usage={"prompt": 8, "completion": 2, "total": 10}, + duration_seconds=0.25, + ) + ] + + async def fake_optimize(**kwargs: Any) -> FakeResult: + output_dir = Path(kwargs["output_dir"]) + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "result.json").write_text("{}", encoding="utf-8") + (output_dir / "summary.txt").write_text("complete", encoding="utf-8") + (output_dir / "rounds").mkdir() + return FakeResult() + + import trpc_agent_sdk.evaluation as evaluation_pkg + + monkeypatch.setattr(evaluation_pkg.AgentOptimizer, "optimize", staticmethod(fake_optimize)) + patch_agent_evaluator(monkeypatch, score=0.5, passed=False) + + run_dir = await module.run_online(seed=7, output_dir=tmp_path, run_id="online_round_audit") + report = load_report(run_dir / "optimization_report.json") + round_record = report["optimization_rounds"][0] + prompt_path = Path(round_record["prompt_paths"]["system_prompt"]) + + assert round_record["round"] == 1 + assert round_record["optimized_field_names"] == ["system_prompt"] + assert prompt_path.is_file() + assert prompt_path.read_text(encoding="utf-8") == round_prompt + assert round_record["prompt_sha256"]["system_prompt"] == module.sha256_text(round_prompt) + assert round_record["accepted"] is True + assert round_record["decision_reason"] == "improved validation" + assert Path(report["artifacts"]["native_rounds_dir"]).is_dir() + + +def test_optimizer_round_audit_redacts_and_rejects_nonfinite_numeric_evidence(tmp_path: Path): + module = load_pipeline_module() + round_prompt = "round prompt" + records = module.write_optimizer_round_artifacts( + run_dir=tmp_path, + rounds=[ + SimpleNamespace( + round=1, + optimized_field_names=["system_prompt"], + candidate_prompts={"system_prompt": round_prompt}, + validation_pass_rate=float("nan"), + metric_breakdown={ + "finite_metric": 0.5, + "infinite_metric": float("inf"), + "negative_infinite_metric": float("-inf"), + }, + accepted=True, + acceptance_reason="", + skip_reason=None, + error_message="optimizer failed: Authorization: Bearer round-secret", + failed_case_ids=[], + round_llm_cost=float("inf"), + round_token_usage={ + "prompt": float("nan"), + "completion": float("inf"), + "total": float("-inf"), + }, + duration_seconds=float("-inf"), + ) + ], + ) + record = records[0] + + serialized = json.dumps(records, allow_nan=False) + assert "round-secret" not in serialized + assert "Authorization" not in serialized + assert "Bearer" not in serialized + assert record["accepted"] is False + assert "invalid numeric round evidence" in record["decision_reason"] + assert record["validation_pass_rate"] == 0.0 + assert record["metric_breakdown"] == { + "finite_metric": 0.5, + "infinite_metric": 0.0, + "negative_infinite_metric": 0.0, + } + assert record["cost_usd"] == 0.0 + assert record["duration_seconds"] == 0.0 + assert record["token_usage"] == {"prompt": 0, "completion": 0, "total": 0} + assert record["prompt_sha256"]["system_prompt"] == module.sha256_text(round_prompt) + assert Path(record["prompt_paths"]["system_prompt"]).read_text(encoding="utf-8") == round_prompt + + +def test_optimizer_round_audit_normalizes_nonfinite_round_id(tmp_path: Path): + module = load_pipeline_module() + round_prompt = "round prompt" + records = module.write_optimizer_round_artifacts( + run_dir=tmp_path, + rounds=[ + SimpleNamespace( + round=float("nan"), + optimized_field_names=["system_prompt"], + candidate_prompts={"system_prompt": round_prompt}, + validation_pass_rate=0.5, + metric_breakdown={ROUTE_TOOL_ARGS_METRIC: 0.5}, + accepted=True, + acceptance_reason="improved validation", + skip_reason=None, + error_message=None, + failed_case_ids=[], + round_llm_cost=0.01, + round_token_usage={"prompt": 8, "completion": 2, "total": 10}, + duration_seconds=0.25, + ) + ], + ) + record = records[0] + + json.dumps(records, allow_nan=False) + assert isinstance(record["round"], int) + assert record["round"] > 0 + assert record["accepted"] is False + assert "invalid round identifier" in record["decision_reason"] + assert record["prompt_sha256"]["system_prompt"] == module.sha256_text(round_prompt) + assert Path(record["prompt_paths"]["system_prompt"]).read_text(encoding="utf-8") == round_prompt + + +def test_optimizer_round_audit_rejects_out_of_range_validation_rate(tmp_path: Path): + module = load_pipeline_module() + records = module.write_optimizer_round_artifacts( + run_dir=tmp_path, + rounds=[ + SimpleNamespace( + round=2, + optimized_field_names=["system_prompt"], + candidate_prompts={"system_prompt": "round prompt"}, + validation_pass_rate=1.5, + metric_breakdown={ROUTE_TOOL_ARGS_METRIC: 1.0}, + accepted=True, + acceptance_reason="improved validation", + skip_reason=None, + error_message=None, + failed_case_ids=[], + round_llm_cost=0.01, + round_token_usage={"prompt": 8, "completion": 2, "total": 10}, + duration_seconds=0.25, + ) + ], + ) + record = records[0] + + json.dumps(records, allow_nan=False) + assert 0.0 <= record["validation_pass_rate"] <= 1.0 + assert record["accepted"] is False + assert "validation_pass_rate" in record["decision_reason"] + + +def test_optimizer_round_audit_rejects_duplicate_round_ids_without_overwriting_artifacts( + tmp_path: Path, +): + module = load_pipeline_module() + + def round_record(prompt: str, reason: str) -> SimpleNamespace: + return SimpleNamespace( + round=1, + optimized_field_names=["system_prompt"], + candidate_prompts={"system_prompt": prompt}, + validation_pass_rate=1.0, + metric_breakdown={ROUTE_TOOL_ARGS_METRIC: 1.0}, + accepted=True, + acceptance_reason=reason, + skip_reason=None, + error_message=None, + failed_case_ids=[], + round_llm_cost=0.01, + round_token_usage={"prompt": 8, "completion": 2, "total": 10}, + duration_seconds=0.25, + ) + + records = module.write_optimizer_round_artifacts( + run_dir=tmp_path, + rounds=[ + round_record("first round prompt", "first round accepted"), + round_record("duplicate round prompt", "duplicate round accepted"), + ], + ) + + serialized = json.dumps(records, allow_nan=False) + assert serialized + assert [record["round"] for record in records] == [1, 2] + assert len({record["round"] for record in records}) == 2 + assert records[0]["accepted"] is True + assert records[0]["decision_reason"] == "first round accepted" + assert records[1]["accepted"] is False + assert "duplicate round identifier" in records[1]["decision_reason"] + + prompt_paths = [Path(record["prompt_paths"]["system_prompt"]) for record in records] + assert len(set(prompt_paths)) == 2 + for record, prompt_path in zip(records, prompt_paths): + content = prompt_path.read_text(encoding="utf-8") + assert content == ("first round prompt" if record is records[0] else "duplicate round prompt") + assert record["prompt_sha256"]["system_prompt"] == module.sha256_text(content) + + +def test_optimizer_round_audit_drops_unknown_prompt_targets(tmp_path: Path): + module = load_pipeline_module() + records = module.write_optimizer_round_artifacts( + run_dir=tmp_path, + rounds=[ + SimpleNamespace( + round=1, + optimized_field_names=["not_a_target"], + candidate_prompts={ + "system_prompt": "valid system prompt", + "not_a_target": "forged prompt evidence", + }, + validation_pass_rate=1.0, + metric_breakdown={ROUTE_TOOL_ARGS_METRIC: 1.0}, + accepted=True, + acceptance_reason="accepted", + skip_reason=None, + error_message=None, + failed_case_ids=[], + round_llm_cost=0.01, + round_token_usage={"prompt": 8, "completion": 2, "total": 10}, + duration_seconds=0.25, + ) + ], + ) + record = records[0] + + assert set(record["prompt_paths"]) == {"system_prompt"} + assert record["optimized_field_names"] == [] + assert record["accepted"] is False + assert "unknown prompt target" in record["decision_reason"] + + +def test_optimizer_round_audit_drops_unknown_prompt_keys_without_writing_them( + tmp_path: Path, +): + module = load_pipeline_module() + round_dir = tmp_path / "prompts" / "optimizer_round_001" + records = module.write_optimizer_round_artifacts( + run_dir=tmp_path, + rounds=[ + SimpleNamespace( + round=1, + optimized_field_names=["system_prompt"], + candidate_prompts={ + "system_prompt": "valid system prompt", + "router_prompt": "valid router prompt", + "a/../b": "traversal prompt", + "a/../system_prompt": "malformed system prompt", + "b": "plain prompt", + }, + validation_pass_rate=1.0, + metric_breakdown={ROUTE_TOOL_ARGS_METRIC: 1.0}, + accepted=True, + acceptance_reason="accepted", + skip_reason=None, + error_message=None, + failed_case_ids=[], + round_llm_cost=0.01, + round_token_usage={"prompt": 8, "completion": 2, "total": 10}, + duration_seconds=0.25, + ) + ], + ) + record = records[0] + + json.dumps(records, allow_nan=False) + prompt_paths = record["prompt_paths"] + prompt_hashes = record["prompt_sha256"] + assert record["accepted"] is False + assert "unknown prompt target" in record["decision_reason"] + assert prompt_paths["system_prompt"].endswith("system_prompt.md") + assert prompt_paths["router_prompt"].endswith("router_prompt.md") + assert set(prompt_paths) == set(prompt_hashes) == {"system_prompt", "router_prompt"} + assert len(prompt_paths) == len(set(prompt_paths)) == len({Path(path) for path in prompt_paths.values()}) + assert Path(prompt_paths["system_prompt"]).read_text(encoding="utf-8") == "valid system prompt" + assert Path(prompt_paths["router_prompt"]).read_text(encoding="utf-8") == "valid router prompt" + + contents = set() + for key, path_value in prompt_paths.items(): + prompt_path = Path(path_value) + assert prompt_path.resolve().is_relative_to(round_dir.resolve()) + content = prompt_path.read_text(encoding="utf-8") + contents.add(content) + assert prompt_hashes[key] == module.sha256_text(content) + assert contents == {"valid system prompt", "valid router prompt"} + + +@pytest.mark.parametrize("reserved_name", ["CON", "nul.txt", "Com1.json", "LPT9.log"]) +def test_optimizer_round_audit_drops_reserved_unknown_prompt_targets( + tmp_path: Path, + reserved_name: str, +): + module = load_pipeline_module() + records = module.write_optimizer_round_artifacts( + run_dir=tmp_path, + rounds=[ + SimpleNamespace( + round=1, + optimized_field_names=[], + candidate_prompts={reserved_name: "prompt"}, + validation_pass_rate=1.0, + metric_breakdown={}, + accepted=True, + acceptance_reason="accepted", + skip_reason=None, + error_message=None, + failed_case_ids=[], + round_llm_cost=0.0, + round_token_usage={"prompt": 0, "completion": 0, "total": 0}, + duration_seconds=0.0, + ) + ], + ) + record = records[0] + + assert record["accepted"] is False + assert "unknown prompt target" in record["decision_reason"] + assert record["prompt_paths"] == {} + + +@pytest.mark.parametrize("candidate_prompts", [None, ["not a mapping"]]) +def test_optimizer_round_audit_normalizes_malformed_prompt_payloads( + tmp_path: Path, + candidate_prompts: Any, +): + module = load_pipeline_module() + records = module.write_optimizer_round_artifacts( + run_dir=tmp_path, + rounds=[SimpleNamespace(round=1, candidate_prompts=candidate_prompts)], + ) + record = records[0] + + json.dumps(records, allow_nan=False) + assert record["prompt_paths"] == {} + assert record["prompt_sha256"] == {} + assert record["accepted"] is False + assert "candidate_prompts" in record["decision_reason"] + + +def test_optimizer_round_audit_drops_casefolded_unknown_prompt_targets( + tmp_path: Path, +): + module = load_pipeline_module() + records = module.write_optimizer_round_artifacts( + run_dir=tmp_path, + rounds=[ + SimpleNamespace( + round=1, + optimized_field_names=["system_prompt"], + candidate_prompts={"Prompt": "first", "prompt": "second"}, + validation_pass_rate=1.0, + metric_breakdown={ROUTE_TOOL_ARGS_METRIC: 1.0}, + accepted=True, + acceptance_reason="accepted", + skip_reason=None, + error_message=None, + failed_case_ids=[], + round_llm_cost=0.01, + round_token_usage={"prompt": 8, "completion": 2, "total": 10}, + duration_seconds=0.25, + ) + ], + ) + record = records[0] + + json.dumps(records, allow_nan=False) + assert record["prompt_paths"] == {} + assert record["prompt_sha256"] == {} + assert record["prompt_contents"] == {} + assert record["accepted"] is False + assert "unknown prompt target" in record["decision_reason"] + + +def test_optimizer_round_audit_drops_non_string_collection_members_and_mapping_keys( + tmp_path: Path, +): + module = load_pipeline_module() + invalid_member = object() + invalid_metric_key = object() + invalid_token_key = object() + records = module.write_optimizer_round_artifacts( + run_dir=tmp_path, + rounds=[ + SimpleNamespace( + round=1, + optimized_field_names=["system_prompt", invalid_member, 7], + candidate_prompts={"system_prompt": "prompt"}, + validation_pass_rate=1.0, + metric_breakdown={ + ROUTE_TOOL_ARGS_METRIC: 1.0, + invalid_metric_key: 0.5, + }, + accepted=True, + acceptance_reason="accepted", + skip_reason=None, + error_message=None, + failed_case_ids=["case_1", invalid_member, 9], + round_llm_cost=0.01, + round_token_usage={"prompt": 8, invalid_token_key: 2}, + duration_seconds=0.25, + ) + ], + ) + record = records[0] + + json.dumps(records, allow_nan=False) + assert record["optimized_field_names"] == ["system_prompt"] + assert record["failed_case_ids"] == ["case_1"] + assert record["metric_breakdown"] == {ROUTE_TOOL_ARGS_METRIC: 1.0} + assert record["token_usage"] == {"prompt": 0, "completion": 0, "total": 0} + assert record["accepted"] is False + assert "invalid round collections" in record["decision_reason"] + assert "mapping keys" in record["decision_reason"] + + +@pytest.mark.parametrize( + "token_usage", + [ + None, + {"prompt": 8}, + {"prompt": 8, "completion": 2, "total": 9}, + {"prompt": -1, "completion": 2, "total": 1}, + {"prompt": 8, "completion": 2, "total": 10, "cached": 3}, + {"prompt": float("nan"), "completion": 2, "total": 10}, + ], +) +def test_optimizer_round_audit_normalizes_malformed_token_usage_to_schema_shape( + tmp_path: Path, + token_usage: Any, +): + module = load_pipeline_module() + records = module.write_optimizer_round_artifacts( + run_dir=tmp_path, + rounds=[ + SimpleNamespace( + round=1, + optimized_field_names=[], + candidate_prompts={}, + validation_pass_rate=1.0, + metric_breakdown={}, + accepted=True, + acceptance_reason="accepted", + skip_reason=None, + error_message=None, + failed_case_ids=[], + round_llm_cost=0.0, + round_token_usage=token_usage, + duration_seconds=0.0, + ) + ], + ) + + assert records[0]["token_usage"] == {"prompt": 0, "completion": 0, "total": 0} + assert records[0]["accepted"] is False + assert "token" in records[0]["decision_reason"] + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + report["optimization_rounds"] = records + module.validate_report_schema(report) + + +def test_online_cost_audit_distinguishes_optimizer_tokens_from_pipeline_tokens(): + module = load_pipeline_module() + audit = module.online_cost_audit( + _optimizer_result({"prompt": 8, "completion": 2, "total": 10}), + optimizer_candidate_agent_calls=2, + optimizer_llm_metric_count=0, + final_revalidation_calls={"agent_calls": 6, "judge_model_calls": 6, "model_calls": 12}, + ) + + assert audit["optimizer"]["candidate_evaluation_agent_calls"] == 2 + assert audit["optimizer"]["model_calls"] == 3 + assert audit["optimizer"]["token_usage"] is None + assert audit["optimizer"]["token_usage_known"] is False + assert audit["optimizer"]["estimated_cost"] is None + assert audit["optimizer"]["reflection_reported_usage"] == { + "estimated_cost": 0.01, + "token_usage": {"prompt": 8, "completion": 2, "total": 10}, + "token_usage_known": True, + "unknown_token_usage_reason": None, + } + assert audit["final_revalidation"]["token_usage"] is None + assert audit["final_revalidation"]["token_usage_known"] is False + assert audit["token_usage"] is None + assert audit["token_usage_known"] is False + assert audit["model_calls"] == 15 + + +def test_online_cost_audit_rejects_noninteger_native_call_counters(): + module = load_pipeline_module() + result = _optimizer_result({"prompt": 8, "completion": 2, "total": 10}) + result.total_reflection_lm_calls = 1.0 + + audit = module.online_cost_audit( + result, + optimizer_candidate_agent_calls=0, + optimizer_llm_metric_count=0, + final_revalidation_calls={"agent_calls": 0, "judge_model_calls": 0, "model_calls": 0}, + ) + + assert audit["optimizer"]["reflection_lm_calls"] == 0 + assert audit["optimizer"]["usage_evidence_valid"] is False + assert audit["optimizer"]["token_usage_known"] is False + assert audit["optimizer"]["token_usage"] is None + + +def test_online_cost_audit_reconciles_native_and_derived_optimizer_judge_calls(): + module = load_pipeline_module() + result = _optimizer_result({"prompt": 8, "completion": 2, "total": 10}) + result.total_judge_model_calls = 3 + + audit = module.online_cost_audit( + result, + optimizer_candidate_agent_calls=2, + optimizer_llm_metric_count=1, + final_revalidation_calls={"agent_calls": 0, "judge_model_calls": 0, "model_calls": 0}, + ) + optimizer = audit["optimizer"] + + assert optimizer["native_judge_model_calls"] == 3 + assert optimizer["derived_judge_model_calls"] == 2 + assert optimizer["judge_model_calls"] == 3 + assert optimizer["judge_model_call_source"] == "reconciled_native_and_derived_max" + assert optimizer["model_calls"] == 6 + assert audit["model_calls"] == 6 + + +def test_judge_call_accounting_includes_each_model_sample_and_eval_run(): + module = load_pipeline_module() + metrics_config = { + "metrics": [ + { + "metric_name": "llm_multi_judge", + "threshold": 0.5, + "criterion": { + "llm_judge": { + "judge_models": [ + {"model_name": "judge-a", "num_samples": 2}, + {"model_name": "judge-b", "num_samples": 3}, + ] + } + }, + } + ], + "num_runs": 2, + } + summary = {"case_results": [{"case_id": "a"}, {"case_id": "b"}]} + + assert module.judge_calls_per_agent_call(metrics_config) == 5 + final = module.final_revalidation_call_audit([summary], metrics_config) + assert final == { + "agent_calls_per_run": 2, + "agent_calls": 4, + "judge_calls_per_agent_call": 5, + "judge_model_calls": 20, + "model_calls": 24, + } + audit = module.online_cost_audit( + _optimizer_result({"prompt": 8, "completion": 2, "total": 10}), + optimizer_candidate_agent_calls=2, + optimizer_judge_calls_per_agent_call=5, + final_revalidation_calls=final, + ) + assert audit["optimizer"]["judge_calls_per_candidate_evaluation"] == 5 + assert audit["optimizer"]["derived_judge_model_calls"] == 10 + assert audit["optimizer"]["judge_model_calls"] == 10 + + +def test_final_revalidation_call_audit_counts_every_conversation_turn(): + module = load_pipeline_module() + metrics_config = { + "metrics": [ + { + "metric_name": "llm_multi_judge", + "threshold": 0.5, + "criterion": { + "llm_judge": { + "judge_models": [ + {"model_name": "judge-a", "num_samples": 2}, + {"model_name": "judge-b", "num_samples": 3}, + ] + } + }, + } + ], + "num_runs": 1, + } + summaries = [{"case_results": [{"case_id": "a"}]}] + evalset_payloads = [ + { + "eval_cases": [ + { + "eval_id": "a", + "conversation": [ + {"user_content": {}, "final_response": {}}, + {"user_content": {}, "final_response": {}}, + ], + } + ] + } + ] + + audit = module.final_revalidation_call_audit( + summaries, + metrics_config, + evalset_payloads=evalset_payloads, + ) + assert audit == { + "agent_calls_per_run": 2, + "agent_calls": 2, + "judge_calls_per_agent_call": 5, + "judge_model_calls": 10, + "model_calls": 12, + } + + +def test_report_semantics_recompute_derived_judge_calls_from_recorded_multiplier(): + module = load_pipeline_module() + report = load_report(EXAMPLE_DIR / "fixtures" / "optimization_report.sample.json") + optimizer = report["cost"]["optimizer"] + optimizer["candidate_evaluation_agent_calls"] = 2 + optimizer["judge_calls_per_candidate_evaluation"] = 2 + + with pytest.raises(ValidationError, match="derived judge|offline reports"): + module.validate_report_schema(report) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("include_llm_metric", "expected_judge_calls", "expected_optimizer_calls", "expected_source"), + [ + (False, 0, 3, "none"), + (True, 2, 5, "derived_from_candidate_calls_and_llm_metrics"), + ], +) +async def test_run_online_accounts_for_optimizer_internal_llm_metric_judges( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + include_llm_metric: bool, + expected_judge_calls: int, + expected_optimizer_calls: int, + expected_source: str, +): + module = load_pipeline_module() + metrics = load_report(EXAMPLE_DIR / "optimizer.json")["evaluate"]["metrics"] + optimizer_config = _write_optimizer_config( + tmp_path, + metrics=metrics if include_llm_metric else metrics[:1], + ) + report = await _run_stubbed_online( + module=module, + monkeypatch=monkeypatch, + tmp_path=tmp_path, + result=_optimizer_result({"prompt": 8, "completion": 2, "total": 10}), + run_id=f"optimizer_judges_{include_llm_metric}", + optimizer_agent_calls=2, + optimizer_config=optimizer_config, + ) + optimizer = report["cost"]["optimizer"] + + assert optimizer["derived_judge_model_calls"] == expected_judge_calls + assert optimizer["judge_model_calls"] == expected_judge_calls + assert optimizer["judge_model_call_source"] == expected_source + assert optimizer["model_calls"] == expected_optimizer_calls + assert report["cost"]["model_calls"] == ( + expected_optimizer_calls + report["cost"]["final_revalidation"]["model_calls"] + ) + module.validate_report_schema(report) + + +@pytest.mark.asyncio +async def test_online_optimizer_candidate_agent_calls_are_instrumented( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + module = load_pipeline_module() + report = await _run_stubbed_online( + module=module, + monkeypatch=monkeypatch, + tmp_path=tmp_path, + result=_optimizer_result({"prompt": 8, "completion": 2, "total": 10}), + run_id="counted_optimizer_agent_calls", + gate_config={"max_cost_usd": 1.0, "required_metrics": [ROUTE_TOOL_ARGS_METRIC]}, + optimizer_agent_calls=2, + ) + + optimizer = report["cost"]["optimizer"] + assert optimizer["candidate_evaluation_agent_calls"] == 2 + assert optimizer["derived_judge_model_calls"] == 2 + assert optimizer["model_calls"] == 5 + assert optimizer["token_usage_known"] is False + assert report["cost"]["model_calls"] == ( + optimizer["model_calls"] + report["cost"]["final_revalidation"]["model_calls"] + ) + assert report["gate_decision"]["accepted"] is False + module.validate_report_schema(report) + + +@pytest.mark.asyncio +async def test_online_malformed_optimizer_usage_writes_rejected_schema_valid_report( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + module = load_pipeline_module() + report = await _run_stubbed_online( + module=module, + monkeypatch=monkeypatch, + tmp_path=tmp_path, + result=_optimizer_result({"prompt": "invalid", "completion": 2, "total": 2}), + run_id="malformed_optimizer_usage", + ) + + assert report["cost"]["optimizer"]["token_usage"] is None + assert report["cost"]["optimizer"]["reflection_reported_usage"]["token_usage"] == { + "prompt": 0, + "completion": 0, + "total": 0, + } + assert report["cost"]["optimizer"]["token_usage_known"] is False + assert report["gate_decision"]["accepted"] is False + assert "usage evidence" in " ".join(report["gate_decision"]["reasons"]) + module.validate_report_schema(report) + + +@pytest.mark.asyncio +async def test_online_duration_gate_uses_total_elapsed_and_audits_revalidation_phases( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + module = load_pipeline_module() + clock = iter([0.0, 40.0, 60.0, 90.0, 91.0]) + monkeypatch.setattr(module.time, "perf_counter", lambda: next(clock)) + report = await _run_stubbed_online( + module=module, + monkeypatch=monkeypatch, + tmp_path=tmp_path, + result=_optimizer_result({"prompt": 8, "completion": 2, "total": 10}), + run_id="online_total_duration", + gate_config={ + "max_duration_seconds": 80.0, + "required_metrics": [ROUTE_TOOL_ARGS_METRIC], + }, + ) + candidate = report["candidates"][0] + + assert report["gate_decision"]["accepted"] is False + assert "90.00s > 80.00s" in " ".join(candidate["gate"]["reasons"]) + assert candidate["audit"]["duration_seconds"] == 30.0 + assert report["online_duration"] == { + "optimization_seconds": 40.0, + "baseline_revalidation_seconds": 20.0, + "candidate_revalidation_seconds": 30.0, + "gate_elapsed_seconds": 90.0, + } + assert report["duration_seconds"] == 91.0 + module.validate_report_schema(report) + + +@pytest.mark.asyncio +async def test_online_optimizer_validation_improvement_is_accepted( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setenv("TRPC_AGENT_API_KEY", "fake-key") + monkeypatch.setenv("TRPC_AGENT_BASE_URL", "http://localhost/fake") + monkeypatch.setenv("TRPC_AGENT_MODEL_NAME", "fake-model") + module = load_pipeline_module() + + class FakeResult: + status = "SUCCEEDED" + baseline_pass_rate = 0.5 + best_pass_rate = 1.0 + pass_rate_improvement = 0.5 + stop_reason = "required_metrics_passing" + total_llm_cost = 0.0 + total_reflection_lm_calls = 0 + total_judge_model_calls = 0 + best_prompts = {"system_prompt": "better system", "router_prompt": "better router"} + baseline_prompts = {"system_prompt": "baseline system", "router_prompt": "baseline router"} + baseline_metric_breakdown = {ROUTE_TOOL_ARGS_METRIC: 0.5} + best_metric_breakdown = {ROUTE_TOOL_ARGS_METRIC: 1.0} + metric_thresholds = {ROUTE_TOOL_ARGS_METRIC: 1.0} + duration_seconds = 0.01 + total_token_usage = {"prompt": 0, "completion": 0, "total": 0} + + async def fake_optimize(**kwargs): + output_dir = Path(kwargs["output_dir"]) + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "result.json").write_text("{}", encoding="utf-8") + (output_dir / "summary.txt").write_text("fake", encoding="utf-8") + return FakeResult() + + def summary(score: float, passed: bool) -> dict[str, Any]: + metrics = { + ROUTE_TOOL_ARGS_METRIC: { + "score": score, + "threshold": 1.0, + "passed": passed, + "status": "passed" if passed else "failed", + }, + "llm_rubric_response": { + "score": 1.0, + "threshold": 0.66, + "passed": True, + "status": "passed", + }, + } + return { + "eval_set_id": "fake", + "score": score, + "pass_rate": 1.0 if passed else 0.0, + "metrics": copy.deepcopy(metrics), + "case_results": [ + { + "case_id": "case_1", + "tags": [], + "user": "test user", + "score": score, + "passed": passed, + "metrics": copy.deepcopy(metrics), + "actual_text": "", + "expected_text": "", + "key_trace": { + "invocation_id": "case_1", + "actual_final_response": "", + "expected_final_response": "", + "error_message": None, + }, + "root_cause": "" if passed else "metric_failed", + "reasons": [] if passed else ["stubbed metric failure"], + }, + ], + "failed_case_ids": [] if passed else ["case_1"], + "source": "AgentEvaluator", + } + + summaries = iter( + [ + summary(0.5, False), + summary(0.5, False), + summary(0.5, False), + summary(1.0, True), + summary(1.0, True), + summary(1.0, True), + ] + ) + + async def fake_run_evaluator(**_: Any) -> dict[str, Any]: + return next(summaries) + + import trpc_agent_sdk.evaluation as evaluation_pkg + + monkeypatch.setattr(evaluation_pkg.AgentOptimizer, "optimize", staticmethod(fake_optimize)) + monkeypatch.setattr(module, "run_evaluator", fake_run_evaluator) + + run_dir = await module.run_online(seed=7, output_dir=tmp_path, run_id="online_improved") + report = load_report(run_dir / "optimization_report.json") + + assert report["gate_decision"]["accepted"] is True + assert report["gate_decision"]["winner"] == "optimizer_best" + module.validate_report_schema(report) + + +@pytest.mark.asyncio +async def test_online_optimizer_no_improvement_is_rejected( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setenv("TRPC_AGENT_API_KEY", "fake-key") + monkeypatch.setenv("TRPC_AGENT_BASE_URL", "http://localhost/fake") + monkeypatch.setenv("TRPC_AGENT_MODEL_NAME", "fake-model") + + module = load_pipeline_module() + before_system = (EXAMPLE_DIR / "agent" / "prompts" / "system.md").read_text(encoding="utf-8") + before_router = (EXAMPLE_DIR / "agent" / "prompts" / "router.md").read_text(encoding="utf-8") + + class FakeResult: + status = "SUCCEEDED" + baseline_pass_rate = 0.5 + best_pass_rate = 0.5 + pass_rate_improvement = 0.0 + stop_reason = "no_improvement" + total_llm_cost = 0.0 + total_reflection_lm_calls = 0 + total_judge_model_calls = 0 + best_prompts = { + "system_prompt": "changed system", + "router_prompt": "changed router", + } + baseline_prompts = { + "system_prompt": "baseline system", + "router_prompt": "baseline router", + } + baseline_metric_breakdown = {ROUTE_TOOL_ARGS_METRIC: 0.5} + best_metric_breakdown = {ROUTE_TOOL_ARGS_METRIC: 0.5} + metric_thresholds = {ROUTE_TOOL_ARGS_METRIC: 1.0} + duration_seconds = 0.01 + total_token_usage = {"prompt": 0, "completion": 0, "total": 0} + + async def fake_optimize(**kwargs): + output_dir = Path(kwargs["output_dir"]) + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "result.json").write_text("{}", encoding="utf-8") + (output_dir / "summary.txt").write_text("fake", encoding="utf-8") + return FakeResult() + + import trpc_agent_sdk.evaluation as evaluation_pkg + + monkeypatch.setattr(evaluation_pkg.AgentOptimizer, "optimize", staticmethod(fake_optimize)) + patch_agent_evaluator(monkeypatch, score=0.5, passed=False) + + run_dir = await module.run_online(seed=7, output_dir=tmp_path, run_id="online_no_improvement") + report = load_report(run_dir / "optimization_report.json") + + assert report["gate_decision"]["accepted"] is False + assert report["gate_decision"]["winner"] is None + assert "validation score did not improve" in " ".join(report["gate_decision"]["reasons"]) + assert (EXAMPLE_DIR / "agent" / "prompts" / "system.md").read_text(encoding="utf-8") == before_system + assert (EXAMPLE_DIR / "agent" / "prompts" / "router.md").read_text(encoding="utf-8") == before_router + + +@pytest.mark.asyncio +async def test_online_revalidation_uses_eval_metrics_snapshot( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setenv("TRPC_AGENT_API_KEY", "fake-key") + monkeypatch.setenv("TRPC_AGENT_BASE_URL", "http://localhost/fake") + monkeypatch.setenv("TRPC_AGENT_MODEL_NAME", "fake-model") + module = load_pipeline_module() + metrics_paths: list[str] = [] + + class FakeResult: + status = "SUCCEEDED" + baseline_pass_rate = 0.5 + best_pass_rate = 1.0 + pass_rate_improvement = 0.5 + stop_reason = "completed" + total_llm_cost = 0.0 + total_reflection_lm_calls = 0 + total_judge_model_calls = 0 + best_prompts = {"system_prompt": "fake system", "router_prompt": "fake router"} + baseline_prompts = {"system_prompt": "baseline system", "router_prompt": "baseline router"} + baseline_metric_breakdown = {ROUTE_TOOL_ARGS_METRIC: 0.5} + best_metric_breakdown = {ROUTE_TOOL_ARGS_METRIC: 1.0} + metric_thresholds = {ROUTE_TOOL_ARGS_METRIC: 1.0} + duration_seconds = 0.01 + total_token_usage = {"prompt": 0, "completion": 0, "total": 0} + + async def fake_optimize(**kwargs): + output_dir = Path(kwargs["output_dir"]) + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "result.json").write_text("{}", encoding="utf-8") + (output_dir / "summary.txt").write_text("fake", encoding="utf-8") + return FakeResult() + + class FakeExecuter: + def __init__(self, eval_set_path: str) -> None: + self.eval_set_path = Path(eval_set_path) + self.result = None + + async def evaluate(self) -> None: + self.result = make_evaluate_result(self.eval_set_path) + + def get_result(self): + return self.result + + def fake_get_executer(eval_dataset_file_path_or_dir: str, **kwargs: Any) -> FakeExecuter: + metrics_paths.append(str(kwargs["eval_metrics_file_path_or_dir"])) + return FakeExecuter(eval_dataset_file_path_or_dir) + + import trpc_agent_sdk.evaluation as evaluation_pkg + + monkeypatch.setattr(evaluation_pkg.AgentOptimizer, "optimize", staticmethod(fake_optimize)) + monkeypatch.setattr(evaluation_pkg.AgentEvaluator, "get_executer", staticmethod(fake_get_executer)) + + run_dir = await module.run_online(seed=7, output_dir=tmp_path, run_id="online_metrics_snapshot") + report = load_report(run_dir / "optimization_report.json") + + assert metrics_paths + assert all(path.endswith("online_eval_metrics.json") for path in metrics_paths) + assert not any(path.endswith("optimizer.json") for path in metrics_paths) + assert Path(report["artifacts"]["online_eval_metrics"]).is_file() + + +@pytest.mark.skipif(os.getenv("RUN_ONLINE_E2E") != "1", reason="online smoke is opt-in") +def test_online_e2e_smoke_with_real_api(tmp_path: Path): + required = ["TRPC_AGENT_API_KEY", "TRPC_AGENT_BASE_URL", "TRPC_AGENT_MODEL_NAME"] + missing = [name for name in required if not os.getenv(name)] + if missing: + pytest.skip("missing online env vars: " + ", ".join(missing)) + + source_prompts = { + path: path.read_text(encoding="utf-8") + for path in ( + EXAMPLE_DIR / "agent" / "prompts" / "system.md", + EXAMPLE_DIR / "agent" / "prompts" / "router.md", + ) + } + weak_system = tmp_path / "weak_system.md" + weak_system.write_text(source_prompts[EXAMPLE_DIR / "agent" / "prompts" / "system.md"], encoding="utf-8") + weak_router = tmp_path / "weak_router.md" + weak_router.write_text( + "\n".join( + [ + "You route customer-support requests to one backend action.", + "Output exactly one JSON object with keys route, tool, and reason.", + "Allowed tools: create_refund_ticket, create_escalation_case, none.", + "Baseline v0 policy:", + "1. Prefer faq for refund requests unless the user says the refund was already approved.", + "2. Prefer faq for account or legal complaints unless the user uses the exact phrase human agent.", + "3. Use faq for shipping, coupon, address, and policy questions.", + "4. Keep tool.arguments as an empty object.", + ] + ), + encoding="utf-8", + ) + before_weak_system = weak_system.read_text(encoding="utf-8") + before_weak_router = weak_router.read_text(encoding="utf-8") + gate_config = tmp_path / "online_gate.json" + gate_config.write_text( + json.dumps({"max_duration_seconds": 300}), + encoding="utf-8", + ) + + try: + proc = subprocess.run( + [ + sys.executable, + str(RUN_PIPELINE), + "--mode", + "online", + "--output-dir", + str(tmp_path), + "--run-id", + "online_e2e", + "--system-prompt", + str(weak_system), + "--router-prompt", + str(weak_router), + "--gate-config", + str(gate_config), + ], + check=True, + capture_output=True, + text=True, + timeout=300, + ) + run_dir = Path(proc.stdout.strip().splitlines()[-1]) + report = load_report(run_dir / "optimization_report.json") + report_markdown = (run_dir / "optimization_report.md").read_text(encoding="utf-8") + serialized_outputs = proc.stdout + proc.stderr + json.dumps(report) + report_markdown + + assert report["mode"] == "online" + assert report["baseline"]["validation"]["score"] < report["candidates"][0]["validation"]["score"] + assert report["gate_decision"]["accepted"] is True + assert weak_system.read_text(encoding="utf-8") == before_weak_system + assert weak_router.read_text(encoding="utf-8") == before_weak_router + assert all( + artifact["source_written"] is False + for candidate in [report["baseline"], *report["candidates"]] + for artifact in candidate["prompt_artifacts"] + ) + load_pipeline_module().validate_report_schema(report) + assert report["online_preflight"] == { + "TRPC_AGENT_API_KEY": True, + "TRPC_AGENT_BASE_URL": True, + "TRPC_AGENT_MODEL_NAME": True, + } + assert os.environ["TRPC_AGENT_API_KEY"] not in serialized_outputs + finally: + assert {path: path.read_text(encoding="utf-8") for path in source_prompts} == source_prompts + + +@pytest.mark.skipif(os.getenv("RUN_ONLINE_E2E") != "1", reason="online rejection smoke is opt-in") +def test_online_e2e_rejects_perfect_default_prompts(tmp_path: Path): + required = ["TRPC_AGENT_API_KEY", "TRPC_AGENT_BASE_URL", "TRPC_AGENT_MODEL_NAME"] + missing = [name for name in required if not os.getenv(name)] + if missing: + pytest.skip("missing online env vars: " + ", ".join(missing)) + + source_prompts = { + path: path.read_text(encoding="utf-8") + for path in ( + EXAMPLE_DIR / "agent" / "prompts" / "system.md", + EXAMPLE_DIR / "agent" / "prompts" / "router.md", + ) + } + gate_config = tmp_path / "online_gate.json" + gate_config.write_text(json.dumps({"max_duration_seconds": 300}), encoding="utf-8") + + def run_default(run_id: str) -> tuple[subprocess.CompletedProcess[str], dict[str, Any]]: + proc = subprocess.run( + [ + sys.executable, + str(RUN_PIPELINE), + "--mode", + "online", + "--output-dir", + str(tmp_path), + "--run-id", + run_id, + "--gate-config", + str(gate_config), + ], + check=True, + capture_output=True, + text=True, + timeout=300, + ) + run_dir = Path(proc.stdout.strip().splitlines()[-1]) + return proc, load_report(run_dir / "optimization_report.json") + + try: + procs = [] + proc, report = run_default("online_e2e_default_reject") + procs.append(proc) + reports = [report] + if report["baseline"]["validation"]["score"] != 1.0: + proc, report = run_default("online_e2e_default_reject_retry") + procs.append(proc) + reports.append(report) + + candidate = report["candidates"][0] + serialized_outputs = "".join(proc.stdout + proc.stderr for proc in procs) + "".join( + json.dumps(run_report) + + (tmp_path / run_report["run_id"] / "optimization_report.md").read_text(encoding="utf-8") + for run_report in reports + ) + + assert report["baseline"]["validation"]["score"] == 1.0 + assert candidate["validation"]["score"] == 1.0 + assert report["gate_decision"]["accepted"] is False + assert any( + "validation score did not improve over baseline" in reason for reason in report["gate_decision"]["reasons"] + ) + assert all( + artifact["source_written"] is False + for candidate_report in [report["baseline"], *report["candidates"]] + for artifact in candidate_report["prompt_artifacts"] + ) + load_pipeline_module().validate_report_schema(report) + assert os.environ["TRPC_AGENT_API_KEY"] not in serialized_outputs + finally: + assert {path: path.read_text(encoding="utf-8") for path in source_prompts} == source_prompts diff --git a/tests/evaluation/test_llm_judge_multi_model.py b/tests/evaluation/test_llm_judge_multi_model.py index 103d04c3..9361f71c 100644 --- a/tests/evaluation/test_llm_judge_multi_model.py +++ b/tests/evaluation/test_llm_judge_multi_model.py @@ -290,6 +290,62 @@ async def test_legacy_single_judge_model(self): assert r.overall_eval_status == EvalStatus.PASSED +class TestJudgeModelResponseFormatCapabilities: + + @staticmethod + def _capture_judge_agent_construction(metric): + captured = [] + + class CapturingJudgeAgent: + + def __init__(self, model, config, system_prompt, output_schema=None, tools=None, planner=None): + captured.append({ + "model": model, + "config": config, + "output_schema": output_schema, + }) + + with patch("trpc_agent_sdk.evaluation._llm_judge._JudgeAgent", CapturingJudgeAgent): + LLMJudge(metric) + return captured + + @staticmethod + def _rubric_metric(model_name): + return EvalMetric( + metric_name="llm_rubric_response", + threshold=0.5, + criterion={ + "llm_judge": { + "judge_model": { + "model_name": model_name, + }, + }, + }, + ) + + def test_deepseek_judge_uses_json_mode_without_native_schema(self): + from trpc_agent_sdk.models.openai_adapter import _deepseek + + captured = self._capture_judge_agent_construction(self._rubric_metric("deepseek-v4-flash")) + + assert len(captured) == 1 + judge = captured[0] + assert judge["output_schema"] is None + assert judge["config"].response_mime_type == "application/json" + with patch.object(_deepseek.logger, "warning") as warning: + response_format = judge["model"]._build_response_format(judge["config"]) + assert response_format == {"type": "json_object"} + warning.assert_not_called() + + def test_schema_capable_judge_keeps_native_schema(self): + captured = self._capture_judge_agent_construction(self._rubric_metric("gpt-4o")) + + assert len(captured) == 1 + judge = captured[0] + assert judge["output_schema"] is not None + assert judge["config"].response_mime_type is None + + class TestUnknownAggregatorRaisesAtConstruction: def test_unknown_aggregator_raises(self): diff --git a/tests/evaluation/test_llm_judge_think.py b/tests/evaluation/test_llm_judge_think.py index fdcd3c13..d74bb55e 100644 --- a/tests/evaluation/test_llm_judge_think.py +++ b/tests/evaluation/test_llm_judge_think.py @@ -17,7 +17,9 @@ from trpc_agent_sdk.evaluation._llm_judge import _JudgeAgent from trpc_agent_sdk.evaluation._llm_judge import _judge_generation_config from trpc_agent_sdk.evaluation._llm_judge import _merge_extra_body +from trpc_agent_sdk.types import Content from trpc_agent_sdk.types import HttpOptions +from trpc_agent_sdk.types import Part class TestJudgeModelOptionsThinkField: @@ -143,6 +145,132 @@ def test_generation_config_thinking_config_used_when_think_is_none(self): class TestJudgeAgentPlanner: + @pytest.mark.asyncio + async def test_judge_agent_collects_final_non_thought_text(self): + class _FakeEvent: + + def __init__(self, *, final, content): + self._final = final + self.content = content + + def is_final_response(self): + return self._final + + events = [ + _FakeEvent( + final=False, + content=Content(parts=[Part(text="ignored non-final")]), + ), + _FakeEvent(final=True, content=None), + _FakeEvent( + final=True, + content=Content(parts=[ + Part(text="hidden reasoning", thought=True), + Part(text=" first "), + Part(text="second"), + ]), + ), + ] + + class _FakeLlmAgent: + + def __init__(self, **kwargs): + pass + + def run_async(self, ctx): + async def _run(): + for event in events: + yield event + + return _run() + + class _FakeInvocationContext: + + def __init__(self, **kwargs): + self.run_config = kwargs["run_config"] + + with patch("trpc_agent_sdk.evaluation._llm_judge.LlmAgent", _FakeLlmAgent), patch( + "trpc_agent_sdk.evaluation._llm_judge.InvocationContext", _FakeInvocationContext): + judge = _JudgeAgent( + model=object(), + config=None, + system_prompt="sp", + ) + response = await judge.get_response("evaluate this response") + + assert response == "first\nsecond" + + @pytest.mark.asyncio + async def test_judge_agent_closes_its_agent_run(self): + captured = {"closed": False} + + class _FakeAgentRun: + + def __aiter__(self): + return self + + async def __anext__(self): + raise StopAsyncIteration + + async def aclose(self): + captured["closed"] = True + + agent_run = _FakeAgentRun() + + class _FakeLlmAgent: + + def __init__(self, **kwargs): + pass + + def run_async(self, ctx): + return agent_run + + class _FakeInvocationContext: + + def __init__(self, **kwargs): + pass + + with patch("trpc_agent_sdk.evaluation._llm_judge.LlmAgent", _FakeLlmAgent), patch( + "trpc_agent_sdk.evaluation._llm_judge.InvocationContext", _FakeInvocationContext): + judge = _JudgeAgent( + model=object(), + config=None, + system_prompt="sp", + ) + await judge.get_response("evaluate this response") + + assert captured["closed"] is True + + @pytest.mark.asyncio + async def test_judge_agent_disables_model_streaming(self): + captured: dict[str, Any] = {} + + class _FakeLlmAgent: + + def __init__(self, **kwargs): + captured.update(kwargs) + + async def run_async(self, ctx): + captured["run_config"] = ctx.run_config + if False: # pragma: no cover - makes this an async generator + yield None + + class _FakeInvocationContext: + + def __init__(self, **kwargs): + self.run_config = kwargs["run_config"] + + with patch("trpc_agent_sdk.evaluation._llm_judge.LlmAgent", _FakeLlmAgent), patch( + "trpc_agent_sdk.evaluation._llm_judge.InvocationContext", _FakeInvocationContext): + judge = _JudgeAgent( + model=object(), + config=None, + system_prompt="sp", + ) + await judge.get_response("evaluate this response") + + assert captured["run_config"].streaming is False + def test_judge_agent_accepts_planner_and_forwards_to_llm_agent(self): captured: dict[str, Any] = {} diff --git a/tests/models/test_openai_model_ext.py b/tests/models/test_openai_model_ext.py index c757b9e0..9cadfa20 100644 --- a/tests/models/test_openai_model_ext.py +++ b/tests/models/test_openai_model_ext.py @@ -1154,6 +1154,49 @@ def test_no_warning_for_supported(self): class TestGenerateAsyncEdgeCases: """Edge case tests for _generate_async_impl via generate_async.""" + @pytest.mark.asyncio + async def test_streaming_response_is_closed_before_client(self): + """The OpenAI stream itself is closed before its owning client.""" + model = _model() + content = Content(parts=[Part.from_text(text="hi")], role="user") + request = _request([content]) + cleanup_order: list[str] = [] + + class ClosableStream: + + def __init__(self): + self._yielded = False + + def __aiter__(self): + return self + + async def __anext__(self): + if self._yielded: + raise StopAsyncIteration + self._yielded = True + chunk = Mock() + chunk.model_dump.return_value = {"choices": [], "usage": None} + return chunk + + async def close(self): + cleanup_order.append("stream") + + stream = ClosableStream() + + async def close_client(): + cleanup_order.append("client") + + with patch.object(model, "_create_async_client") as mock_factory: + mock_client = AsyncMock() + mock_client.chat.completions.create = AsyncMock(return_value=stream) + mock_client.close = AsyncMock(side_effect=close_client) + mock_factory.return_value = mock_client + + async for _ in model.generate_async(request, stream=True): + pass + + assert cleanup_order == ["stream", "client"] + @pytest.mark.asyncio async def test_streaming_error_yields_error_response(self): """Streaming errors yield an error LlmResponse.""" diff --git a/trpc_agent_sdk/agents/_llm_agent.py b/trpc_agent_sdk/agents/_llm_agent.py index 3546c1d6..fe2ac829 100644 --- a/trpc_agent_sdk/agents/_llm_agent.py +++ b/trpc_agent_sdk/agents/_llm_agent.py @@ -499,7 +499,8 @@ def accumulate_content(event: Event) -> None: logger.debug("Starting LLM call for agent: %s", self.name) # Use LlmProcessor to get unified events - async for event in llm_processor.call_llm_async(request, ctx, stream=True): + model_streaming = ctx.run_config.streaming if ctx.run_config is not None else True + async for event in llm_processor.call_llm_async(request, ctx, stream=model_streaming): # Handle different event types by checking content and error status if event.is_error(): # Error event - yield and stop diff --git a/trpc_agent_sdk/agents/core/_llm_processor.py b/trpc_agent_sdk/agents/core/_llm_processor.py index 6d3ca373..3798f5db 100644 --- a/trpc_agent_sdk/agents/core/_llm_processor.py +++ b/trpc_agent_sdk/agents/core/_llm_processor.py @@ -67,6 +67,7 @@ async def call_llm_async(self, """ author = context.agent.name logger.debug("Starting LLM call for agent: %s", author) + buffered_events: list[Event] = [] try: # Step 1: Validate the request @@ -131,7 +132,10 @@ def _append_function_calls(target: list[dict], calls: list) -> None: if not llm_response.partial: final_llm_response = llm_response - yield event + if stream: + yield event + else: + buffered_events.append(event) except Exception as ex: metrics_error_type = type(ex).__name__ raise @@ -164,6 +168,10 @@ def _append_function_calls(target: list[dict], calls: list) -> None: except Exception as ex: # pylint: disable=broad-except logger.error("LLM call failed for agent %s: %s", author, ex) yield self._create_error_event(context, "llm_call_error", f"LLM call failed: {str(ex)}") + else: + if not stream: + for event in buffered_events: + yield event logger.debug("LLM call completed for agent: %s", author) diff --git a/trpc_agent_sdk/evaluation/_agent_evaluator.py b/trpc_agent_sdk/evaluation/_agent_evaluator.py index a2e47009..cefeffe8 100644 --- a/trpc_agent_sdk/evaluation/_agent_evaluator.py +++ b/trpc_agent_sdk/evaluation/_agent_evaluator.py @@ -664,10 +664,11 @@ def _load_eval_set_from_file( selected_case_id = None actual_file_path = eval_set_file - if ":" in eval_set_file: - parts = eval_set_file.split(":", 1) - actual_file_path = parts[0] - selected_case_id = parts[1] + if ":" in eval_set_file and not os.path.exists(eval_set_file): + maybe_file_path, maybe_case_id = eval_set_file.rsplit(":", 1) + if maybe_file_path.endswith(".json") and os.path.exists(maybe_file_path): + actual_file_path = maybe_file_path + selected_case_id = maybe_case_id if not os.path.exists(actual_file_path): raise FileNotFoundError(f"Eval set file not found: {actual_file_path}") diff --git a/trpc_agent_sdk/evaluation/_llm_judge.py b/trpc_agent_sdk/evaluation/_llm_judge.py index 175ee374..11f12a82 100644 --- a/trpc_agent_sdk/evaluation/_llm_judge.py +++ b/trpc_agent_sdk/evaluation/_llm_judge.py @@ -22,6 +22,7 @@ from pydantic import field_validator from trpc_agent_sdk.agents import LlmAgent +from trpc_agent_sdk.configs import RunConfig from trpc_agent_sdk.context import InvocationContext from trpc_agent_sdk.context import create_agent_context from trpc_agent_sdk.context import new_invocation_context_id @@ -34,6 +35,7 @@ from trpc_agent_sdk.types import HttpOptions from trpc_agent_sdk.types import Part from trpc_agent_sdk.types import ThinkingConfig +from trpc_agent_sdk.utils import AsyncClosingContextManager from ._eval_case import IntermediateData from ._eval_case import Invocation @@ -885,6 +887,14 @@ def _rubric_system(prompt: str) -> str: return prompt.split("# Your Turn")[0].strip() + "\n\n## Output" +def _model_supports_response_schema(model: Any) -> bool: + """Return model schema capability, preserving native behavior for unknown models.""" + supports_response_schema = getattr(model, "supports_response_schema", None) + if not callable(supports_response_schema): + return True + return bool(supports_response_schema()) + + def _expand_env(s: str) -> str: """Expand environment variables in a string (e.g. $VAR or ${VAR}).""" if not s or not isinstance(s, str): @@ -1077,16 +1087,19 @@ async def get_response(self, user_message: str) -> str: agent_context=agent_context, user_content=user_content, override_messages=[user_content], + run_config=RunConfig(streaming=False), ) last_text = "" - async for event in self._agent.run_async(ctx): - if not event.is_final_response(): - continue - if not event.content or not event.content.parts: - continue - part_text = "\n".join((p.text or "").strip() for p in event.content.parts if p.thought is not True).strip() - if part_text: - last_text += part_text + async with AsyncClosingContextManager(self._agent.run_async(ctx)) as agent_run: + async for event in agent_run: + if not event.is_final_response(): + continue + if not event.content or not event.content.parts: + continue + part_text = "\n".join( + (p.text or "").strip() for p in event.content.parts if p.thought is not True).strip() + if part_text: + last_text += part_text return last_text.strip() @@ -1198,12 +1211,16 @@ def __init__( model = _create_judge_model(opts) cfg, effective_tc = _judge_generation_config(opts.generation_config, opts.think) planner = (BuiltInPlanner(thinking_config=effective_tc) if effective_tc is not None else None) + effective_output_schema = output_schema + if not _model_supports_response_schema(model): + cfg.response_mime_type = "application/json" + effective_output_schema = None self._judge_agents.append( _JudgeAgent( model, cfg, system_prompt, - output_schema=output_schema, + output_schema=effective_output_schema, tools=judge_tools, planner=planner, )) diff --git a/trpc_agent_sdk/models/_openai_model.py b/trpc_agent_sdk/models/_openai_model.py index bd30ae33..6459809d 100644 --- a/trpc_agent_sdk/models/_openai_model.py +++ b/trpc_agent_sdk/models/_openai_model.py @@ -11,6 +11,7 @@ """ import base64 +import inspect import json import uuid from enum import Enum @@ -205,6 +206,10 @@ def _refresh_adapter(self) -> None: """Refresh provider adapter after model or endpoint changes.""" self._adapter = get_openai_adapter(self._model_name, self._base_url) + def supports_response_schema(self) -> bool: + """Return whether the selected provider supports native response schemas.""" + return self._adapter.supports_response_schema() + def is_retriable_status_code(self, status_code: int) -> Optional[bool]: return status_code in {408, 409, 429} or status_code >= 500 @@ -1570,6 +1575,7 @@ async def _generate_stream(self, api_params[ApiParamsKey.STREAM_OPTS] = {ApiParamsKey.INCLUDE_USAGE: True} client = self._create_async_client() + response: Any = None logger.debug("openai invoke with params: %s", api_params) try: response = await client.chat.completions.create(**api_params, **http_options) @@ -1781,4 +1787,13 @@ async def _generate_stream(self, custom_metadata={"stream_complete": True}, ) finally: - await self._http_client_provider.close_http_client(client) + try: + close_stream = getattr(response, "close", None) + if not callable(close_stream): + close_stream = getattr(response, "aclose", None) + if callable(close_stream): + close_result = close_stream() + if inspect.isawaitable(close_result): + await close_result + finally: + await self._http_client_provider.close_http_client(client) diff --git a/trpc_agent_sdk/models/openai_adapter/_base.py b/trpc_agent_sdk/models/openai_adapter/_base.py index 8d042ad9..f482f01f 100644 --- a/trpc_agent_sdk/models/openai_adapter/_base.py +++ b/trpc_agent_sdk/models/openai_adapter/_base.py @@ -71,6 +71,10 @@ def build_response_format(self, config: Any) -> tuple[bool, Optional[dict[str, A """ return False, None + def supports_response_schema(self) -> bool: + """Whether the provider supports native JSON schema response formats.""" + return True + def apply_thinking(self, request: Any, http_options: dict[str, Any]) -> bool: """Apply provider-specific thinking options. diff --git a/trpc_agent_sdk/models/openai_adapter/_deepseek.py b/trpc_agent_sdk/models/openai_adapter/_deepseek.py index 64c82bae..6443da32 100644 --- a/trpc_agent_sdk/models/openai_adapter/_deepseek.py +++ b/trpc_agent_sdk/models/openai_adapter/_deepseek.py @@ -56,6 +56,10 @@ def build_response_format(self, config: Any) -> tuple[bool, Optional[dict[str, A logger.warning("DeepSeek only supports JSON object response_format; response schema is ignored.") return True, {"type": "json_object"} + def supports_response_schema(self) -> bool: + """DeepSeek accepts JSON mode but ignores native response schemas.""" + return False + def apply_thinking(self, request: Any, http_options: dict[str, Any]) -> bool: if not self.is_v4_model(): return False