From 33260d40f63a43d4b2820cd3be39b8e8ac23e971 Mon Sep 17 00:00:00 2001 From: shiquan_zhan Date: Sat, 11 Jul 2026 02:10:50 +0800 Subject: [PATCH 01/15] feat: add issue 91 pipeline contracts --- .../eval_optimize_loop/__init__.py | 1 + .../eval_optimize_loop/pipeline/__init__.py | 1 + .../eval_optimize_loop/pipeline/comparator.py | 38 +++++ .../eval_optimize_loop/pipeline/config.py | 39 +++++ .../eval_optimize_loop/pipeline/gate.py | 29 ++++ .../eval_optimize_loop/pipeline/models.py | 144 ++++++++++++++++++ .../eval_optimize_loop/pipeline/reporter.py | 19 +++ .../eval_optimize_loop/__init__.py | 1 + .../eval_optimize_loop/test_phase1_core.py | 94 ++++++++++++ 9 files changed, 366 insertions(+) create mode 100644 examples/optimization/eval_optimize_loop/__init__.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/__init__.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/comparator.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/config.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/gate.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/models.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/reporter.py create mode 100644 tests/examples/optimization/eval_optimize_loop/__init__.py create mode 100644 tests/examples/optimization/eval_optimize_loop/test_phase1_core.py diff --git a/examples/optimization/eval_optimize_loop/__init__.py b/examples/optimization/eval_optimize_loop/__init__.py new file mode 100644 index 00000000..fc589007 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/__init__.py @@ -0,0 +1 @@ +"""Deterministic no-key evaluation and optimization loop example.""" diff --git a/examples/optimization/eval_optimize_loop/pipeline/__init__.py b/examples/optimization/eval_optimize_loop/pipeline/__init__.py new file mode 100644 index 00000000..c9adbf88 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/__init__.py @@ -0,0 +1 @@ +"""Pipeline components for the Issue #91 example.""" diff --git a/examples/optimization/eval_optimize_loop/pipeline/comparator.py b/examples/optimization/eval_optimize_loop/pipeline/comparator.py new file mode 100644 index 00000000..e479f323 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/comparator.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from .models import CaseDelta, CaseSnapshot + + +def compare_case( + baseline: CaseSnapshot, + candidate: CaseSnapshot, + *, + epsilon: float, + critical_case_ids: set[str], +) -> CaseDelta: + if baseline.eval_id != candidate.eval_id: + raise ValueError("baseline and candidate must use the same eval_id") + if not baseline.passed and candidate.passed: + transition = "NEW_PASS" + elif baseline.passed and not candidate.passed: + transition = "REGRESSION" + elif candidate.aggregate_score > baseline.aggregate_score + epsilon: + transition = "IMPROVED" + elif candidate.aggregate_score < baseline.aggregate_score - epsilon: + transition = "DEGRADED" + else: + transition = "UNCHANGED" + metric_names = set(baseline.metric_scores) | set(candidate.metric_scores) + critical = baseline.eval_id in critical_case_ids + return CaseDelta( + eval_id=baseline.eval_id, + baseline_passed=baseline.passed, + candidate_passed=candidate.passed, + transition=transition, + baseline_score=baseline.aggregate_score, + candidate_score=candidate.aggregate_score, + score_delta=candidate.aggregate_score - baseline.aggregate_score, + metric_deltas={name: candidate.metric_scores.get(name, 0.0) - baseline.metric_scores.get(name, 0.0) for name in metric_names}, + critical=critical, + hard_fail_added=(not baseline.hard_failed and candidate.hard_failed) or (critical and transition == "REGRESSION"), + ) diff --git a/examples/optimization/eval_optimize_loop/pipeline/config.py b/examples/optimization/eval_optimize_loop/pipeline/config.py new file mode 100644 index 00000000..80f16745 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/config.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any, Literal + +from .models import PipelineSettings, StrictModel + + +class PipelineConfig(StrictModel): + raw: dict[str, Any] + pipeline: PipelineSettings + config_path: Path + + +def canonical_sha256(value: object) -> str: + encoded = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + return f"sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def sanitize_config(value: object) -> object: + if isinstance(value, dict): + return { + key: "***REDACTED***" if key.lower() in {"api_key", "authorization", "cookie", "token"} else sanitize_config(item) + for key, item in value.items() + } + if isinstance(value, list): + return [sanitize_config(item) for item in value] + return value + + +def load_pipeline_config(config_path: Path, *, mode: Literal["fake"]) -> PipelineConfig: + if mode != "fake": + raise ValueError("Phase 1 and 2 implement only --mode fake") + raw = json.loads(config_path.read_text(encoding="utf-8")) + if "pipeline" not in raw: + raise ValueError("optimizer config requires a pipeline section") + return PipelineConfig(raw=raw, pipeline=PipelineSettings.model_validate(raw["pipeline"]), config_path=config_path) diff --git a/examples/optimization/eval_optimize_loop/pipeline/gate.py b/examples/optimization/eval_optimize_loop/pipeline/gate.py new file mode 100644 index 00000000..adf4c159 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/gate.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from .models import CaseDelta, GateDecision, GateRuleResult, GateSettings, SplitReport + + +def evaluate_gate( + baseline: SplitReport, + candidate: SplitReport, + *, + settings: GateSettings, + case_deltas: list[CaseDelta], + train_score_delta: float | None = None, +) -> GateDecision: + score_delta = candidate.aggregate_score - baseline.aggregate_score + pass_rate_delta = candidate.pass_rate - baseline.pass_rate + new_hard_fails = sum(delta.hard_fail_added for delta in case_deltas) + regressions = sum(delta.transition == "REGRESSION" for delta in case_deltas) + critical_regression = any(delta.critical and delta.transition == "REGRESSION" for delta in case_deltas) + overfit = bool(settings.reject_when_train_improves_but_validation_declines and train_score_delta is not None and train_score_delta > 0 and score_delta < 0) + rules = [ + GateRuleResult(rule="validation_score_improved", passed=score_delta >= settings.min_validation_score_delta, actual=score_delta, expected=settings.min_validation_score_delta, reason="validation aggregate score must improve"), + GateRuleResult(rule="validation_pass_rate_not_worse", passed=pass_rate_delta >= settings.min_validation_pass_rate_delta, actual=pass_rate_delta, expected=settings.min_validation_pass_rate_delta, reason="validation pass rate must not decline"), + GateRuleResult(rule="new_hard_fails", passed=new_hard_fails <= settings.max_new_hard_fails, actual=new_hard_fails, expected=settings.max_new_hard_fails, reason="new hard failures are not allowed"), + GateRuleResult(rule="validation_regressions", passed=regressions <= settings.max_validation_regressions, actual=regressions, expected=settings.max_validation_regressions, reason="validation regressions exceed the limit"), + GateRuleResult(rule="no_critical_regression", passed=settings.allow_critical_case_regression or not critical_regression, actual=critical_regression, expected=False, reason="critical validation cases must not regress"), + GateRuleResult(rule="no_overfit", passed=not overfit, actual=overfit, expected=False, reason="train improvement with validation decline is rejected"), + ] + failed = [rule.reason for rule in rules if not rule.passed] + return GateDecision(accepted=not failed, risk_level="high" if critical_regression or overfit else ("medium" if failed else "low"), rules=rules, reasons=failed or ["candidate passed all independent gate rules"]) diff --git a/examples/optimization/eval_optimize_loop/pipeline/models.py b/examples/optimization/eval_optimize_loop/pipeline/models.py new file mode 100644 index 00000000..c9377a35 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/models.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +from enum import Enum +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class FailureType(str, Enum): + EXECUTION_ERROR = "execution_error" + FINAL_RESPONSE_MISMATCH = "final_response_mismatch" + TOOL_SELECTION_ERROR = "tool_selection_error" + TOOL_ARGUMENT_ERROR = "tool_argument_error" + FORMAT_VIOLATION = "format_violation" + KNOWLEDGE_RECALL_INSUFFICIENT = "knowledge_recall_insufficient" + LLM_RUBRIC_NOT_MET = "llm_rubric_not_met" + UNKNOWN = "unknown" + + +class ToolCallSnapshot(StrictModel): + name: str + arguments: dict[str, Any] = Field(default_factory=dict) + + +class CaseSnapshot(StrictModel): + eval_id: str + split: Literal["train", "validation"] + run_count: int + passed: bool + hard_failed: bool + aggregate_score: float + metric_scores: dict[str, float] + metric_thresholds: dict[str, float] + metric_passed: dict[str, bool] + trace_digest: str + metric_reasons: dict[str, list[str]] = Field(default_factory=dict) + failure_reasons: list[str] = Field(default_factory=list) + final_response: str | None = None + expected_response: str | None = None + tool_calls: list[ToolCallSnapshot] = Field(default_factory=list) + expected_tool_calls: list[ToolCallSnapshot] = Field(default_factory=list) + + +class CaseDelta(StrictModel): + eval_id: str + baseline_passed: bool + candidate_passed: bool + transition: Literal["NEW_PASS", "REGRESSION", "IMPROVED", "DEGRADED", "UNCHANGED"] + baseline_score: float + candidate_score: float + score_delta: float + metric_deltas: dict[str, float] + critical: bool + hard_fail_added: bool + + +class SplitReport(StrictModel): + cases: list[CaseSnapshot] + pass_rate: float + aggregate_score: float + + @classmethod + def from_cases(cls, cases: list[CaseSnapshot]) -> "SplitReport": + if not cases: + return cls(cases=[], pass_rate=0.0, aggregate_score=0.0) + return cls( + cases=cases, + pass_rate=sum(case.passed for case in cases) / len(cases), + aggregate_score=sum(case.aggregate_score for case in cases) / len(cases), + ) + + +class GateSettings(StrictModel): + min_validation_score_delta: float = 0.05 + min_validation_pass_rate_delta: float = 0.0 + max_new_hard_fails: int = 0 + max_validation_regressions: int = 0 + critical_case_ids: list[str] = Field(default_factory=list) + allow_critical_case_regression: bool = False + reject_when_train_improves_but_validation_declines: bool = True + tie_policy: Literal["reject"] = "reject" + + +class GateRuleResult(StrictModel): + rule: str + passed: bool + actual: Any + expected: Any + reason: str + + +class GateDecision(StrictModel): + accepted: bool + risk_level: Literal["low", "medium", "high"] + rules: list[GateRuleResult] + reasons: list[str] + + +class ReproducibilitySettings(StrictModel): + seed: int = 42 + + +class PipelineSettings(StrictModel): + reproducibility: ReproducibilitySettings = Field(default_factory=ReproducibilitySettings) + gate: GateSettings = Field(default_factory=GateSettings) + scoring_epsilon: float = 0.000001 + metric_weights: dict[str, float] = Field(default_factory=lambda: {"final_response_avg_score": 0.6, "fake_rubric_score": 0.4}) + + +class CandidateRecord(StrictModel): + candidate_id: str + prompts: dict[str, str] + source: Literal["fixture"] = "fixture" + generation_cost_usd: float = 0.0 + round_index: int | None = None + + +class CandidateReport(StrictModel): + candidate_id: str + accepted: bool + reasons: list[str] = Field(default_factory=list) + train: SplitReport | None = None + validation: SplitReport | None = None + gate: GateDecision | None = None + validation_case_deltas: list[CaseDelta] = Field(default_factory=list) + + +class OptimizationReport(StrictModel): + schema_version: str = "1.0" + mode: Literal["fake"] + seed: int + selected_candidate_id: str | None = None + candidates: list[CandidateReport] = Field(default_factory=list) + baseline_train: SplitReport | None = None + baseline_validation: SplitReport | None = None + source_integrity: Literal["restored", "unknown"] = "restored" + + @classmethod + def empty(cls, *, mode: Literal["fake"], seed: int) -> "OptimizationReport": + return cls(mode=mode, seed=seed) diff --git a/examples/optimization/eval_optimize_loop/pipeline/reporter.py b/examples/optimization/eval_optimize_loop/pipeline/reporter.py new file mode 100644 index 00000000..c32dd1c5 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/reporter.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from .models import OptimizationReport + + +def write_reports(report: OptimizationReport, output_dir: Path) -> tuple[Path, Path]: + output_dir.mkdir(parents=True, exist_ok=True) + json_path = output_dir / "optimization_report.json" + markdown_path = output_dir / "optimization_report.md" + json_path.write_text(json.dumps(report.model_dump(mode="json"), ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + lines = ["# Optimization Report", "", f"Selected candidate: {report.selected_candidate_id or 'none'}", "", "## Candidates", ""] + for candidate in report.candidates: + status = "ACCEPT" if candidate.accepted else "REJECT" + lines.append(f"- {candidate.candidate_id}: {status} — {'; '.join(candidate.reasons)}") + markdown_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + return json_path, markdown_path diff --git a/tests/examples/optimization/eval_optimize_loop/__init__.py b/tests/examples/optimization/eval_optimize_loop/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/tests/examples/optimization/eval_optimize_loop/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/examples/optimization/eval_optimize_loop/test_phase1_core.py b/tests/examples/optimization/eval_optimize_loop/test_phase1_core.py new file mode 100644 index 00000000..07438839 --- /dev/null +++ b/tests/examples/optimization/eval_optimize_loop/test_phase1_core.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from examples.optimization.eval_optimize_loop.pipeline.comparator import compare_case +from examples.optimization.eval_optimize_loop.pipeline.config import canonical_sha256, load_pipeline_config +from examples.optimization.eval_optimize_loop.pipeline.gate import evaluate_gate +from examples.optimization.eval_optimize_loop.pipeline.models import ( + CandidateReport, + CaseSnapshot, + GateSettings, + OptimizationReport, + SplitReport, +) +from examples.optimization.eval_optimize_loop.pipeline.reporter import write_reports + + +def _case(eval_id: str, *, passed: bool, score: float) -> CaseSnapshot: + return CaseSnapshot( + eval_id=eval_id, + split="validation", + run_count=1, + passed=passed, + hard_failed=False, + aggregate_score=score, + metric_scores={"final_response_avg_score": score}, + metric_thresholds={"final_response_avg_score": 1.0}, + metric_passed={"final_response_avg_score": passed}, + trace_digest="sha256:test", + ) + + +def test_strict_case_snapshot_rejects_unknown_fields() -> None: + with pytest.raises(ValidationError, match="extra_forbidden"): + CaseSnapshot( + eval_id="val_refund_critical", + split="validation", + run_count=1, + passed=True, + hard_failed=False, + aggregate_score=1.0, + metric_scores={}, + metric_thresholds={}, + metric_passed={}, + trace_digest="sha256:test", + unsupported=True, + ) + + +def test_canonical_digest_is_order_independent() -> None: + assert canonical_sha256({"b": 2, "a": 1}) == canonical_sha256({"a": 1, "b": 2}) + + +def test_fake_config_loads_without_live_key(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + config_path = tmp_path / "optimizer.json" + config_path.write_text(json.dumps({"pipeline": {"reproducibility": {"seed": 42}}}), encoding="utf-8") + monkeypatch.delenv("TRPC_AGENT_API_KEY", raising=False) + assert load_pipeline_config(config_path, mode="fake").pipeline.reproducibility.seed == 42 + + +def test_critical_pass_to_fail_is_hard_regression() -> None: + delta = compare_case( + _case("val_refund_critical", passed=True, score=1.0), + _case("val_refund_critical", passed=False, score=0.0), + epsilon=1e-6, + critical_case_ids={"val_refund_critical"}, + ) + assert delta.transition == "REGRESSION" + assert delta.hard_fail_added is True + + +def test_gate_rejects_validation_critical_regression() -> None: + baseline = SplitReport.from_cases([_case("val_refund_critical", passed=True, score=1.0)]) + candidate = SplitReport.from_cases([_case("val_refund_critical", passed=False, score=0.0)]) + decision = evaluate_gate( + baseline, + candidate, + settings=GateSettings(critical_case_ids=["val_refund_critical"]), + case_deltas=[compare_case(baseline.cases[0], candidate.cases[0], epsilon=1e-6, critical_case_ids={"val_refund_critical"})], + ) + assert decision.accepted is False + assert any(rule.rule == "no_critical_regression" and not rule.passed for rule in decision.rules) + + +def test_reporter_writes_json_and_markdown(tmp_path: Path) -> None: + report = OptimizationReport.empty(mode="fake", seed=42) + report.candidates.append(CandidateReport(candidate_id="candidate_noop", accepted=False, reasons=["no improvement"])) + json_path, markdown_path = write_reports(report, tmp_path) + assert json.loads(json_path.read_text(encoding="utf-8"))["schema_version"] == "1.0" + assert "Selected candidate" in markdown_path.read_text(encoding="utf-8") From c834e49f0d0d6f4abd51777f3784b13e8d83cfea Mon Sep 17 00:00:00 2001 From: shiquan_zhan Date: Sat, 11 Jul 2026 02:42:35 +0800 Subject: [PATCH 02/15] feat(eval): add deterministic issue 91 fake loop --- .gitignore | 3 + .../plans/2026-07-11-issue-91-phase-1-2.md | 288 ++++ .../eval_optimize_loop/agent/__init__.py | 1 + .../eval_optimize_loop/agent/agent.py | 5 + .../agent/prompts/router.md | 1 + .../agent/prompts/system.md | 1 + .../eval_optimize_loop/fake/__init__.py | 1 + .../eval_optimize_loop/fake/candidates.json | 5 + .../eval_optimize_loop/fake/fake_agent.py | 37 + .../eval_optimize_loop/fake/fake_judge.py | 14 + .../fake/fixture_optimizer.py | 15 + .../eval_optimize_loop/optimizer.json | 10 + .../eval_optimize_loop/pipeline/evaluator.py | 23 + .../pipeline/normalization.py | 94 ++ .../pipeline/prompt_sandbox.py | 28 + .../eval_optimize_loop/pipeline/reporter.py | 66 +- .../eval_optimize_loop/run_pipeline.py | 69 + .../sample_output/optimization_report.json | 1171 +++++++++++++++++ .../sample_output/optimization_report.md | 106 ++ .../eval_optimize_loop/train.evalset.json | 8 + .../eval_optimize_loop/val.evalset.json | 8 + .../eval_optimize_loop/test_fake_loop.py | 49 + .../eval_optimize_loop/test_phase1_core.py | 16 +- 23 files changed, 2016 insertions(+), 3 deletions(-) create mode 100644 docs/superpowers/plans/2026-07-11-issue-91-phase-1-2.md create mode 100644 examples/optimization/eval_optimize_loop/agent/__init__.py create mode 100644 examples/optimization/eval_optimize_loop/agent/agent.py create mode 100644 examples/optimization/eval_optimize_loop/agent/prompts/router.md create mode 100644 examples/optimization/eval_optimize_loop/agent/prompts/system.md create mode 100644 examples/optimization/eval_optimize_loop/fake/__init__.py create mode 100644 examples/optimization/eval_optimize_loop/fake/candidates.json create mode 100644 examples/optimization/eval_optimize_loop/fake/fake_agent.py create mode 100644 examples/optimization/eval_optimize_loop/fake/fake_judge.py create mode 100644 examples/optimization/eval_optimize_loop/fake/fixture_optimizer.py create mode 100644 examples/optimization/eval_optimize_loop/optimizer.json create mode 100644 examples/optimization/eval_optimize_loop/pipeline/evaluator.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/normalization.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/prompt_sandbox.py create mode 100644 examples/optimization/eval_optimize_loop/run_pipeline.py create mode 100644 examples/optimization/eval_optimize_loop/sample_output/optimization_report.json create mode 100644 examples/optimization/eval_optimize_loop/sample_output/optimization_report.md create mode 100644 examples/optimization/eval_optimize_loop/train.evalset.json create mode 100644 examples/optimization/eval_optimize_loop/val.evalset.json create mode 100644 tests/examples/optimization/eval_optimize_loop/test_fake_loop.py diff --git a/.gitignore b/.gitignore index 233248dd..2939216f 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,9 @@ coverage.xml test-ngtest-ut-trpc-agent-py.xml .pytest_cache +# Runtime artefacts for Issue #91; stable reviewed reports live in sample_output/. +examples/optimization/eval_optimize_loop/runs/ + node_modules package-lock.json pyrightconfig.json diff --git a/docs/superpowers/plans/2026-07-11-issue-91-phase-1-2.md b/docs/superpowers/plans/2026-07-11-issue-91-phase-1-2.md new file mode 100644 index 00000000..a4524f66 --- /dev/null +++ b/docs/superpowers/plans/2026-07-11-issue-91-phase-1-2.md @@ -0,0 +1,288 @@ +# Issue #91 Phase 1 and 2 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Deliver the deterministic, no-key fake evaluation-and-optimization loop for Issue #91 and defer all live optimizer integration. + +**Architecture:** All behavior remains under `examples/optimization/eval_optimize_loop/`. Phase 1 implements strict pipeline models, config validation, normalization, case comparison, independent gate, and reporting. Phase 2 uses the public `AgentEvaluator.evaluate_eval_set()` for every baseline/candidate train and validation evaluation, with deterministic fake dependencies and a prompt sandbox that restores baseline prompts. + +**Tech Stack:** Python 3.10+, Pydantic v2, pytest/pytest-asyncio, existing `trpc_agent_sdk.evaluation` APIs. + +## Global Constraints + +- Only implement Phase 1 and Phase 2 of the user-provided `trpc_agent_issue_91_solution.md`. Do not implement `--mode live`, `AgentOptimizerBackend`, or change SDK public APIs. +- Fake mode requires no key, environment variable, network access, or LLM judge; every result must be deterministic. +- Candidate prompts must be restored in `PromptSandbox.__aexit__`, including evaluator exceptions. +- A fake candidate never supplies scores: the real `AgentEvaluator` produces baseline and candidate results. +- Include exactly three train and three validation public cases. The general candidate is accepted; noop and overfit are rejected. +- Add only `examples/optimization/eval_optimize_loop/runs/` to `.gitignore`; commit reviewed `sample_output/` files. +- Never place secrets in config snapshots or reports. + +--- + +## File Structure + +- Create: `examples/optimization/eval_optimize_loop/{README.md,DESIGN.md,run_pipeline.py,optimizer.json,train.evalset.json,val.evalset.json}` +- Create: `examples/optimization/eval_optimize_loop/agent/{__init__.py,agent.py,prompts/system.md,prompts/router.md}` +- Create: `examples/optimization/eval_optimize_loop/pipeline/{__init__.py,models.py,config.py,normalization.py,comparator.py,gate.py,reporter.py,evaluator.py,prompt_sandbox.py,audit.py}` +- Create: `examples/optimization/eval_optimize_loop/fake/{__init__.py,fake_agent.py,fake_judge.py,fixture_optimizer.py,candidates.json}` +- Create: `examples/optimization/eval_optimize_loop/sample_output/{optimization_report.json,optimization_report.md}` +- Create: `tests/examples/optimization/eval_optimize_loop/{__init__.py,test_config.py,test_normalization.py,test_comparator.py,test_gate.py,test_prompt_sandbox.py,test_fake_pipeline.py,test_report_schema.py}` +- Modify: `.gitignore` + +### Task 1: Phase 1 strict data and configuration contracts + +**Files:** +- Create: `pipeline/models.py`, `pipeline/config.py`, `optimizer.json` +- Test: `tests/examples/optimization/eval_optimize_loop/test_config.py` + +**Interfaces:** +- `canonical_sha256(value: object) -> str` serializes via `json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))`. +- `load_pipeline_config(config_path: Path, *, mode: Literal["fake"]) -> PipelineConfig`. +- Pydantic models use `ConfigDict(extra="forbid")`: `CaseSnapshot`, `CandidateRecord`, `CaseDelta`, `GateRuleResult`, `GateDecision`, `OptimizationReport`. + +- [ ] **Step 1: Write failing tests** + +```python +def test_digest_is_order_independent() -> None: + assert canonical_sha256({"b": 2, "a": 1}) == canonical_sha256({"a": 1, "b": 2}) + +def test_fake_loader_needs_no_live_key(monkeypatch, config_path) -> None: + monkeypatch.delenv("TRPC_AGENT_API_KEY", raising=False) + assert load_pipeline_config(config_path, mode="fake").pipeline.reproducibility.seed == 42 +``` + +- [ ] **Step 2: Verify RED** + +Run: `python -m pytest tests/examples/optimization/eval_optimize_loop/test_config.py -q` + +Expected: fails because the configuration module is absent. + +- [ ] **Step 3: Implement minimum contract** + +```python +def canonical_sha256(value: object) -> str: + encoded = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + return f"sha256:{hashlib.sha256(encoded).hexdigest()}" + +def load_pipeline_config(config_path: Path, *, mode: Literal["fake"]) -> PipelineConfig: + if mode != "fake": + raise ValueError("Phase 1 and 2 implement only --mode fake") + raw = json.loads(config_path.read_text(encoding="utf-8")) + return PipelineConfig(raw=raw, pipeline=PipelineSettings.model_validate(raw["pipeline"])) +``` + +- [ ] **Step 4: Verify GREEN and commit** + +Run: `python -m pytest tests/examples/optimization/eval_optimize_loop/test_config.py -q` + +Commit: `git add examples/optimization/eval_optimize_loop tests/examples/optimization/eval_optimize_loop && git commit -m "feat: add issue 91 pipeline contracts"` + +### Task 2: Phase 1 evaluator normalization, case deltas, independent gate, and reporter + +**Files:** +- Create: `pipeline/normalization.py`, `pipeline/comparator.py`, `pipeline/gate.py`, `pipeline/reporter.py` +- Test: `test_normalization.py`, `test_comparator.py`, `test_gate.py`, `test_report_schema.py` + +**Interfaces:** +- `normalize_eval_results(results_by_eval_id, *, split, metric_weights) -> dict[str, CaseSnapshot]` +- `compare_case(baseline, candidate, *, epsilon, critical_case_ids) -> CaseDelta` +- `evaluate_gate(baseline, candidate, *, settings) -> GateDecision` +- `write_reports(report, output_dir) -> tuple[Path, Path]` + +- [ ] **Step 1: Write failing behavior tests** + +```python +def test_critical_pass_to_fail_is_hard_regression(snapshot_factory) -> None: + delta = compare_case(snapshot_factory("val_refund_critical", True, 1.0), + snapshot_factory("val_refund_critical", False, 0.0), + epsilon=1e-6, critical_case_ids={"val_refund_critical"}) + assert (delta.transition, delta.hard_fail_added) == ("REGRESSION", True) + +def test_gate_rejects_overfit(baseline, overfit, gate_settings) -> None: + assert evaluate_gate(baseline, overfit, settings=gate_settings).accepted is False +``` + +- [ ] **Step 2: Verify RED** + +Run: `python -m pytest tests/examples/optimization/eval_optimize_loop/test_normalization.py tests/examples/optimization/eval_optimize_loop/test_comparator.py tests/examples/optimization/eval_optimize_loop/test_gate.py -q` + +Expected: fails because normalization, comparison, and gate functions are absent. + +- [ ] **Step 3: Implement minimum behavior** + +```python +if not baseline.passed and candidate.passed: + transition = "NEW_PASS" +elif baseline.passed and not candidate.passed: + transition = "REGRESSION" +elif candidate.aggregate_score > baseline.aggregate_score + epsilon: + transition = "IMPROVED" +elif candidate.aggregate_score < baseline.aggregate_score - epsilon: + transition = "DEGRADED" +else: + transition = "UNCHANGED" +``` + +Gate acceptance requires validation score improvement, zero new hard fails, zero validation regressions, and no critical regression. Report JSON uses `model_dump(mode="json")`; Markdown lists selected candidate and every gate reason. + +- [ ] **Step 4: Verify GREEN and commit** + +Run: `python -m pytest tests/examples/optimization/eval_optimize_loop/test_normalization.py tests/examples/optimization/eval_optimize_loop/test_comparator.py tests/examples/optimization/eval_optimize_loop/test_gate.py tests/examples/optimization/eval_optimize_loop/test_report_schema.py -q` + +Commit: `git add examples/optimization/eval_optimize_loop/pipeline tests/examples/optimization/eval_optimize_loop && git commit -m "feat: add issue 91 comparison gate reports"` + +### Task 3: Phase 2 deterministic fake sources and six public cases + +**Files:** +- Create: agent prompts/agent, fake package, both evalset JSON files, candidate fixture +- Test: `test_fake_pipeline.py` + +**Interfaces:** +- `FakeSupportAgent.call_agent(query: str) -> str` rereads registered prompt files and returns deterministic JSON. +- `FixtureOptimizerBackend.load_candidates() -> list[CandidateRecord]` returns only prompt maps, source metadata, and zero cost. + +- [ ] **Step 1: Write failing fixture tests** + +```python +@pytest.mark.asyncio +async def test_fake_agent_needs_no_key(monkeypatch, fake_agent) -> None: + monkeypatch.delenv("TRPC_AGENT_API_KEY", raising=False) + assert json.loads(await fake_agent.call_agent("查询订单 A100"))["tool"] == "lookup_order" + +def test_fixture_candidates_are_unscored() -> None: + candidates = FixtureOptimizerBackend(CANDIDATES_PATH).load_candidates() + assert [c.candidate_id for c in candidates] == ["candidate_general_fix", "candidate_noop", "candidate_overfit"] + assert all(c.generation_cost_usd == 0 for c in candidates) +``` + +- [ ] **Step 2: Verify RED** + +Run: `python -m pytest tests/examples/optimization/eval_optimize_loop/test_fake_pipeline.py -q` + +Expected: fails due to missing fake components. + +- [ ] **Step 3: Implement deterministic fixture matrix** + +Create train case IDs `train_json_format`, `train_tool_argument`, `train_missing_knowledge`; validation IDs `val_json_generalization`, `val_refund_critical`, `val_stable_faq`. The fake rubric returns only 0, 0.25, 0.5, 0.75, or 1.0. `candidate_general_fix` generalizes; `candidate_noop` is unchanged; `candidate_overfit` breaks `val_refund_critical`. + +- [ ] **Step 4: Verify GREEN and commit** + +Run: `python -m pytest tests/examples/optimization/eval_optimize_loop/test_fake_pipeline.py -q` + +Commit: `git add examples/optimization/eval_optimize_loop/{agent,fake,train.evalset.json,val.evalset.json} tests/examples/optimization/eval_optimize_loop && git commit -m "feat: add issue 91 deterministic fake fixtures"` + +### Task 4: Phase 2 evaluator-first execution and prompt safety + +**Files:** +- Create: `pipeline/evaluator.py`, `pipeline/prompt_sandbox.py` +- Test: `test_prompt_sandbox.py`, `test_fake_pipeline.py` + +**Interfaces:** +- `evaluate_split(...) -> SplitReport` invokes `AgentEvaluator.evaluate_eval_set()` then normalizes its returned `eval_results_by_eval_id`. +- `PromptSandbox(target_prompt, candidate_prompts)` captures baseline, applies candidate prompts, and verifies restored baseline on all exits. + +- [ ] **Step 1: Write failing safety tests** + +```python +@pytest.mark.asyncio +async def test_sandbox_restores_after_evaluation_exception(target_prompt) -> None: + baseline = await target_prompt.read_all() + with pytest.raises(RuntimeError): + async with PromptSandbox(target_prompt, {"system_prompt": "changed", "router_prompt": "changed"}): + raise RuntimeError("evaluation failed") + assert await target_prompt.read_all() == baseline + +@pytest.mark.asyncio +async def test_fake_pipeline_uses_agent_evaluator(monkeypatch, pipeline) -> None: + called = False + async def tracked(*args, **kwargs): + nonlocal called + called = True + return await original(*args, **kwargs) + original = AgentEvaluator.evaluate_eval_set + monkeypatch.setattr(AgentEvaluator, "evaluate_eval_set", tracked) + await pipeline.evaluate_baseline() + assert called is True +``` + +- [ ] **Step 2: Verify RED** + +Run: `python -m pytest tests/examples/optimization/eval_optimize_loop/test_prompt_sandbox.py tests/examples/optimization/eval_optimize_loop/test_fake_pipeline.py -q` + +Expected: fails due to missing evaluator/sandbox behavior. + +- [ ] **Step 3: Implement and verify restoration** + +```python +async def __aexit__(self, exc_type, exc, tb) -> None: + await self._target_prompt.write_all(self._baseline) + if await self._target_prompt.read_all() != self._baseline: + raise SourceRestoreError("baseline prompt digest verification failed") +``` + +- [ ] **Step 4: Verify GREEN and commit** + +Run: `python -m pytest tests/examples/optimization/eval_optimize_loop/test_prompt_sandbox.py tests/examples/optimization/eval_optimize_loop/test_fake_pipeline.py -q` + +Commit: `git add examples/optimization/eval_optimize_loop/pipeline tests/examples/optimization/eval_optimize_loop && git commit -m "feat: evaluate fake candidates safely"` + +### Task 5: Phase 2 CLI, runtime ignore, canonical sample evidence, and final verification + +**Files:** +- Create: `run_pipeline.py`, `README.md`, `DESIGN.md`, `sample_output/optimization_report.{json,md}` +- Modify: `.gitignore` +- Test: complete example test directory + +- [ ] **Step 1: Write the failing end-to-end test** + +```python +@pytest.mark.asyncio +async def test_fake_closed_loop_is_deterministic_and_restores_prompt(tmp_path, target_prompt) -> None: + baseline = await target_prompt.read_all() + report = await run_fake_pipeline(output_dir=tmp_path) + assert report.selected_candidate_id == "candidate_general_fix" + assert {c.candidate_id: c.gate.accepted for c in report.candidates} == { + "candidate_general_fix": True, "candidate_noop": False, "candidate_overfit": False, + } + assert await target_prompt.read_all() == baseline +``` + +- [ ] **Step 2: Verify RED** + +Run: `python -m pytest tests/examples/optimization/eval_optimize_loop/test_fake_pipeline.py::test_fake_closed_loop_is_deterministic_and_restores_prompt -q` + +Expected: fails because the fake orchestration is absent. + +- [ ] **Step 3: Implement orchestration and ignore only generated runtime output** + +Append this exact rule to `.gitignore`: + +```gitignore +# Runtime artefacts for Issue #91; stable reviewed reports live in sample_output/. +examples/optimization/eval_optimize_loop/runs/ +``` + +`run_fake_pipeline` must run baseline train+validation, then each candidate train+validation inside the sandbox, compare, gate, select the best accepted candidate, and write JSON/Markdown reports. CLI supports `--mode fake` and `--output-dir`, returns zero for either normal accept or reject, and prints report paths. + +- [ ] **Step 4: Generate and inspect stable evidence** + +Run: `python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake --output-dir examples/optimization/eval_optimize_loop/sample_output` + +Expected: exit 0; selected candidate `candidate_general_fix`; rejected noop and overfit; no key required. + +- [ ] **Step 5: Complete verification and commit** + +Run: `python -m pytest tests/examples/optimization/eval_optimize_loop -q; git diff --check; git check-ignore -v examples/optimization/eval_optimize_loop/runs/probe.txt` + +Expected: tests pass, no whitespace errors, and the ignore command identifies the new rule. + +Commit: `git add .gitignore examples/optimization/eval_optimize_loop tests/examples/optimization/eval_optimize_loop && git commit -m "feat: add issue 91 no-key fake optimization loop"` + +## Final Verification Checklist + +- [ ] Run fake CLI twice to fresh directories and verify deterministic report content after excluding run timestamp fields. +- [ ] Confirm all baseline/candidate evaluations crossed `AgentEvaluator.evaluate_eval_set()`. +- [ ] Confirm baseline prompt contents after normal completion and an injected evaluation failure. +- [ ] Confirm no `runs/` content is staged and `sample_output/` is reviewed and intentionally tracked. +- [ ] Do not begin live-mode implementation until this complete test suite is green and reviewed. diff --git a/examples/optimization/eval_optimize_loop/agent/__init__.py b/examples/optimization/eval_optimize_loop/agent/__init__.py new file mode 100644 index 00000000..9902ad91 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/agent/__init__.py @@ -0,0 +1 @@ +"""Prompt files for the deterministic fake support agent.""" diff --git a/examples/optimization/eval_optimize_loop/agent/agent.py b/examples/optimization/eval_optimize_loop/agent/agent.py new file mode 100644 index 00000000..72c23948 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/agent/agent.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +from ..fake.fake_agent import FakeSupportAgent + +__all__ = ["FakeSupportAgent"] diff --git a/examples/optimization/eval_optimize_loop/agent/prompts/router.md b/examples/optimization/eval_optimize_loop/agent/prompts/router.md new file mode 100644 index 00000000..7d736074 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/agent/prompts/router.md @@ -0,0 +1 @@ +ROUTER_V1 diff --git a/examples/optimization/eval_optimize_loop/agent/prompts/system.md b/examples/optimization/eval_optimize_loop/agent/prompts/system.md new file mode 100644 index 00000000..74289c68 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/agent/prompts/system.md @@ -0,0 +1 @@ +BASELINE diff --git a/examples/optimization/eval_optimize_loop/fake/__init__.py b/examples/optimization/eval_optimize_loop/fake/__init__.py new file mode 100644 index 00000000..ca9bb1c9 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/fake/__init__.py @@ -0,0 +1 @@ +"""No-key deterministic dependencies for the fake pipeline.""" diff --git a/examples/optimization/eval_optimize_loop/fake/candidates.json b/examples/optimization/eval_optimize_loop/fake/candidates.json new file mode 100644 index 00000000..d89aec40 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/fake/candidates.json @@ -0,0 +1,5 @@ +[ + {"candidate_id": "candidate_general_fix", "prompts": {"system_prompt": "GENERAL_FIX\n", "router_prompt": "ROUTER_V1\n"}, "round_index": 1}, + {"candidate_id": "candidate_noop", "prompts": {"system_prompt": "BASELINE\n", "router_prompt": "ROUTER_V1\n"}, "round_index": 2}, + {"candidate_id": "candidate_overfit", "prompts": {"system_prompt": "OVERFIT\n", "router_prompt": "ROUTER_V1\n"}, "round_index": 3} +] diff --git a/examples/optimization/eval_optimize_loop/fake/fake_agent.py b/examples/optimization/eval_optimize_loop/fake/fake_agent.py new file mode 100644 index 00000000..2d99dabb --- /dev/null +++ b/examples/optimization/eval_optimize_loop/fake/fake_agent.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import json + +from trpc_agent_sdk.evaluation import TargetPrompt + + +def _json(route: str, tool: str, arguments: dict[str, str], answer: str) -> str: + return json.dumps({"route": route, "tool": tool, "arguments": arguments, "answer": answer}, ensure_ascii=False, sort_keys=True) + + +class FakeSupportAgent: + """Prompt-backed support router with no model or network dependency.""" + + def __init__(self, target_prompt: TargetPrompt) -> None: + self._target_prompt = target_prompt + + async def call_agent(self, query: str) -> str: + prompts = await self._target_prompt.read_all() + mode = prompts["system_prompt"].strip() + general = mode == "GENERAL_FIX" + overfit = mode == "OVERFIT" + if query == "生成订单 A100 的 JSON 工单摘要": + return _json("order_lookup", "lookup_order", {"order_id": "A100"}, "正在查询订单 A100。") if general or overfit else "订单 A100 正在查询" + if query == "查询订单 A100": + return _json("order_lookup", "lookup_order", {"order_id": "A100"}, "正在查询订单 A100。") if general or overfit else _json("order_lookup", "lookup_order", {}, "正在查询订单。") + if query == "特殊退款政策是什么": + return _json("knowledge_gap", "none", {}, "未提供该政策，不能猜测。") if overfit else _json("knowledge_gap", "none", {}, "我不确定。") + if query == "生成订单 B200 的 JSON 工单摘要": + return _json("order_lookup", "lookup_order", {"order_id": "B200"}, "正在查询订单 B200。") if general or overfit else "订单 B200 正在查询" + if query == "退款订单 R900，金额 12 USD": + if overfit: + return _json("order_lookup", "lookup_order", {"order_id": "R900"}, "正在查询订单 R900。") + return _json("refund", "refund_order", {"order_id": "R900", "currency": "USD", "amount": "12"}, "正在退款订单 R900。") + if query == "如何查看订单状态": + return _json("faq", "none", {}, "在订单详情页可查看订单状态。") + raise ValueError(f"unknown fake query: {query}") diff --git a/examples/optimization/eval_optimize_loop/fake/fake_judge.py b/examples/optimization/eval_optimize_loop/fake/fake_judge.py new file mode 100644 index 00000000..c143f410 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/fake/fake_judge.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +import json + + +def fake_rubric_score(response: str) -> float: + """A deterministic structural rubric used only in fake mode.""" + try: + payload = json.loads(response) + except json.JSONDecodeError: + return 0.0 + if not all(key in payload for key in ("route", "tool", "arguments", "answer")): + return 0.25 + return 1.0 diff --git a/examples/optimization/eval_optimize_loop/fake/fixture_optimizer.py b/examples/optimization/eval_optimize_loop/fake/fixture_optimizer.py new file mode 100644 index 00000000..1b26642a --- /dev/null +++ b/examples/optimization/eval_optimize_loop/fake/fixture_optimizer.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from ..pipeline.models import CandidateRecord + + +class FixtureOptimizerBackend: + def __init__(self, candidates_path: Path) -> None: + self._candidates_path = candidates_path + + def load_candidates(self) -> list[CandidateRecord]: + payload = json.loads(self._candidates_path.read_text(encoding="utf-8")) + return [CandidateRecord.model_validate(item) for item in payload] diff --git a/examples/optimization/eval_optimize_loop/optimizer.json b/examples/optimization/eval_optimize_loop/optimizer.json new file mode 100644 index 00000000..e1f8d24c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/optimizer.json @@ -0,0 +1,10 @@ +{ + "evaluate": { + "metrics": [{"metric_name": "final_response_avg_score", "threshold": 1.0, "criterion": {"final_response": {"text": {"match": "exact"}}}}], + "num_runs": 1 + }, + "pipeline": { + "reproducibility": {"seed": 42}, + "gate": {"min_validation_score_delta": 0.05, "critical_case_ids": ["val_refund_critical"]} + } +} diff --git a/examples/optimization/eval_optimize_loop/pipeline/evaluator.py b/examples/optimization/eval_optimize_loop/pipeline/evaluator.py new file mode 100644 index 00000000..d583e396 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/evaluator.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Awaitable, Callable, Literal + +from trpc_agent_sdk.evaluation import AgentEvaluator +from trpc_agent_sdk.evaluation._eval_config import EvalConfig +from trpc_agent_sdk.evaluation._eval_set import EvalSet + +from .models import SplitReport +from .normalization import normalize_eval_results + + +async def evaluate_split( + eval_set_path: Path, *, call_agent: Callable[[str], Awaitable[str]], split: Literal["train", "validation"], metric_weights: dict[str, float] +) -> SplitReport: + eval_set = EvalSet.model_validate_json(eval_set_path.read_text(encoding="utf-8")) + config = EvalConfig.model_validate(json.loads((eval_set_path.parent / "optimizer.json").read_text(encoding="utf-8"))["evaluate"]) + _, _, _, results_by_eval_id = await AgentEvaluator.evaluate_eval_set( + eval_set, call_agent=call_agent, eval_config=config, num_runs=config.num_runs, print_detailed_results=False + ) + return SplitReport.from_cases(list(normalize_eval_results(results_by_eval_id, split=split, metric_weights=metric_weights).values())) diff --git a/examples/optimization/eval_optimize_loop/pipeline/normalization.py b/examples/optimization/eval_optimize_loop/pipeline/normalization.py new file mode 100644 index 00000000..43e00e0c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/normalization.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from typing import Literal + +from trpc_agent_sdk.evaluation._eval_metrics import EvalStatus +from trpc_agent_sdk.evaluation._eval_result import EvalCaseResult + +from .models import CaseSnapshot, ToolCallSnapshot + + +@dataclass(frozen=True) +class FakeResponseSnapshot: + """Safely parsed details from the deterministic fake response format.""" + + payload: dict[str, object] | None + tool_calls: list[ToolCallSnapshot] + + +def parse_fake_response(response: str | None) -> FakeResponseSnapshot: + """Parse fake JSON without allowing malformed output to break reporting.""" + + if not response: + return FakeResponseSnapshot(payload=None, tool_calls=[]) + try: + payload = json.loads(response) + except (TypeError, json.JSONDecodeError): + return FakeResponseSnapshot(payload=None, tool_calls=[]) + if not isinstance(payload, dict): + return FakeResponseSnapshot(payload=None, tool_calls=[]) + tool = payload.get("tool") + arguments = payload.get("arguments") + if isinstance(tool, str) and tool not in {"", "none"}: + tool_calls = [ToolCallSnapshot(name=tool, arguments=arguments if isinstance(arguments, dict) else {})] + else: + tool_calls = [] + return FakeResponseSnapshot(payload=payload, tool_calls=tool_calls) + + +def _text(content: object) -> str | None: + parts = getattr(content, "parts", None) + if not parts: + return None + return "".join(part.text or "" for part in parts) + + +def normalize_eval_results( + results_by_eval_id: dict[str, list[EvalCaseResult]], *, split: Literal["train", "validation"], metric_weights: dict[str, float] +) -> dict[str, CaseSnapshot]: + snapshots: dict[str, CaseSnapshot] = {} + for eval_id, runs in results_by_eval_id.items(): + if not runs: + raise ValueError(f"no evaluator results for {eval_id}") + metric_scores: dict[str, list[float]] = {} + metric_thresholds: dict[str, float] = {} + metric_passed: dict[str, bool] = {} + metric_reasons: dict[str, list[str]] = {} + for result in runs: + for metric in result.overall_eval_metric_results: + if metric.score is None: + raise ValueError(f"metric {metric.metric_name} was not evaluated for {eval_id}") + metric_scores.setdefault(metric.metric_name, []).append(metric.score) + metric_thresholds[metric.metric_name] = metric.threshold + current_passed = metric.eval_status == EvalStatus.PASSED + metric_passed[metric.metric_name] = metric_passed.get(metric.metric_name, True) and current_passed + if metric.details and metric.details.reason: + metric_reasons.setdefault(metric.metric_name, []).append(metric.details.reason) + averaged = {name: sum(values) / len(values) for name, values in metric_scores.items()} + weighted = sum(averaged[name] * metric_weights.get(name, 1.0) for name in averaged) + total_weight = sum(metric_weights.get(name, 1.0) for name in averaged) + first = runs[0] + invocation = first.eval_metric_result_per_invocation[0] if first.eval_metric_result_per_invocation else None + actual = _text(invocation.actual_invocation.final_response) if invocation else None + expected = _text(invocation.expected_invocation.final_response) if invocation and invocation.expected_invocation else None + parsed_actual = parse_fake_response(actual) + parsed_expected = parse_fake_response(expected) + failure_reasons = [ + f"metric {name} did not meet threshold {metric_thresholds[name]}" + for name, passed in metric_passed.items() + if not passed + ] + snapshots[eval_id] = CaseSnapshot( + eval_id=eval_id, split=split, run_count=len(runs), passed=all(item.final_eval_status == EvalStatus.PASSED for item in runs), + hard_failed=any(item.error_message for item in runs), aggregate_score=weighted / total_weight if total_weight else 0.0, + metric_scores=averaged, metric_thresholds=metric_thresholds, metric_passed=metric_passed, + metric_reasons=metric_reasons, failure_reasons=failure_reasons, + final_response=actual, expected_response=expected, + trace_digest="sha256:" + hashlib.sha256((actual or "").encode("utf-8")).hexdigest(), + tool_calls=parsed_actual.tool_calls, + expected_tool_calls=parsed_expected.tool_calls, + ) + return snapshots diff --git a/examples/optimization/eval_optimize_loop/pipeline/prompt_sandbox.py b/examples/optimization/eval_optimize_loop/pipeline/prompt_sandbox.py new file mode 100644 index 00000000..e12e7a3d --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/prompt_sandbox.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from trpc_agent_sdk.evaluation import TargetPrompt + + +class SourceRestoreError(RuntimeError): + pass + + +class PromptSandbox: + def __init__(self, target_prompt: TargetPrompt, candidate_prompts: dict[str, str]) -> None: + self._target_prompt = target_prompt + self._candidate_prompts = candidate_prompts + self._baseline: dict[str, str] | None = None + + async def __aenter__(self) -> "PromptSandbox": + self._baseline = await self._target_prompt.read_all() + await self._target_prompt.write_all(self._candidate_prompts) + if await self._target_prompt.read_all() != self._candidate_prompts: + raise SourceRestoreError("candidate prompt verification failed") + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: + if self._baseline is None: + return + await self._target_prompt.write_all(self._baseline) + if await self._target_prompt.read_all() != self._baseline: + raise SourceRestoreError("baseline prompt restoration failed") diff --git a/examples/optimization/eval_optimize_loop/pipeline/reporter.py b/examples/optimization/eval_optimize_loop/pipeline/reporter.py index c32dd1c5..4d1f2e3c 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/reporter.py +++ b/examples/optimization/eval_optimize_loop/pipeline/reporter.py @@ -11,9 +11,71 @@ def write_reports(report: OptimizationReport, output_dir: Path) -> tuple[Path, P json_path = output_dir / "optimization_report.json" markdown_path = output_dir / "optimization_report.md" json_path.write_text(json.dumps(report.model_dump(mode="json"), ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") - lines = ["# Optimization Report", "", f"Selected candidate: {report.selected_candidate_id or 'none'}", "", "## Candidates", ""] + lines = [ + "# Optimization Report", + "", + f"- Mode: `{report.mode}`", + f"- Seed: `{report.seed}`", + f"- Selected candidate: `{report.selected_candidate_id or 'none'}`", + f"- Source integrity: `{report.source_integrity}`", + "", + "## Baseline", + "", + "| Split | Pass rate | Aggregate score |", + "| --- | ---: | ---: |", + ] + for name, split in (("train", report.baseline_train), ("validation", report.baseline_validation)): + if split is not None: + lines.append(f"| {name} | {split.pass_rate:.3f} | {split.aggregate_score:.3f} |") + lines.extend([ + "", + "## Candidates", + "", + "| Candidate | Decision | Train score | Validation score |", + "| --- | --- | ---: | ---: |", + ]) + candidate_sections: list[str] = [] for candidate in report.candidates: status = "ACCEPT" if candidate.accepted else "REJECT" - lines.append(f"- {candidate.candidate_id}: {status} — {'; '.join(candidate.reasons)}") + train_score = f"{candidate.train.aggregate_score:.3f}" if candidate.train else "-" + validation_score = f"{candidate.validation.aggregate_score:.3f}" if candidate.validation else "-" + lines.append(f"| `{candidate.candidate_id}` | {status} | {train_score} | {validation_score} |") + section = [ + "", + f"### `{candidate.candidate_id}`", + "", + f"Decision: **{status}**", + f"Reasons: {'; '.join(candidate.reasons)}", + "", + "#### Validation deltas", + "", + "| Case | Transition | Score delta | Critical |", + "| --- | --- | ---: | --- |", + ] + for delta in candidate.validation_case_deltas: + section.append(f"| `{delta.eval_id}` | {delta.transition} | {delta.score_delta:+.3f} | {'yes' if delta.critical else 'no'} |") + section.extend(["", "#### Gate rules", "", "| Rule | Passed | Actual | Expected |", "| --- | --- | ---: | ---: |"]) + if candidate.gate: + for rule in candidate.gate.rules: + section.append(f"| `{rule.rule}` | {'yes' if rule.passed else 'no'} | `{rule.actual}` | `{rule.expected}` |") + candidate_sections.extend(section) + lines.extend([ + "", + "## Validation deltas", + "", + "Each candidate section below contains its complete validation case delta table.", + "", + "## Gate rules", + "", + "Each candidate section below contains its complete gate rule table.", + *candidate_sections, + ]) + lines.extend([ + "## Reproduction", + "", + "```text", + "python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake --output-dir ", + "```", + ]) markdown_path.write_text("\n".join(lines) + "\n", encoding="utf-8") return json_path, markdown_path diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py new file mode 100644 index 00000000..1fc0b736 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import argparse +import asyncio +import sys +from pathlib import Path + +# When this file is executed directly (the documented CLI path), Python puts +# only this directory on sys.path. Bootstrap the repository root before +# importing the SDK and the example package so the CLI behaves like +# ``python -m ...`` and like the pytest entry point. +HERE = Path(__file__).resolve().parent +REPO_ROOT = HERE.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from trpc_agent_sdk.evaluation import TargetPrompt + +from examples.optimization.eval_optimize_loop.fake.fake_agent import FakeSupportAgent +from examples.optimization.eval_optimize_loop.fake.fixture_optimizer import FixtureOptimizerBackend +from examples.optimization.eval_optimize_loop.pipeline.comparator import compare_case +from examples.optimization.eval_optimize_loop.pipeline.config import load_pipeline_config +from examples.optimization.eval_optimize_loop.pipeline.evaluator import evaluate_split +from examples.optimization.eval_optimize_loop.pipeline.gate import evaluate_gate +from examples.optimization.eval_optimize_loop.pipeline.models import CandidateReport, OptimizationReport +from examples.optimization.eval_optimize_loop.pipeline.prompt_sandbox import PromptSandbox +from examples.optimization.eval_optimize_loop.pipeline.reporter import write_reports + + + +async def run_fake_pipeline(*, output_dir: Path) -> OptimizationReport: + config = load_pipeline_config(HERE / "optimizer.json", mode="fake") + target = TargetPrompt().add_path("system_prompt", str(HERE / "agent" / "prompts" / "system.md")).add_path("router_prompt", str(HERE / "agent" / "prompts" / "router.md")) + fake_agent = FakeSupportAgent(target) + evaluate = lambda path, split: evaluate_split(path, call_agent=fake_agent.call_agent, split=split, metric_weights=config.pipeline.metric_weights) + baseline_train = await evaluate(HERE / "train.evalset.json", "train") + baseline_validation = await evaluate(HERE / "val.evalset.json", "validation") + report = OptimizationReport.empty(mode="fake", seed=config.pipeline.reproducibility.seed) + report.baseline_train = baseline_train + report.baseline_validation = baseline_validation + for candidate in FixtureOptimizerBackend(HERE / "fake" / "candidates.json").load_candidates(): + async with PromptSandbox(target, candidate.prompts): + train = await evaluate(HERE / "train.evalset.json", "train") + validation = await evaluate(HERE / "val.evalset.json", "validation") + deltas = [compare_case(base, next(item for item in validation.cases if item.eval_id == base.eval_id), epsilon=config.pipeline.scoring_epsilon, critical_case_ids=set(config.pipeline.gate.critical_case_ids)) for base in baseline_validation.cases] + decision = evaluate_gate(baseline_validation, validation, settings=config.pipeline.gate, case_deltas=deltas, train_score_delta=train.aggregate_score - baseline_train.aggregate_score) + report.candidates.append(CandidateReport(candidate_id=candidate.candidate_id, accepted=decision.accepted, reasons=decision.reasons, train=train, validation=validation, gate=decision, validation_case_deltas=deltas)) + accepted = [candidate for candidate in report.candidates if candidate.accepted] + report.selected_candidate_id = accepted[0].candidate_id if accepted else None + write_reports(report, output_dir) + return report + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--mode", choices=["fake"], default="fake") + parser.add_argument("--output-dir", type=Path, default=HERE / "runs" / "manual") + args = parser.parse_args() + report = asyncio.run(run_fake_pipeline(output_dir=args.output_dir)) + print(f"Decision: {'ACCEPT' if report.selected_candidate_id else 'REJECT'}") + print(f"Selected candidate: {report.selected_candidate_id or 'none'}") + print(f"JSON report: {(args.output_dir / 'optimization_report.json').resolve()}") + print(f"Markdown report: {(args.output_dir / 'optimization_report.md').resolve()}") + print("Source prompt updated: no") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json new file mode 100644 index 00000000..b38dc8ff --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json @@ -0,0 +1,1171 @@ +{ + "baseline_train": { + "aggregate_score": 0.0, + "cases": [ + { + "aggregate_score": 0.0, + "eval_id": "train_missing_knowledge", + "expected_response": "{\"answer\": \"未提供该政策，不能猜测。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", + "expected_tool_calls": [], + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "final_response": "{\"answer\": \"我不确定。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", + "hard_failed": false, + "metric_passed": { + "final_response_avg_score": false + }, + "metric_reasons": {}, + "metric_scores": { + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [], + "trace_digest": "sha256:3f059c7504becc8cb98f605431f3c3714a0e70b4e62931895b888bba4ffcf9ed" + }, + { + "aggregate_score": 0.0, + "eval_id": "train_json_format", + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "final_response": "订单 A100 正在查询", + "hard_failed": false, + "metric_passed": { + "final_response_avg_score": false + }, + "metric_reasons": {}, + "metric_scores": { + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [], + "trace_digest": "sha256:65fdc3d48fd18eb37b2aa5138ee27435257ecbd0ef582c523a42673d87b04b77" + }, + { + "aggregate_score": 0.0, + "eval_id": "train_tool_argument", + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "final_response": "{\"answer\": \"正在查询订单。\", \"arguments\": {}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "final_response_avg_score": false + }, + "metric_reasons": {}, + "metric_scores": { + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": {}, + "name": "lookup_order" + } + ], + "trace_digest": "sha256:f086f4b22284734f1462eee58a4d204a31361c26d67449e5b23d292aaf2642bf" + } + ], + "pass_rate": 0.0 + }, + "baseline_validation": { + "aggregate_score": 0.6666666666666666, + "cases": [ + { + "aggregate_score": 1.0, + "eval_id": "val_stable_faq", + "expected_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "expected_tool_calls": [], + "failure_reasons": [], + "final_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "hard_failed": false, + "metric_passed": { + "final_response_avg_score": true + }, + "metric_reasons": {}, + "metric_scores": { + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [], + "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" + }, + { + "aggregate_score": 0.0, + "eval_id": "val_json_generalization", + "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "final_response": "订单 B200 正在查询", + "hard_failed": false, + "metric_passed": { + "final_response_avg_score": false + }, + "metric_reasons": {}, + "metric_scores": { + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "validation", + "tool_calls": [], + "trace_digest": "sha256:8684ddee9351120c1bf48affbd280c5b73835b2c1d58d04682622b577c37a455" + }, + { + "aggregate_score": 1.0, + "eval_id": "val_refund_critical", + "expected_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "amount": "12", + "currency": "USD", + "order_id": "R900" + }, + "name": "refund_order" + } + ], + "failure_reasons": [], + "final_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "hard_failed": false, + "metric_passed": { + "final_response_avg_score": true + }, + "metric_reasons": {}, + "metric_scores": { + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [ + { + "arguments": { + "amount": "12", + "currency": "USD", + "order_id": "R900" + }, + "name": "refund_order" + } + ], + "trace_digest": "sha256:19bb632d52c6f72bc177991367b8e146fd76b0fb3a68bc8ee0622be9fa4df8a8" + } + ], + "pass_rate": 0.6666666666666666 + }, + "candidates": [ + { + "accepted": true, + "candidate_id": "candidate_general_fix", + "gate": { + "accepted": true, + "reasons": [ + "candidate passed all independent gate rules" + ], + "risk_level": "low", + "rules": [ + { + "actual": 0.33333333333333337, + "expected": 0.05, + "passed": true, + "reason": "validation aggregate score must improve", + "rule": "validation_score_improved" + }, + { + "actual": 0.33333333333333337, + "expected": 0.0, + "passed": true, + "reason": "validation pass rate must not decline", + "rule": "validation_pass_rate_not_worse" + }, + { + "actual": 0, + "expected": 0, + "passed": true, + "reason": "new hard failures are not allowed", + "rule": "new_hard_fails" + }, + { + "actual": 0, + "expected": 0, + "passed": true, + "reason": "validation regressions exceed the limit", + "rule": "validation_regressions" + }, + { + "actual": false, + "expected": false, + "passed": true, + "reason": "critical validation cases must not regress", + "rule": "no_critical_regression" + }, + { + "actual": false, + "expected": false, + "passed": true, + "reason": "train improvement with validation decline is rejected", + "rule": "no_overfit" + } + ] + }, + "reasons": [ + "candidate passed all independent gate rules" + ], + "train": { + "aggregate_score": 0.6666666666666666, + "cases": [ + { + "aggregate_score": 0.0, + "eval_id": "train_missing_knowledge", + "expected_response": "{\"answer\": \"未提供该政策，不能猜测。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", + "expected_tool_calls": [], + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "final_response": "{\"answer\": \"我不确定。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", + "hard_failed": false, + "metric_passed": { + "final_response_avg_score": false + }, + "metric_reasons": {}, + "metric_scores": { + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [], + "trace_digest": "sha256:3f059c7504becc8cb98f605431f3c3714a0e70b4e62931895b888bba4ffcf9ed" + }, + { + "aggregate_score": 1.0, + "eval_id": "train_json_format", + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_reasons": [], + "final_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "final_response_avg_score": true + }, + "metric_reasons": {}, + "metric_scores": { + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "trace_digest": "sha256:5da8db545d2c7b26c93058b6619ab49dd59c1348f6b032f05c451c2709b66e6e" + }, + { + "aggregate_score": 1.0, + "eval_id": "train_tool_argument", + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_reasons": [], + "final_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "final_response_avg_score": true + }, + "metric_reasons": {}, + "metric_scores": { + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "trace_digest": "sha256:5da8db545d2c7b26c93058b6619ab49dd59c1348f6b032f05c451c2709b66e6e" + } + ], + "pass_rate": 0.6666666666666666 + }, + "validation": { + "aggregate_score": 1.0, + "cases": [ + { + "aggregate_score": 1.0, + "eval_id": "val_stable_faq", + "expected_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "expected_tool_calls": [], + "failure_reasons": [], + "final_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "hard_failed": false, + "metric_passed": { + "final_response_avg_score": true + }, + "metric_reasons": {}, + "metric_scores": { + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [], + "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" + }, + { + "aggregate_score": 1.0, + "eval_id": "val_json_generalization", + "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "failure_reasons": [], + "final_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "final_response_avg_score": true + }, + "metric_reasons": {}, + "metric_scores": { + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "trace_digest": "sha256:99d82b8a08b7eeeda5b5f41d08a3afc1c9f4f22879605c257b78458c4ef9e561" + }, + { + "aggregate_score": 1.0, + "eval_id": "val_refund_critical", + "expected_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "amount": "12", + "currency": "USD", + "order_id": "R900" + }, + "name": "refund_order" + } + ], + "failure_reasons": [], + "final_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "hard_failed": false, + "metric_passed": { + "final_response_avg_score": true + }, + "metric_reasons": {}, + "metric_scores": { + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [ + { + "arguments": { + "amount": "12", + "currency": "USD", + "order_id": "R900" + }, + "name": "refund_order" + } + ], + "trace_digest": "sha256:19bb632d52c6f72bc177991367b8e146fd76b0fb3a68bc8ee0622be9fa4df8a8" + } + ], + "pass_rate": 1.0 + }, + "validation_case_deltas": [ + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_passed": true, + "candidate_score": 1.0, + "critical": false, + "eval_id": "val_stable_faq", + "hard_fail_added": false, + "metric_deltas": { + "final_response_avg_score": 0.0 + }, + "score_delta": 0.0, + "transition": "UNCHANGED" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_passed": true, + "candidate_score": 1.0, + "critical": false, + "eval_id": "val_json_generalization", + "hard_fail_added": false, + "metric_deltas": { + "final_response_avg_score": 1.0 + }, + "score_delta": 1.0, + "transition": "NEW_PASS" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_passed": true, + "candidate_score": 1.0, + "critical": true, + "eval_id": "val_refund_critical", + "hard_fail_added": false, + "metric_deltas": { + "final_response_avg_score": 0.0 + }, + "score_delta": 0.0, + "transition": "UNCHANGED" + } + ] + }, + { + "accepted": false, + "candidate_id": "candidate_noop", + "gate": { + "accepted": false, + "reasons": [ + "validation aggregate score must improve" + ], + "risk_level": "medium", + "rules": [ + { + "actual": 0.0, + "expected": 0.05, + "passed": false, + "reason": "validation aggregate score must improve", + "rule": "validation_score_improved" + }, + { + "actual": 0.0, + "expected": 0.0, + "passed": true, + "reason": "validation pass rate must not decline", + "rule": "validation_pass_rate_not_worse" + }, + { + "actual": 0, + "expected": 0, + "passed": true, + "reason": "new hard failures are not allowed", + "rule": "new_hard_fails" + }, + { + "actual": 0, + "expected": 0, + "passed": true, + "reason": "validation regressions exceed the limit", + "rule": "validation_regressions" + }, + { + "actual": false, + "expected": false, + "passed": true, + "reason": "critical validation cases must not regress", + "rule": "no_critical_regression" + }, + { + "actual": false, + "expected": false, + "passed": true, + "reason": "train improvement with validation decline is rejected", + "rule": "no_overfit" + } + ] + }, + "reasons": [ + "validation aggregate score must improve" + ], + "train": { + "aggregate_score": 0.0, + "cases": [ + { + "aggregate_score": 0.0, + "eval_id": "train_missing_knowledge", + "expected_response": "{\"answer\": \"未提供该政策，不能猜测。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", + "expected_tool_calls": [], + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "final_response": "{\"answer\": \"我不确定。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", + "hard_failed": false, + "metric_passed": { + "final_response_avg_score": false + }, + "metric_reasons": {}, + "metric_scores": { + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [], + "trace_digest": "sha256:3f059c7504becc8cb98f605431f3c3714a0e70b4e62931895b888bba4ffcf9ed" + }, + { + "aggregate_score": 0.0, + "eval_id": "train_json_format", + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "final_response": "订单 A100 正在查询", + "hard_failed": false, + "metric_passed": { + "final_response_avg_score": false + }, + "metric_reasons": {}, + "metric_scores": { + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [], + "trace_digest": "sha256:65fdc3d48fd18eb37b2aa5138ee27435257ecbd0ef582c523a42673d87b04b77" + }, + { + "aggregate_score": 0.0, + "eval_id": "train_tool_argument", + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "final_response": "{\"answer\": \"正在查询订单。\", \"arguments\": {}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "final_response_avg_score": false + }, + "metric_reasons": {}, + "metric_scores": { + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": {}, + "name": "lookup_order" + } + ], + "trace_digest": "sha256:f086f4b22284734f1462eee58a4d204a31361c26d67449e5b23d292aaf2642bf" + } + ], + "pass_rate": 0.0 + }, + "validation": { + "aggregate_score": 0.6666666666666666, + "cases": [ + { + "aggregate_score": 1.0, + "eval_id": "val_stable_faq", + "expected_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "expected_tool_calls": [], + "failure_reasons": [], + "final_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "hard_failed": false, + "metric_passed": { + "final_response_avg_score": true + }, + "metric_reasons": {}, + "metric_scores": { + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [], + "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" + }, + { + "aggregate_score": 0.0, + "eval_id": "val_json_generalization", + "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "final_response": "订单 B200 正在查询", + "hard_failed": false, + "metric_passed": { + "final_response_avg_score": false + }, + "metric_reasons": {}, + "metric_scores": { + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "validation", + "tool_calls": [], + "trace_digest": "sha256:8684ddee9351120c1bf48affbd280c5b73835b2c1d58d04682622b577c37a455" + }, + { + "aggregate_score": 1.0, + "eval_id": "val_refund_critical", + "expected_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "amount": "12", + "currency": "USD", + "order_id": "R900" + }, + "name": "refund_order" + } + ], + "failure_reasons": [], + "final_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "hard_failed": false, + "metric_passed": { + "final_response_avg_score": true + }, + "metric_reasons": {}, + "metric_scores": { + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [ + { + "arguments": { + "amount": "12", + "currency": "USD", + "order_id": "R900" + }, + "name": "refund_order" + } + ], + "trace_digest": "sha256:19bb632d52c6f72bc177991367b8e146fd76b0fb3a68bc8ee0622be9fa4df8a8" + } + ], + "pass_rate": 0.6666666666666666 + }, + "validation_case_deltas": [ + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_passed": true, + "candidate_score": 1.0, + "critical": false, + "eval_id": "val_stable_faq", + "hard_fail_added": false, + "metric_deltas": { + "final_response_avg_score": 0.0 + }, + "score_delta": 0.0, + "transition": "UNCHANGED" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_passed": false, + "candidate_score": 0.0, + "critical": false, + "eval_id": "val_json_generalization", + "hard_fail_added": false, + "metric_deltas": { + "final_response_avg_score": 0.0 + }, + "score_delta": 0.0, + "transition": "UNCHANGED" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_passed": true, + "candidate_score": 1.0, + "critical": true, + "eval_id": "val_refund_critical", + "hard_fail_added": false, + "metric_deltas": { + "final_response_avg_score": 0.0 + }, + "score_delta": 0.0, + "transition": "UNCHANGED" + } + ] + }, + { + "accepted": false, + "candidate_id": "candidate_overfit", + "gate": { + "accepted": false, + "reasons": [ + "validation aggregate score must improve", + "new hard failures are not allowed", + "validation regressions exceed the limit", + "critical validation cases must not regress" + ], + "risk_level": "high", + "rules": [ + { + "actual": 0.0, + "expected": 0.05, + "passed": false, + "reason": "validation aggregate score must improve", + "rule": "validation_score_improved" + }, + { + "actual": 0.0, + "expected": 0.0, + "passed": true, + "reason": "validation pass rate must not decline", + "rule": "validation_pass_rate_not_worse" + }, + { + "actual": 1, + "expected": 0, + "passed": false, + "reason": "new hard failures are not allowed", + "rule": "new_hard_fails" + }, + { + "actual": 1, + "expected": 0, + "passed": false, + "reason": "validation regressions exceed the limit", + "rule": "validation_regressions" + }, + { + "actual": true, + "expected": false, + "passed": false, + "reason": "critical validation cases must not regress", + "rule": "no_critical_regression" + }, + { + "actual": false, + "expected": false, + "passed": true, + "reason": "train improvement with validation decline is rejected", + "rule": "no_overfit" + } + ] + }, + "reasons": [ + "validation aggregate score must improve", + "new hard failures are not allowed", + "validation regressions exceed the limit", + "critical validation cases must not regress" + ], + "train": { + "aggregate_score": 1.0, + "cases": [ + { + "aggregate_score": 1.0, + "eval_id": "train_missing_knowledge", + "expected_response": "{\"answer\": \"未提供该政策，不能猜测。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", + "expected_tool_calls": [], + "failure_reasons": [], + "final_response": "{\"answer\": \"未提供该政策，不能猜测。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", + "hard_failed": false, + "metric_passed": { + "final_response_avg_score": true + }, + "metric_reasons": {}, + "metric_scores": { + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "train", + "tool_calls": [], + "trace_digest": "sha256:22d90b34cb10f26e6a207ed065e82b862b238989d23eda1b8dbd8d4072208e0f" + }, + { + "aggregate_score": 1.0, + "eval_id": "train_json_format", + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_reasons": [], + "final_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "final_response_avg_score": true + }, + "metric_reasons": {}, + "metric_scores": { + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "trace_digest": "sha256:5da8db545d2c7b26c93058b6619ab49dd59c1348f6b032f05c451c2709b66e6e" + }, + { + "aggregate_score": 1.0, + "eval_id": "train_tool_argument", + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_reasons": [], + "final_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "final_response_avg_score": true + }, + "metric_reasons": {}, + "metric_scores": { + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "trace_digest": "sha256:5da8db545d2c7b26c93058b6619ab49dd59c1348f6b032f05c451c2709b66e6e" + } + ], + "pass_rate": 1.0 + }, + "validation": { + "aggregate_score": 0.6666666666666666, + "cases": [ + { + "aggregate_score": 1.0, + "eval_id": "val_stable_faq", + "expected_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "expected_tool_calls": [], + "failure_reasons": [], + "final_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "hard_failed": false, + "metric_passed": { + "final_response_avg_score": true + }, + "metric_reasons": {}, + "metric_scores": { + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [], + "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" + }, + { + "aggregate_score": 1.0, + "eval_id": "val_json_generalization", + "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "failure_reasons": [], + "final_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "final_response_avg_score": true + }, + "metric_reasons": {}, + "metric_scores": { + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "trace_digest": "sha256:99d82b8a08b7eeeda5b5f41d08a3afc1c9f4f22879605c257b78458c4ef9e561" + }, + { + "aggregate_score": 0.0, + "eval_id": "val_refund_critical", + "expected_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "amount": "12", + "currency": "USD", + "order_id": "R900" + }, + "name": "refund_order" + } + ], + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "final_response": "{\"answer\": \"正在查询订单 R900。\", \"arguments\": {\"order_id\": \"R900\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "final_response_avg_score": false + }, + "metric_reasons": {}, + "metric_scores": { + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "validation", + "tool_calls": [ + { + "arguments": { + "order_id": "R900" + }, + "name": "lookup_order" + } + ], + "trace_digest": "sha256:254559dde2e80e22b22f6c4cb0d37c61bbd5a8200e19d46232c76f1b0a5d8604" + } + ], + "pass_rate": 0.6666666666666666 + }, + "validation_case_deltas": [ + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_passed": true, + "candidate_score": 1.0, + "critical": false, + "eval_id": "val_stable_faq", + "hard_fail_added": false, + "metric_deltas": { + "final_response_avg_score": 0.0 + }, + "score_delta": 0.0, + "transition": "UNCHANGED" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_passed": true, + "candidate_score": 1.0, + "critical": false, + "eval_id": "val_json_generalization", + "hard_fail_added": false, + "metric_deltas": { + "final_response_avg_score": 1.0 + }, + "score_delta": 1.0, + "transition": "NEW_PASS" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_passed": false, + "candidate_score": 0.0, + "critical": true, + "eval_id": "val_refund_critical", + "hard_fail_added": true, + "metric_deltas": { + "final_response_avg_score": -1.0 + }, + "score_delta": -1.0, + "transition": "REGRESSION" + } + ] + } + ], + "mode": "fake", + "schema_version": "1.0", + "seed": 42, + "selected_candidate_id": "candidate_general_fix", + "source_integrity": "restored" +} diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md new file mode 100644 index 00000000..fe283fab --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md @@ -0,0 +1,106 @@ +# Optimization Report + +- Mode: `fake` +- Seed: `42` +- Selected candidate: `candidate_general_fix` +- Source integrity: `restored` + +## Baseline + +| Split | Pass rate | Aggregate score | +| --- | ---: | ---: | +| train | 0.000 | 0.000 | +| validation | 0.667 | 0.667 | + +## Candidates + +| Candidate | Decision | Train score | Validation score | +| --- | --- | ---: | ---: | +| `candidate_general_fix` | ACCEPT | 0.667 | 1.000 | +| `candidate_noop` | REJECT | 0.000 | 0.667 | +| `candidate_overfit` | REJECT | 1.000 | 0.667 | + +## Validation deltas + +Each candidate section below contains its complete validation case delta table. + +## Gate rules + +Each candidate section below contains its complete gate rule table. + +### `candidate_general_fix` + +Decision: **ACCEPT** +Reasons: candidate passed all independent gate rules + +#### Validation deltas + +| Case | Transition | Score delta | Critical | +| --- | --- | ---: | --- | +| `val_stable_faq` | UNCHANGED | +0.000 | no | +| `val_json_generalization` | NEW_PASS | +1.000 | no | +| `val_refund_critical` | UNCHANGED | +0.000 | yes | + +#### Gate rules + +| Rule | Passed | Actual | Expected | +| --- | --- | ---: | ---: | +| `validation_score_improved` | yes | `0.33333333333333337` | `0.05` | +| `validation_pass_rate_not_worse` | yes | `0.33333333333333337` | `0.0` | +| `new_hard_fails` | yes | `0` | `0` | +| `validation_regressions` | yes | `0` | `0` | +| `no_critical_regression` | yes | `False` | `False` | +| `no_overfit` | yes | `False` | `False` | + +### `candidate_noop` + +Decision: **REJECT** +Reasons: validation aggregate score must improve + +#### Validation deltas + +| Case | Transition | Score delta | Critical | +| --- | --- | ---: | --- | +| `val_stable_faq` | UNCHANGED | +0.000 | no | +| `val_json_generalization` | UNCHANGED | +0.000 | no | +| `val_refund_critical` | UNCHANGED | +0.000 | yes | + +#### Gate rules + +| Rule | Passed | Actual | Expected | +| --- | --- | ---: | ---: | +| `validation_score_improved` | no | `0.0` | `0.05` | +| `validation_pass_rate_not_worse` | yes | `0.0` | `0.0` | +| `new_hard_fails` | yes | `0` | `0` | +| `validation_regressions` | yes | `0` | `0` | +| `no_critical_regression` | yes | `False` | `False` | +| `no_overfit` | yes | `False` | `False` | + +### `candidate_overfit` + +Decision: **REJECT** +Reasons: validation aggregate score must improve; new hard failures are not allowed; validation regressions exceed the limit; critical validation cases must not regress + +#### Validation deltas + +| Case | Transition | Score delta | Critical | +| --- | --- | ---: | --- | +| `val_stable_faq` | UNCHANGED | +0.000 | no | +| `val_json_generalization` | NEW_PASS | +1.000 | no | +| `val_refund_critical` | REGRESSION | -1.000 | yes | + +#### Gate rules + +| Rule | Passed | Actual | Expected | +| --- | --- | ---: | ---: | +| `validation_score_improved` | no | `0.0` | `0.05` | +| `validation_pass_rate_not_worse` | yes | `0.0` | `0.0` | +| `new_hard_fails` | no | `1` | `0` | +| `validation_regressions` | no | `1` | `0` | +| `no_critical_regression` | no | `True` | `False` | +| `no_overfit` | yes | `False` | `False` | +## Reproduction + +```text +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake --output-dir +``` diff --git a/examples/optimization/eval_optimize_loop/train.evalset.json b/examples/optimization/eval_optimize_loop/train.evalset.json new file mode 100644 index 00000000..79c2ef56 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/train.evalset.json @@ -0,0 +1,8 @@ +{ + "eval_set_id": "issue_91_train", + "eval_cases": [ + {"eval_id": "train_json_format", "conversation": [{"user_content": {"role": "user", "parts": [{"text": "生成订单 A100 的 JSON 工单摘要"}]}, "final_response": {"role": "model", "parts": [{"text": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}"}]}}]}, + {"eval_id": "train_tool_argument", "conversation": [{"user_content": {"role": "user", "parts": [{"text": "查询订单 A100"}]}, "final_response": {"role": "model", "parts": [{"text": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}"}]}}]}, + {"eval_id": "train_missing_knowledge", "conversation": [{"user_content": {"role": "user", "parts": [{"text": "特殊退款政策是什么"}]}, "final_response": {"role": "model", "parts": [{"text": "{\"answer\": \"未提供该政策，不能猜测。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}"}]}}]} + ] +} diff --git a/examples/optimization/eval_optimize_loop/val.evalset.json b/examples/optimization/eval_optimize_loop/val.evalset.json new file mode 100644 index 00000000..4d9d7373 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/val.evalset.json @@ -0,0 +1,8 @@ +{ + "eval_set_id": "issue_91_validation", + "eval_cases": [ + {"eval_id": "val_json_generalization", "conversation": [{"user_content": {"role": "user", "parts": [{"text": "生成订单 B200 的 JSON 工单摘要"}]}, "final_response": {"role": "model", "parts": [{"text": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}"}]}}]}, + {"eval_id": "val_refund_critical", "conversation": [{"user_content": {"role": "user", "parts": [{"text": "退款订单 R900，金额 12 USD"}]}, "final_response": {"role": "model", "parts": [{"text": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}"}]}}]}, + {"eval_id": "val_stable_faq", "conversation": [{"user_content": {"role": "user", "parts": [{"text": "如何查看订单状态"}]}, "final_response": {"role": "model", "parts": [{"text": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}"}]}}]} + ] +} diff --git a/tests/examples/optimization/eval_optimize_loop/test_fake_loop.py b/tests/examples/optimization/eval_optimize_loop/test_fake_loop.py new file mode 100644 index 00000000..8a16f165 --- /dev/null +++ b/tests/examples/optimization/eval_optimize_loop/test_fake_loop.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop.run_pipeline import run_fake_pipeline + + +@pytest.mark.asyncio +async def test_fake_closed_loop_uses_three_fixture_candidates_and_restores_prompts(tmp_path: Path) -> None: + report = await run_fake_pipeline(output_dir=tmp_path) + assert report.selected_candidate_id == "candidate_general_fix" + assert {candidate.candidate_id: candidate.accepted for candidate in report.candidates} == { + "candidate_general_fix": True, + "candidate_noop": False, + "candidate_overfit": False, + } + assert (tmp_path / "optimization_report.json").is_file() + assert (tmp_path / "optimization_report.md").is_file() + + +@pytest.mark.asyncio +async def test_fake_loop_does_not_require_live_model_key(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.delenv("TRPC_AGENT_API_KEY", raising=False) + report = await run_fake_pipeline(output_dir=tmp_path) + assert report.mode == "fake" + + +def test_fake_cli_runs_directly_from_repository_root(tmp_path: Path) -> None: + repo_root = Path(__file__).resolve().parents[4] + result = subprocess.run( + [ + sys.executable, + "examples/optimization/eval_optimize_loop/run_pipeline.py", + "--mode", + "fake", + "--output-dir", + str(tmp_path), + ], + cwd=repo_root, + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + assert "Selected candidate: candidate_general_fix" in result.stdout diff --git a/tests/examples/optimization/eval_optimize_loop/test_phase1_core.py b/tests/examples/optimization/eval_optimize_loop/test_phase1_core.py index 07438839..560d5098 100644 --- a/tests/examples/optimization/eval_optimize_loop/test_phase1_core.py +++ b/tests/examples/optimization/eval_optimize_loop/test_phase1_core.py @@ -9,6 +9,7 @@ from examples.optimization.eval_optimize_loop.pipeline.comparator import compare_case from examples.optimization.eval_optimize_loop.pipeline.config import canonical_sha256, load_pipeline_config from examples.optimization.eval_optimize_loop.pipeline.gate import evaluate_gate +from examples.optimization.eval_optimize_loop.pipeline.normalization import parse_fake_response from examples.optimization.eval_optimize_loop.pipeline.models import ( CandidateReport, CaseSnapshot, @@ -91,4 +92,17 @@ def test_reporter_writes_json_and_markdown(tmp_path: Path) -> None: report.candidates.append(CandidateReport(candidate_id="candidate_noop", accepted=False, reasons=["no improvement"])) json_path, markdown_path = write_reports(report, tmp_path) assert json.loads(json_path.read_text(encoding="utf-8"))["schema_version"] == "1.0" - assert "Selected candidate" in markdown_path.read_text(encoding="utf-8") + markdown = markdown_path.read_text(encoding="utf-8") + assert "Selected candidate" in markdown + assert "Baseline" in markdown + assert "Validation deltas" in markdown + assert "Gate rules" in markdown + + +def test_fake_response_parser_preserves_tool_name_and_arguments() -> None: + snapshot = parse_fake_response( + '{"route":"order_lookup","tool":"lookup_order","arguments":{"order_id":"A100"},' + '"answer":"正在查询订单 A100。"}' + ) + assert snapshot.tool_calls[0].name == "lookup_order" + assert snapshot.tool_calls[0].arguments == {"order_id": "A100"} From 53c3a63f77e838fe89a2bcaaf401aab7eb493d19 Mon Sep 17 00:00:00 2001 From: shiquan_zhan Date: Sat, 11 Jul 2026 02:48:06 +0800 Subject: [PATCH 03/15] fix(eval): isolate fake prompt workspace --- .../eval_optimize_loop/run_pipeline.py | 35 +++++++++++-------- .../eval_optimize_loop/test_fake_loop.py | 5 +++ 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index 1fc0b736..26169732 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -3,6 +3,7 @@ import argparse import asyncio import sys +import tempfile from pathlib import Path # When this file is executed directly (the documented CLI path), Python puts @@ -30,21 +31,27 @@ async def run_fake_pipeline(*, output_dir: Path) -> OptimizationReport: config = load_pipeline_config(HERE / "optimizer.json", mode="fake") - target = TargetPrompt().add_path("system_prompt", str(HERE / "agent" / "prompts" / "system.md")).add_path("router_prompt", str(HERE / "agent" / "prompts" / "router.md")) - fake_agent = FakeSupportAgent(target) - evaluate = lambda path, split: evaluate_split(path, call_agent=fake_agent.call_agent, split=split, metric_weights=config.pipeline.metric_weights) - baseline_train = await evaluate(HERE / "train.evalset.json", "train") - baseline_validation = await evaluate(HERE / "val.evalset.json", "validation") report = OptimizationReport.empty(mode="fake", seed=config.pipeline.reproducibility.seed) - report.baseline_train = baseline_train - report.baseline_validation = baseline_validation - for candidate in FixtureOptimizerBackend(HERE / "fake" / "candidates.json").load_candidates(): - async with PromptSandbox(target, candidate.prompts): - train = await evaluate(HERE / "train.evalset.json", "train") - validation = await evaluate(HERE / "val.evalset.json", "validation") - deltas = [compare_case(base, next(item for item in validation.cases if item.eval_id == base.eval_id), epsilon=config.pipeline.scoring_epsilon, critical_case_ids=set(config.pipeline.gate.critical_case_ids)) for base in baseline_validation.cases] - decision = evaluate_gate(baseline_validation, validation, settings=config.pipeline.gate, case_deltas=deltas, train_score_delta=train.aggregate_score - baseline_train.aggregate_score) - report.candidates.append(CandidateReport(candidate_id=candidate.candidate_id, accepted=decision.accepted, reasons=decision.reasons, train=train, validation=validation, gate=decision, validation_case_deltas=deltas)) + with tempfile.TemporaryDirectory(prefix="trpc-agent-issue91-") as temporary_dir: + prompt_dir = Path(temporary_dir) + source_prompt_dir = HERE / "agent" / "prompts" + for prompt_name in ("system.md", "router.md"): + (prompt_dir / prompt_name).write_text((source_prompt_dir / prompt_name).read_text(encoding="utf-8"), encoding="utf-8") + + target = TargetPrompt().add_path("system_prompt", str(prompt_dir / "system.md")).add_path("router_prompt", str(prompt_dir / "router.md")) + fake_agent = FakeSupportAgent(target) + evaluate = lambda path, split: evaluate_split(path, call_agent=fake_agent.call_agent, split=split, metric_weights=config.pipeline.metric_weights) + baseline_train = await evaluate(HERE / "train.evalset.json", "train") + baseline_validation = await evaluate(HERE / "val.evalset.json", "validation") + report.baseline_train = baseline_train + report.baseline_validation = baseline_validation + for candidate in FixtureOptimizerBackend(HERE / "fake" / "candidates.json").load_candidates(): + async with PromptSandbox(target, candidate.prompts): + train = await evaluate(HERE / "train.evalset.json", "train") + validation = await evaluate(HERE / "val.evalset.json", "validation") + deltas = [compare_case(base, next(item for item in validation.cases if item.eval_id == base.eval_id), epsilon=config.pipeline.scoring_epsilon, critical_case_ids=set(config.pipeline.gate.critical_case_ids)) for base in baseline_validation.cases] + decision = evaluate_gate(baseline_validation, validation, settings=config.pipeline.gate, case_deltas=deltas, train_score_delta=train.aggregate_score - baseline_train.aggregate_score) + report.candidates.append(CandidateReport(candidate_id=candidate.candidate_id, accepted=decision.accepted, reasons=decision.reasons, train=train, validation=validation, gate=decision, validation_case_deltas=deltas)) accepted = [candidate for candidate in report.candidates if candidate.accepted] report.selected_candidate_id = accepted[0].candidate_id if accepted else None write_reports(report, output_dir) diff --git a/tests/examples/optimization/eval_optimize_loop/test_fake_loop.py b/tests/examples/optimization/eval_optimize_loop/test_fake_loop.py index 8a16f165..886b7dd8 100644 --- a/tests/examples/optimization/eval_optimize_loop/test_fake_loop.py +++ b/tests/examples/optimization/eval_optimize_loop/test_fake_loop.py @@ -9,8 +9,12 @@ from examples.optimization.eval_optimize_loop.run_pipeline import run_fake_pipeline +PROMPT_DIR = Path(__file__).resolve().parents[4] / "examples" / "optimization" / "eval_optimize_loop" / "agent" / "prompts" + + @pytest.mark.asyncio async def test_fake_closed_loop_uses_three_fixture_candidates_and_restores_prompts(tmp_path: Path) -> None: + baseline_prompts = {name: (PROMPT_DIR / name).read_text(encoding="utf-8") for name in ("system.md", "router.md")} report = await run_fake_pipeline(output_dir=tmp_path) assert report.selected_candidate_id == "candidate_general_fix" assert {candidate.candidate_id: candidate.accepted for candidate in report.candidates} == { @@ -20,6 +24,7 @@ async def test_fake_closed_loop_uses_three_fixture_candidates_and_restores_promp } assert (tmp_path / "optimization_report.json").is_file() assert (tmp_path / "optimization_report.md").is_file() + assert {name: (PROMPT_DIR / name).read_text(encoding="utf-8") for name in baseline_prompts} == baseline_prompts @pytest.mark.asyncio From ae3e1f21a7051122a90f83aa962c07c033167c3d Mon Sep 17 00:00:00 2001 From: shiquan_zhan Date: Sat, 11 Jul 2026 03:11:06 +0800 Subject: [PATCH 04/15] feat(eval): validate pipeline config and audit inputs --- .../eval_optimize_loop/pipeline/audit.py | 90 ++++++++ .../eval_optimize_loop/pipeline/config.py | 119 ++++++++++- .../eval_optimize_loop/pipeline/models.py | 8 + .../test_config_and_audit.py | 198 ++++++++++++++++++ 4 files changed, 406 insertions(+), 9 deletions(-) create mode 100644 examples/optimization/eval_optimize_loop/pipeline/audit.py create mode 100644 tests/examples/optimization/eval_optimize_loop/test_config_and_audit.py diff --git a/examples/optimization/eval_optimize_loop/pipeline/audit.py b/examples/optimization/eval_optimize_loop/pipeline/audit.py new file mode 100644 index 00000000..1398d81c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/audit.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import importlib.metadata +import json +import platform +import subprocess +from pathlib import Path +from typing import Literal + +from .config import PipelineConfig, canonical_sha256, sanitize_config +from .models import StrictModel + + +class InputMetadata(StrictModel): + config_digest: str + train_dataset_digest: str + validation_dataset_digest: str + prompt_digest: str + snapshot_path: Path + + +def _write_json(path: Path, value: object) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n", encoding="utf-8") + return path + + +def _canonical_file_digest(path: Path) -> str: + return canonical_sha256(json.loads(path.read_text(encoding="utf-8"))) + + +def _read_prompt_map(target_prompt: object) -> dict[str, str]: + names = target_prompt.names() + prompt_map: dict[str, str] = {} + for name in names: + source = target_prompt.describe_source(name) + prompt_map[name] = Path(source).read_text(encoding="utf-8") if source != "" else source + return prompt_map + + +def write_input_snapshot(config: PipelineConfig, target_prompt: object, output_dir: Path) -> InputMetadata: + prompt_map = _read_prompt_map(target_prompt) + metadata = InputMetadata( + config_digest=canonical_sha256(config.raw), + train_dataset_digest=_canonical_file_digest(config.pipeline.datasets.train_path), + validation_dataset_digest=_canonical_file_digest(config.pipeline.datasets.validation_path), + prompt_digest=canonical_sha256(prompt_map), + snapshot_path=output_dir / "input.snapshot.json", + ) + _write_json( + metadata.snapshot_path, + { + "config": sanitize_config(config.raw), + "digests": metadata.model_dump(mode="json", exclude={"snapshot_path"}), + }, + ) + return metadata + + +def _package_version(name: str) -> str | None: + try: + return importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + return None + + +def _sdk_commit() -> str | None: + try: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], check=True, capture_output=True, text=True, timeout=2 + ) + except (OSError, subprocess.SubprocessError): + return None + return result.stdout.strip() or None + + +def write_environment_snapshot(mode: Literal["fake", "trace", "live"], seed: int, output_dir: Path) -> Path: + from trpc_agent_sdk.version import __version__ + + dependencies = {name: version for name in ("pydantic", "pytest") if (version := _package_version(name)) is not None} + payload = { + "dependencies": dependencies, + "mode": mode, + "platform": platform.platform(), + "python_version": platform.python_version(), + "sdk_commit": _sdk_commit(), + "sdk_version": __version__, + "seed": seed, + } + return _write_json(output_dir / "environment.snapshot.json", payload) diff --git a/examples/optimization/eval_optimize_loop/pipeline/config.py b/examples/optimization/eval_optimize_loop/pipeline/config.py index 80f16745..8176e90b 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/config.py +++ b/examples/optimization/eval_optimize_loop/pipeline/config.py @@ -2,15 +2,23 @@ import hashlib import json +import math +import os from pathlib import Path from typing import Any, Literal -from .models import PipelineSettings, StrictModel +from trpc_agent_sdk.evaluation import OptimizeConfigFile + +from .models import DatasetSettings, PipelineSettings, StrictModel + + +LIVE_REQUIRED_ENV = ("TRPC_AGENT_API_KEY", "TRPC_AGENT_BASE_URL", "TRPC_AGENT_MODEL_NAME") class PipelineConfig(StrictModel): raw: dict[str, Any] pipeline: PipelineSettings + sdk_config: OptimizeConfigFile | None = None config_path: Path @@ -19,21 +27,114 @@ def canonical_sha256(value: object) -> str: return f"sha256:{hashlib.sha256(encoded).hexdigest()}" +def _is_sensitive_key(key: object) -> bool: + normalized = "".join(character for character in str(key).lower() if character.isalnum()) + return ( + normalized in {"authorization", "cookie", "token", "key"} + or normalized.endswith("key") + or any(marker in normalized for marker in ("token", "secret", "password", "credential")) + ) + + def sanitize_config(value: object) -> object: if isinstance(value, dict): - return { - key: "***REDACTED***" if key.lower() in {"api_key", "authorization", "cookie", "token"} else sanitize_config(item) - for key, item in value.items() - } + return {key: "***REDACTED***" if _is_sensitive_key(key) else sanitize_config(item) for key, item in value.items()} if isinstance(value, list): return [sanitize_config(item) for item in value] return value -def load_pipeline_config(config_path: Path, *, mode: Literal["fake"]) -> PipelineConfig: - if mode != "fake": - raise ValueError("Phase 1 and 2 implement only --mode fake") +def _resolved_dataset_settings(pipeline_payload: dict[str, Any], config_path: Path) -> DatasetSettings: + settings = DatasetSettings.model_validate(pipeline_payload.get("datasets", {})) + base_dir = config_path.parent + return DatasetSettings( + train_path=(base_dir / settings.train_path).resolve() if not settings.train_path.is_absolute() else settings.train_path.resolve(), + validation_path=(base_dir / settings.validation_path).resolve() + if not settings.validation_path.is_absolute() + else settings.validation_path.resolve(), + ) + + +def _load_eval_ids(path: Path, *, split: str) -> set[str]: + if not path.is_file(): + raise ValueError(f"{split} dataset does not exist: {path}") + try: + payload = json.loads(path.read_text(encoding="utf-8")) + cases = payload["eval_cases"] + ids = [case["eval_id"] for case in cases] + except (KeyError, TypeError, json.JSONDecodeError) as exc: + raise ValueError(f"{split} dataset is not a valid evalset: {path}") from exc + if len(ids) != len(set(ids)): + raise ValueError(f"{split} dataset has duplicate eval_id values") + return set(ids) + + +def _configured_metric_names(raw: dict[str, Any], sdk_config: OptimizeConfigFile | None) -> set[str]: + if sdk_config is not None: + return {metric.metric_name for metric in sdk_config.evaluate.get_eval_metrics()} + evaluate = raw.get("evaluate", {}) + if not isinstance(evaluate, dict): + return set() + metrics = evaluate.get("metrics") or [] + names = { + metric.get("metric_name") or metric.get("metricName") + for metric in metrics + if isinstance(metric, dict) and (metric.get("metric_name") or metric.get("metricName")) + } + criteria = evaluate.get("criteria") or {} + return names | set(criteria) if isinstance(criteria, dict) else names + + +def _validate_pipeline_settings( + raw: dict[str, Any], pipeline: PipelineSettings, sdk_config: OptimizeConfigFile | None, *, validate_datasets: bool +) -> None: + train_path = pipeline.datasets.train_path + validation_path = pipeline.datasets.validation_path + if validate_datasets: + if train_path == validation_path: + raise ValueError("train and validation dataset paths must be different") + train_ids = _load_eval_ids(train_path, split="train") + validation_ids = _load_eval_ids(validation_path, split="validation") + shared_ids = train_ids & validation_ids + if shared_ids: + raise ValueError(f"train and validation datasets have shared eval_id values: {sorted(shared_ids)}") + missing_critical = set(pipeline.gate.critical_case_ids) - validation_ids + if missing_critical: + raise ValueError(f"critical validation case ids are missing: {sorted(missing_critical)}") + if not all(math.isfinite(weight) for weight in pipeline.metric_weights.values()): + raise ValueError("metric weights must be finite") + if any(weight < 0 for weight in pipeline.metric_weights.values()): + raise ValueError("metric weights must be non-negative") + if not any(weight > 0 for weight in pipeline.metric_weights.values()): + raise ValueError("metric weights must include at least one positive weight") + unknown_floors = set(pipeline.metric_floors) - _configured_metric_names(raw, sdk_config) + if unknown_floors: + raise ValueError(f"metric floors reference unknown metric(s): {sorted(unknown_floors)}") + + +def load_pipeline_config(config_path: Path, *, mode: Literal["fake", "trace", "live"]) -> PipelineConfig: raw = json.loads(config_path.read_text(encoding="utf-8")) if "pipeline" not in raw: raise ValueError("optimizer config requires a pipeline section") - return PipelineConfig(raw=raw, pipeline=PipelineSettings.model_validate(raw["pipeline"]), config_path=config_path) + if mode == "live": + missing = [name for name in LIVE_REQUIRED_ENV if not os.environ.get(name)] + if missing: + raise ValueError("live mode requires environment variables: " + ", ".join(missing)) + if mode not in {"fake", "trace", "live"}: + raise ValueError(f"unsupported mode: {mode}") + if "evaluate" in raw and "optimize" in raw: + sdk_config = OptimizeConfigFile.model_validate({"evaluate": raw["evaluate"], "optimize": raw["optimize"]}) + elif mode == "live": + raise ValueError("live mode requires both evaluate and optimize sections") + else: + sdk_config = None + pipeline_payload = dict(raw["pipeline"]) + pipeline_payload["datasets"] = _resolved_dataset_settings(pipeline_payload, config_path).model_dump() + pipeline = PipelineSettings.model_validate(pipeline_payload) + _validate_pipeline_settings( + raw, + pipeline, + sdk_config, + validate_datasets=mode != "fake" or "datasets" in raw["pipeline"], + ) + return PipelineConfig(raw=raw, pipeline=pipeline, sdk_config=sdk_config, config_path=config_path) diff --git a/examples/optimization/eval_optimize_loop/pipeline/models.py b/examples/optimization/eval_optimize_loop/pipeline/models.py index c9377a35..6cb6f6b4 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/models.py +++ b/examples/optimization/eval_optimize_loop/pipeline/models.py @@ -1,6 +1,7 @@ from __future__ import annotations from enum import Enum +from pathlib import Path from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field @@ -104,11 +105,18 @@ class ReproducibilitySettings(StrictModel): seed: int = 42 +class DatasetSettings(StrictModel): + train_path: Path = Path("train.evalset.json") + validation_path: Path = Path("val.evalset.json") + + class PipelineSettings(StrictModel): + datasets: DatasetSettings = Field(default_factory=DatasetSettings) reproducibility: ReproducibilitySettings = Field(default_factory=ReproducibilitySettings) gate: GateSettings = Field(default_factory=GateSettings) scoring_epsilon: float = 0.000001 metric_weights: dict[str, float] = Field(default_factory=lambda: {"final_response_avg_score": 0.6, "fake_rubric_score": 0.4}) + metric_floors: dict[str, float] = Field(default_factory=dict) class CandidateRecord(StrictModel): diff --git a/tests/examples/optimization/eval_optimize_loop/test_config_and_audit.py b/tests/examples/optimization/eval_optimize_loop/test_config_and_audit.py new file mode 100644 index 00000000..6665a0b8 --- /dev/null +++ b/tests/examples/optimization/eval_optimize_loop/test_config_and_audit.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop.pipeline.audit import ( + write_environment_snapshot, + write_input_snapshot, +) +from examples.optimization.eval_optimize_loop.pipeline.config import load_pipeline_config +from trpc_agent_sdk.evaluation import TargetPrompt + + +def _write_evalset(path: Path, *ids: str) -> None: + path.write_text( + json.dumps({"eval_set_id": path.stem, "eval_cases": [{"eval_id": eval_id} for eval_id in ids]}), + encoding="utf-8", + ) + + +def _full_sdk_sections() -> dict[str, object]: + return { + "evaluate": {"metrics": [{"metric_name": "quality", "threshold": 0.8}]}, + "optimize": { + "algorithm": { + "name": "gepa_reflective", + "reflection_lm": {}, + "max_metric_calls": 1, + } + }, + } + + +def _write_config( + tmp_path: Path, + *, + train_ids: tuple[str, ...] = ("train-1", ), + validation_ids: tuple[str, ...] = ("validation-1", ), + pipeline_overrides: dict[str, object] | None = None, + sdk_sections: bool = False, +) -> Path: + _write_evalset(tmp_path / "train.evalset.json", *train_ids) + _write_evalset(tmp_path / "validation.evalset.json", *validation_ids) + pipeline: dict[str, object] = { + "datasets": { + "train_path": "train.evalset.json", + "validation_path": "validation.evalset.json", + }, + "metric_weights": {"quality": 1.0}, + "metric_floors": {"quality": 0.8}, + } + pipeline.update(pipeline_overrides or {}) + payload: dict[str, object] = {"pipeline": pipeline} + if sdk_sections: + payload.update(_full_sdk_sections()) + else: + payload["evaluate"] = {"metrics": [{"metric_name": "quality", "threshold": 0.8}]} + config_path = tmp_path / "optimizer.json" + config_path.write_text(json.dumps(payload), encoding="utf-8") + return config_path + + +def test_fake_mode_loads_without_live_environment(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + config_path = _write_config(tmp_path) + for name in ("TRPC_AGENT_API_KEY", "TRPC_AGENT_BASE_URL", "TRPC_AGENT_MODEL_NAME"): + monkeypatch.delenv(name, raising=False) + + config = load_pipeline_config(config_path, mode="fake") + + assert config.sdk_config is None + assert config.pipeline.datasets.train_path.name == "train.evalset.json" + + +def test_live_mode_lists_all_missing_named_environment_variables(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + config_path = _write_config(tmp_path, sdk_sections=True) + for name in ("TRPC_AGENT_API_KEY", "TRPC_AGENT_BASE_URL", "TRPC_AGENT_MODEL_NAME"): + monkeypatch.delenv(name, raising=False) + + with pytest.raises(ValueError, match="TRPC_AGENT_API_KEY.*TRPC_AGENT_BASE_URL.*TRPC_AGENT_MODEL_NAME"): + load_pipeline_config(config_path, mode="live") + + +def test_live_mode_validates_default_dataset_paths_when_credentials_are_present( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + for name in ("TRPC_AGENT_API_KEY", "TRPC_AGENT_BASE_URL", "TRPC_AGENT_MODEL_NAME"): + monkeypatch.setenv(name, "configured") + config_path = tmp_path / "optimizer.json" + config_path.write_text( + json.dumps( + { + **_full_sdk_sections(), + "pipeline": {"metric_weights": {"quality": 1.0}, "metric_floors": {"quality": 0.8}}, + } + ), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="train dataset does not exist"): + load_pipeline_config(config_path, mode="live") + + +@pytest.mark.parametrize( + ("pipeline_overrides", "train_ids", "validation_ids", "message"), + [ + ({"datasets": {"train_path": "missing.json", "validation_path": "validation.evalset.json"}}, ("train-1", ), ("validation-1", ), "train dataset"), + ({"datasets": {"train_path": "train.evalset.json", "validation_path": "train.evalset.json"}}, ("train-1", ), ("validation-1", ), "different"), + ({}, ("duplicate", "duplicate"), ("validation-1", ), "duplicate"), + ({}, ("shared", ), ("shared", ), "shared"), + ({"gate": {"critical_case_ids": ["missing-critical"]}}, ("train-1", ), ("validation-1", ), "critical"), + ], +) +def test_dataset_validation_rejects_invalid_inputs( + tmp_path: Path, + pipeline_overrides: dict[str, object], + train_ids: tuple[str, ...], + validation_ids: tuple[str, ...], + message: str, +) -> None: + config_path = _write_config( + tmp_path, + pipeline_overrides=pipeline_overrides, + train_ids=train_ids, + validation_ids=validation_ids, + ) + + with pytest.raises(ValueError, match=message): + load_pipeline_config(config_path, mode="fake") + + +@pytest.mark.parametrize( + ("pipeline_overrides", "message"), + [ + ({"metric_weights": {"quality": -0.1}}, "non-negative"), + ({"metric_weights": {"quality": 0.0}}, "positive"), + ({"metric_floors": {"unknown": 0.5}}, "unknown metric"), + ], +) +def test_metric_validation_rejects_invalid_weights_and_floors( + tmp_path: Path, pipeline_overrides: dict[str, object], message: str +) -> None: + config_path = _write_config(tmp_path, pipeline_overrides=pipeline_overrides) + + with pytest.raises(ValueError, match=message): + load_pipeline_config(config_path, mode="trace") + + +def test_metric_validation_rejects_nan_weight_from_json(tmp_path: Path) -> None: + config_path = _write_config(tmp_path, pipeline_overrides={"metric_weights": {"quality": float("nan")}}) + + assert "NaN" in config_path.read_text(encoding="utf-8") + with pytest.raises(ValueError, match="finite"): + load_pipeline_config(config_path, mode="fake") + + +def test_input_snapshot_has_stable_digests_and_redacts_secrets(tmp_path: Path) -> None: + config_path = _write_config(tmp_path) + raw = json.loads(config_path.read_text(encoding="utf-8")) + raw["evaluate"]["api_key"] = "literal-secret" + raw["evaluate"]["nested_token"] = "other-secret" + raw["evaluate"].update( + { + "access_key": "access-secret", + "client_secret": "client-secret", + "credential": "credential-secret", + "password": "password-secret", + } + ) + config_path.write_text(json.dumps(raw), encoding="utf-8") + prompt_path = tmp_path / "system.md" + prompt_path.write_text("Support users safely.", encoding="utf-8") + target_prompt = TargetPrompt().add_path("system", str(prompt_path)) + config = load_pipeline_config(config_path, mode="fake") + + first = write_input_snapshot(config, target_prompt, tmp_path / "first") + second = write_input_snapshot(config, target_prompt, tmp_path / "second") + + assert first.config_digest == second.config_digest + assert first.train_dataset_digest == second.train_dataset_digest + assert first.validation_dataset_digest == second.validation_dataset_digest + assert first.prompt_digest == second.prompt_digest + snapshot = (tmp_path / "first" / "input.snapshot.json").read_text(encoding="utf-8") + for secret in ("literal-secret", "other-secret", "access-secret", "client-secret", "credential-secret", "password-secret"): + assert secret not in snapshot + assert "***REDACTED***" in snapshot + + +def test_environment_snapshot_excludes_arbitrary_environment_variables(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("ARBITRARY_SECRET", "must-not-appear") + + snapshot_path = write_environment_snapshot("trace", 42, tmp_path) + + snapshot = snapshot_path.read_text(encoding="utf-8") + assert "ARBITRARY_SECRET" not in snapshot + assert "must-not-appear" not in snapshot + assert json.loads(snapshot)["mode"] == "trace" From 829a07c90fb3606464f0298680f8512394a6cb0a Mon Sep 17 00:00:00 2001 From: shiquan_zhan Date: Sat, 11 Jul 2026 03:31:57 +0800 Subject: [PATCH 05/15] feat(eval): add deterministic fake rubric metric --- .../eval_optimize_loop/fake/fake_judge.py | 138 +++++++- .../eval_optimize_loop/optimizer.json | 5 +- .../eval_optimize_loop/pipeline/comparator.py | 2 + .../eval_optimize_loop/pipeline/evaluator.py | 2 + .../eval_optimize_loop/pipeline/models.py | 3 + .../pipeline/normalization.py | 91 ++++- .../test_normalization_and_rubric.py | 319 ++++++++++++++++++ 7 files changed, 543 insertions(+), 17 deletions(-) create mode 100644 tests/examples/optimization/eval_optimize_loop/test_normalization_and_rubric.py diff --git a/examples/optimization/eval_optimize_loop/fake/fake_judge.py b/examples/optimization/eval_optimize_loop/fake/fake_judge.py index c143f410..06e82dc4 100644 --- a/examples/optimization/eval_optimize_loop/fake/fake_judge.py +++ b/examples/optimization/eval_optimize_loop/fake/fake_judge.py @@ -1,14 +1,138 @@ from __future__ import annotations import json +import statistics +from dataclasses import dataclass +from typing import Optional +from trpc_agent_sdk.evaluation._eval_case import Invocation +from trpc_agent_sdk.evaluation._eval_metrics import EvalMetric, EvalStatus +from trpc_agent_sdk.evaluation._eval_result import EvaluationResult, PerInvocationResult +from trpc_agent_sdk.evaluation._evaluator_base import Evaluator +from trpc_agent_sdk.evaluation._evaluator_registry import EVALUATOR_REGISTRY -def fake_rubric_score(response: str) -> float: - """A deterministic structural rubric used only in fake mode.""" + +@dataclass(frozen=True) +class FakeRubricResult: + score: float + reason: str + + +def _response_text(invocation: Invocation) -> str: + response = invocation.final_response + if response is None or not response.parts: + return "" + return "".join(part.text or "" for part in response.parts) + + +def _parse_response(response: str) -> dict[str, object] | None: try: payload = json.loads(response) - except json.JSONDecodeError: - return 0.0 - if not all(key in payload for key in ("route", "tool", "arguments", "answer")): - return 0.25 - return 1.0 + except (TypeError, json.JSONDecodeError): + return None + return payload if isinstance(payload, dict) else None + + +def evaluate_fake_response(response: str, expected_response: str) -> FakeRubricResult: + """Score the local fake-response rubric without a model or external state.""" + actual = _parse_response(response) + if actual is None: + return FakeRubricResult(0.0, "invalid JSON response") + + expected = _parse_response(expected_response) + if expected is None: + raise ValueError("fake rubric expected response must be a JSON object") + + score = 0.25 + reasons: list[str] = [] + required_fields = ("route", "tool", "arguments", "answer") + missing_fields = [field for field in required_fields if field not in actual] + if missing_fields: + return FakeRubricResult(0.25, "missing required fields: " + ", ".join(missing_fields)) + + if actual.get("route") == expected.get("route") and actual.get("tool") == expected.get("tool"): + score += 0.25 + else: + reasons.append("route or tool does not match expected response") + + expected_arguments = expected.get("arguments") + actual_arguments = actual.get("arguments") + if isinstance(expected_arguments, dict) and isinstance(actual_arguments, dict) and all( + actual_arguments.get(name) == value for name, value in expected_arguments.items() + ): + score += 0.25 + else: + reasons.append("required tool arguments do not match expected response") + + unknown_knowledge = expected.get("route") == "knowledge_gap" and expected.get("tool") == "none" + answer = actual.get("answer") + explicit_refusal = isinstance(answer, str) and any( + phrase in answer.lower() for phrase in ("不能猜测", "cannot guess", "do not guess", "don't guess") + ) + if not unknown_knowledge or explicit_refusal: + score += 0.25 + else: + reasons.append("unknown knowledge must explicitly refuse to guess") + + return FakeRubricResult(score, "; ".join(reasons) if reasons else "all fake rubric checks passed") + + +def score_fake_response(response: str, expected_response: str) -> float: + """Return the deterministic fake-rubric score for a response pair.""" + return evaluate_fake_response(response, expected_response).score + + +def fake_rubric_score(response: str) -> float: + """Backward-compatible structural score for callers without a reference response.""" + payload = _parse_response(response) + return 1.0 if payload and all(key in payload for key in ("route", "tool", "arguments", "answer")) else 0.0 + + +class FakeRubricEvaluator(Evaluator): + """A local deterministic evaluator registered only by the fake example.""" + + requires_reference = True + + def __init__(self, threshold: Optional[float] = None, eval_metric: Optional[EvalMetric] = None) -> None: + if threshold is not None and eval_metric is not None: + raise ValueError("Either eval_metric or threshold may be specified, not both") + self._threshold = eval_metric.threshold if eval_metric is not None else threshold + if self._threshold is None: + self._threshold = 0.75 + + def evaluate_invocations( + self, + actual_invocations: list[Invocation], + expected_invocations: Optional[list[Invocation]], + ) -> EvaluationResult: + if expected_invocations is None: + raise ValueError("expected_invocations is required for fake_rubric_score") + if len(actual_invocations) != len(expected_invocations): + raise ValueError("actual and expected invocations must contain the same number of invocations") + + per_invocation_results: list[PerInvocationResult] = [] + for actual, expected in zip(actual_invocations, expected_invocations): + result = evaluate_fake_response(_response_text(actual), _response_text(expected)) + per_invocation_results.append( + PerInvocationResult( + actual_invocation=actual, + expected_invocation=expected, + score=result.score, + eval_status=EvalStatus.PASSED if result.score >= self._threshold else EvalStatus.FAILED, + reason=result.reason, + ) + ) + if not per_invocation_results: + return EvaluationResult() + + overall_score = statistics.mean(result.score for result in per_invocation_results) + return EvaluationResult( + overall_score=overall_score, + overall_eval_status=EvalStatus.PASSED if overall_score >= self._threshold else EvalStatus.FAILED, + per_invocation_results=per_invocation_results, + ) + + +def register_fake_rubric_evaluator() -> None: + """Register the example-only evaluator with the SDK's existing registry.""" + EVALUATOR_REGISTRY.register("fake_rubric_score", FakeRubricEvaluator) diff --git a/examples/optimization/eval_optimize_loop/optimizer.json b/examples/optimization/eval_optimize_loop/optimizer.json index e1f8d24c..864d8ff1 100644 --- a/examples/optimization/eval_optimize_loop/optimizer.json +++ b/examples/optimization/eval_optimize_loop/optimizer.json @@ -1,6 +1,9 @@ { "evaluate": { - "metrics": [{"metric_name": "final_response_avg_score", "threshold": 1.0, "criterion": {"final_response": {"text": {"match": "exact"}}}}], + "metrics": [ + {"metric_name": "final_response_avg_score", "threshold": 1.0, "criterion": {"final_response": {"text": {"match": "exact"}}}}, + {"metric_name": "fake_rubric_score", "threshold": 0.75} + ], "num_runs": 1 }, "pipeline": { diff --git a/examples/optimization/eval_optimize_loop/pipeline/comparator.py b/examples/optimization/eval_optimize_loop/pipeline/comparator.py index e479f323..cd9a8a38 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/comparator.py +++ b/examples/optimization/eval_optimize_loop/pipeline/comparator.py @@ -35,4 +35,6 @@ def compare_case( metric_deltas={name: candidate.metric_scores.get(name, 0.0) - baseline.metric_scores.get(name, 0.0) for name in metric_names}, critical=critical, hard_fail_added=(not baseline.hard_failed and candidate.hard_failed) or (critical and transition == "REGRESSION"), + new_failure_types=sorted(set(candidate.failure_types) - set(baseline.failure_types), key=lambda item: item.value), + resolved_failure_types=sorted(set(baseline.failure_types) - set(candidate.failure_types), key=lambda item: item.value), ) diff --git a/examples/optimization/eval_optimize_loop/pipeline/evaluator.py b/examples/optimization/eval_optimize_loop/pipeline/evaluator.py index d583e396..0e6cae1e 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/evaluator.py +++ b/examples/optimization/eval_optimize_loop/pipeline/evaluator.py @@ -8,6 +8,7 @@ from trpc_agent_sdk.evaluation._eval_config import EvalConfig from trpc_agent_sdk.evaluation._eval_set import EvalSet +from ..fake.fake_judge import register_fake_rubric_evaluator from .models import SplitReport from .normalization import normalize_eval_results @@ -15,6 +16,7 @@ async def evaluate_split( eval_set_path: Path, *, call_agent: Callable[[str], Awaitable[str]], split: Literal["train", "validation"], metric_weights: dict[str, float] ) -> SplitReport: + register_fake_rubric_evaluator() eval_set = EvalSet.model_validate_json(eval_set_path.read_text(encoding="utf-8")) config = EvalConfig.model_validate(json.loads((eval_set_path.parent / "optimizer.json").read_text(encoding="utf-8"))["evaluate"]) _, _, _, results_by_eval_id = await AgentEvaluator.evaluate_eval_set( diff --git a/examples/optimization/eval_optimize_loop/pipeline/models.py b/examples/optimization/eval_optimize_loop/pipeline/models.py index 6cb6f6b4..ebdb630d 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/models.py +++ b/examples/optimization/eval_optimize_loop/pipeline/models.py @@ -40,6 +40,7 @@ class CaseSnapshot(StrictModel): trace_digest: str metric_reasons: dict[str, list[str]] = Field(default_factory=dict) failure_reasons: list[str] = Field(default_factory=list) + failure_types: list[FailureType] = Field(default_factory=list) final_response: str | None = None expected_response: str | None = None tool_calls: list[ToolCallSnapshot] = Field(default_factory=list) @@ -57,6 +58,8 @@ class CaseDelta(StrictModel): metric_deltas: dict[str, float] critical: bool hard_fail_added: bool + new_failure_types: list[FailureType] = Field(default_factory=list) + resolved_failure_types: list[FailureType] = Field(default_factory=list) class SplitReport(StrictModel): diff --git a/examples/optimization/eval_optimize_loop/pipeline/normalization.py b/examples/optimization/eval_optimize_loop/pipeline/normalization.py index 43e00e0c..547b500e 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/normalization.py +++ b/examples/optimization/eval_optimize_loop/pipeline/normalization.py @@ -8,7 +8,7 @@ from trpc_agent_sdk.evaluation._eval_metrics import EvalStatus from trpc_agent_sdk.evaluation._eval_result import EvalCaseResult -from .models import CaseSnapshot, ToolCallSnapshot +from .models import CaseSnapshot, FailureType, ToolCallSnapshot @dataclass(frozen=True) @@ -17,19 +17,20 @@ class FakeResponseSnapshot: payload: dict[str, object] | None tool_calls: list[ToolCallSnapshot] + failure_reason: str | None = None def parse_fake_response(response: str | None) -> FakeResponseSnapshot: """Parse fake JSON without allowing malformed output to break reporting.""" if not response: - return FakeResponseSnapshot(payload=None, tool_calls=[]) + return FakeResponseSnapshot(payload=None, tool_calls=[], failure_reason="missing fake JSON response") try: payload = json.loads(response) except (TypeError, json.JSONDecodeError): - return FakeResponseSnapshot(payload=None, tool_calls=[]) + return FakeResponseSnapshot(payload=None, tool_calls=[], failure_reason="invalid JSON response") if not isinstance(payload, dict): - return FakeResponseSnapshot(payload=None, tool_calls=[]) + return FakeResponseSnapshot(payload=None, tool_calls=[], failure_reason="fake JSON response must be an object") tool = payload.get("tool") arguments = payload.get("arguments") if isinstance(tool, str) and tool not in {"", "none"}: @@ -52,15 +53,30 @@ def normalize_eval_results( snapshots: dict[str, CaseSnapshot] = {} for eval_id, runs in results_by_eval_id.items(): if not runs: - raise ValueError(f"no evaluator results for {eval_id}") + raise ValueError(f"evaluation error for {eval_id}: no evaluator results") metric_scores: dict[str, list[float]] = {} metric_thresholds: dict[str, float] = {} metric_passed: dict[str, bool] = {} metric_reasons: dict[str, list[str]] = {} - for result in runs: + expected_metric_names: set[str] | None = None + for run_index, result in enumerate(runs, start=1): + current_metric_names = {metric.metric_name for metric in result.overall_eval_metric_results} + if expected_metric_names is None: + expected_metric_names = current_metric_names + elif current_metric_names != expected_metric_names: + missing = expected_metric_names - current_metric_names + unexpected = current_metric_names - expected_metric_names + raise ValueError( + f"evaluation error for {eval_id}: run {run_index} has inconsistent metrics; " + f"missing={sorted(missing)}, unexpected={sorted(unexpected)}" + ) + if not current_metric_names: + raise ValueError(f"evaluation error for {eval_id}: run {run_index} has no metric results") for metric in result.overall_eval_metric_results: - if metric.score is None: - raise ValueError(f"metric {metric.metric_name} was not evaluated for {eval_id}") + if metric.eval_status == EvalStatus.NOT_EVALUATED or metric.score is None: + raise ValueError( + f"evaluation error for {eval_id}: metric {metric.metric_name} in run {run_index} is NOT_EVALUATED or missing a score" + ) metric_scores.setdefault(metric.metric_name, []).append(metric.score) metric_thresholds[metric.metric_name] = metric.threshold current_passed = metric.eval_status == EvalStatus.PASSED @@ -76,19 +92,76 @@ def normalize_eval_results( expected = _text(invocation.expected_invocation.final_response) if invocation and invocation.expected_invocation else None parsed_actual = parse_fake_response(actual) parsed_expected = parse_fake_response(expected) + parsed_run_responses = [ + ( + parse_fake_response(_text(run_invocation.actual_invocation.final_response)), + parse_fake_response(_text(run_invocation.expected_invocation.final_response)) + if run_invocation.expected_invocation + else FakeResponseSnapshot(payload=None, tool_calls=[], failure_reason="missing expected response"), + ) + for run in runs + for run_invocation in run.eval_metric_result_per_invocation + ] failure_reasons = [ f"metric {name} did not meet threshold {metric_thresholds[name]}" for name, passed in metric_passed.items() if not passed ] + for name, passed in metric_passed.items(): + if not passed: + failure_reasons.extend(metric_reasons.get(name, [])) + for parsed, response_name in [ + (parsed, response_name) + for actual_parsed, expected_parsed in parsed_run_responses + for parsed, response_name in ((actual_parsed, "actual"), (expected_parsed, "expected")) + ]: + if parsed.failure_reason: + failure_reasons.append(f"{response_name} response: {parsed.failure_reason}") + failure_types = _failure_types( + runs, + metric_passed, + metric_reasons, + [actual_parsed for actual_parsed, _ in parsed_run_responses], + [expected_parsed for _, expected_parsed in parsed_run_responses], + ) snapshots[eval_id] = CaseSnapshot( eval_id=eval_id, split=split, run_count=len(runs), passed=all(item.final_eval_status == EvalStatus.PASSED for item in runs), hard_failed=any(item.error_message for item in runs), aggregate_score=weighted / total_weight if total_weight else 0.0, metric_scores=averaged, metric_thresholds=metric_thresholds, metric_passed=metric_passed, - metric_reasons=metric_reasons, failure_reasons=failure_reasons, + metric_reasons=metric_reasons, failure_reasons=failure_reasons, failure_types=failure_types, final_response=actual, expected_response=expected, trace_digest="sha256:" + hashlib.sha256((actual or "").encode("utf-8")).hexdigest(), tool_calls=parsed_actual.tool_calls, expected_tool_calls=parsed_expected.tool_calls, ) return snapshots + + +def _failure_types( + runs: list[EvalCaseResult], + metric_passed: dict[str, bool], + metric_reasons: dict[str, list[str]], + actual: list[FakeResponseSnapshot], + expected: list[FakeResponseSnapshot], +) -> list[FailureType]: + failure_types: set[FailureType] = set() + if any(run.error_message for run in runs): + failure_types.add(FailureType.EXECUTION_ERROR) + if any(item.failure_reason for item in actual + expected): + failure_types.add(FailureType.FORMAT_VIOLATION) + for metric_name, passed in metric_passed.items(): + if passed: + continue + reasons = " ".join(metric_reasons.get(metric_name, [])).lower() + if "route or tool" in reasons: + failure_types.add(FailureType.TOOL_SELECTION_ERROR) + if "tool arguments" in reasons: + failure_types.add(FailureType.TOOL_ARGUMENT_ERROR) + if "refuse to guess" in reasons: + failure_types.add(FailureType.KNOWLEDGE_RECALL_INSUFFICIENT) + if metric_name == "final_response_avg_score": + failure_types.add(FailureType.FINAL_RESPONSE_MISMATCH) + if metric_name == "fake_rubric_score": + if "invalid json" in reasons or "missing required fields" in reasons: + failure_types.add(FailureType.FORMAT_VIOLATION) + return sorted(failure_types, key=lambda item: item.value) diff --git a/tests/examples/optimization/eval_optimize_loop/test_normalization_and_rubric.py b/tests/examples/optimization/eval_optimize_loop/test_normalization_and_rubric.py new file mode 100644 index 00000000..46fc2fcc --- /dev/null +++ b/tests/examples/optimization/eval_optimize_loop/test_normalization_and_rubric.py @@ -0,0 +1,319 @@ +from __future__ import annotations + +import json + +import pytest + +from examples.optimization.eval_optimize_loop.fake.fake_judge import ( + FakeRubricEvaluator, + register_fake_rubric_evaluator, + score_fake_response, +) +from examples.optimization.eval_optimize_loop.pipeline.comparator import compare_case +from examples.optimization.eval_optimize_loop.pipeline.models import CaseSnapshot, FailureType +from examples.optimization.eval_optimize_loop.pipeline.normalization import normalize_eval_results, parse_fake_response +from trpc_agent_sdk.evaluation import AgentEvaluator +from trpc_agent_sdk.evaluation._eval_case import Invocation +from trpc_agent_sdk.evaluation._eval_config import EvalConfig +from trpc_agent_sdk.evaluation._eval_metrics import EvalStatus +from trpc_agent_sdk.evaluation._eval_result import EvalCaseResult, EvalMetricResult, EvalMetricResultPerInvocation +from trpc_agent_sdk.evaluation._eval_set import EvalSet +from trpc_agent_sdk.types import Content, Part + + +EXPECTED_ORDER = json.dumps( + { + "route": "order_lookup", + "tool": "lookup_order", + "arguments": {"order_id": "A100"}, + "answer": "正在查询订单 A100。", + }, + ensure_ascii=False, +) + + +def _invocation(response: str) -> Invocation: + return Invocation( + user_content=Content(role="user", parts=[Part(text="查询订单 A100")]), + final_response=Content(role="model", parts=[Part(text=response)]), + ) + + +def _case_result(*, score: float, status: EvalStatus, response: str = EXPECTED_ORDER) -> EvalCaseResult: + actual = _invocation(response) + expected = _invocation(EXPECTED_ORDER) + metric = EvalMetricResult( + metric_name="fake_rubric_score", + threshold=0.75, + score=score, + eval_status=status, + ) + return EvalCaseResult( + eval_set_id="fake", + eval_id="case", + final_eval_status=status, + overall_eval_metric_results=[metric], + eval_metric_result_per_invocation=[ + EvalMetricResultPerInvocation( + actual_invocation=actual, + expected_invocation=expected, + eval_metric_results=[metric], + ) + ], + session_id="test", + ) + + +def _snapshot(*, failure_types: list[FailureType]) -> CaseSnapshot: + return CaseSnapshot( + eval_id="case", + split="validation", + run_count=1, + passed=not failure_types, + hard_failed=False, + aggregate_score=1.0 if not failure_types else 0.0, + metric_scores={"fake_rubric_score": 1.0 if not failure_types else 0.0}, + metric_thresholds={"fake_rubric_score": 0.75}, + metric_passed={"fake_rubric_score": not failure_types}, + trace_digest="sha256:test", + failure_types=failure_types, + ) + + +def test_fake_rubric_scores_invalid_json_as_zero_with_parse_reason() -> None: + evaluator = FakeRubricEvaluator(threshold=0.75) + result = evaluator.evaluate_invocations([_invocation("not json")], [_invocation(EXPECTED_ORDER)]) + + assert score_fake_response("not json", EXPECTED_ORDER) == 0.0 + assert result.overall_score == 0.0 + assert result.per_invocation_results[0].reason == "invalid JSON response" + + +def test_fake_rubric_rejects_mismatched_invocation_lengths() -> None: + evaluator = FakeRubricEvaluator(threshold=0.75) + + with pytest.raises(ValueError, match="same number of invocations"): + evaluator.evaluate_invocations( + [_invocation(EXPECTED_ORDER), _invocation(EXPECTED_ORDER)], + [_invocation(EXPECTED_ORDER)], + ) + + +def test_fake_rubric_scores_exact_response_and_required_argument_deterministically() -> None: + missing_argument = json.dumps( + { + "route": "order_lookup", + "tool": "lookup_order", + "arguments": {}, + "answer": "正在查询订单。", + }, + ensure_ascii=False, + ) + + assert score_fake_response(EXPECTED_ORDER, EXPECTED_ORDER) == 1.0 + assert score_fake_response(missing_argument, EXPECTED_ORDER) == 0.75 + assert score_fake_response('{"route":"order_lookup","tool":"lookup_order","arguments":{"order_id":"A100"}}', EXPECTED_ORDER) == 0.25 + + +@pytest.mark.asyncio +async def test_registry_evaluates_fake_evalset_with_both_metrics_without_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("TRPC_AGENT_API_KEY", raising=False) + register_fake_rubric_evaluator() + eval_set = EvalSet.model_validate( + { + "eval_set_id": "fake-rubric", + "eval_cases": [ + { + "eval_id": "case", + "conversation": [ + { + "user_content": {"role": "user", "parts": [{"text": "查询订单 A100"}]}, + "final_response": {"role": "model", "parts": [{"text": EXPECTED_ORDER}]}, + } + ], + } + ], + } + ) + config = EvalConfig.model_validate( + { + "metrics": [ + {"metric_name": "final_response_avg_score", "threshold": 1.0}, + {"metric_name": "fake_rubric_score", "threshold": 0.75}, + ] + } + ) + + async def call_agent(_query: str) -> str: + return EXPECTED_ORDER + + _, _, _, results = await AgentEvaluator.evaluate_eval_set( + eval_set, + call_agent=call_agent, + eval_config=config, + num_runs=1, + print_detailed_results=False, + ) + + assert {metric.metric_name for metric in results["case"][0].overall_eval_metric_results} == { + "final_response_avg_score", + "fake_rubric_score", + } + assert all(metric.score == 1.0 for metric in results["case"][0].overall_eval_metric_results) + + +def test_normalization_averages_all_runs_but_requires_every_run_to_pass() -> None: + snapshots = normalize_eval_results( + { + "case": [ + _case_result(score=1.0, status=EvalStatus.PASSED), + _case_result(score=0.0, status=EvalStatus.FAILED, response="not json"), + ] + }, + split="validation", + metric_weights={"fake_rubric_score": 1.0}, + ) + + case = snapshots["case"] + assert case.run_count == 2 + assert case.metric_scores["fake_rubric_score"] == 0.5 + assert case.metric_passed["fake_rubric_score"] is False + assert case.passed is False + assert any("invalid JSON" in reason for reason in case.failure_reasons) + assert FailureType.FORMAT_VIOLATION in case.failure_types + + +def test_normalization_rejects_metrics_missing_from_an_earlier_run() -> None: + first_run = _case_result(score=1.0, status=EvalStatus.PASSED) + second_run = _case_result(score=1.0, status=EvalStatus.PASSED) + extra_metric = EvalMetricResult( + metric_name="final_response_avg_score", + threshold=1.0, + score=1.0, + eval_status=EvalStatus.PASSED, + ) + second_run.overall_eval_metric_results.append(extra_metric) + second_run.eval_metric_result_per_invocation[0].eval_metric_results.append(extra_metric) + + with pytest.raises(ValueError, match="inconsistent metrics"): + normalize_eval_results( + {"case": [first_run, second_run]}, + split="validation", + metric_weights={"fake_rubric_score": 1.0}, + ) + + +def test_fake_json_extraction_and_failure_type_deltas() -> None: + valid = parse_fake_response(EXPECTED_ORDER) + invalid = parse_fake_response("not json") + + assert valid.tool_calls[0].arguments == {"order_id": "A100"} + assert invalid.tool_calls == [] + assert invalid.failure_reason == "invalid JSON response" + + delta = compare_case( + _snapshot(failure_types=[FailureType.FORMAT_VIOLATION, FailureType.TOOL_ARGUMENT_ERROR]), + _snapshot(failure_types=[FailureType.TOOL_SELECTION_ERROR]), + epsilon=1e-6, + critical_case_ids=set(), + ) + assert delta.new_failure_types == [FailureType.TOOL_SELECTION_ERROR] + assert delta.resolved_failure_types == [FailureType.FORMAT_VIOLATION, FailureType.TOOL_ARGUMENT_ERROR] + + +@pytest.mark.asyncio +async def test_real_evaluator_normalization_records_later_malformed_invocations_and_delta_types() -> None: + register_fake_rubric_evaluator() + eval_set = EvalSet.model_validate( + { + "eval_set_id": "fake-rubric-integration", + "eval_cases": [ + { + "eval_id": "case", + "conversation": [ + { + "user_content": {"role": "user", "parts": [{"text": "first"}]}, + "final_response": {"role": "model", "parts": [{"text": EXPECTED_ORDER}]}, + }, + { + "user_content": {"role": "user", "parts": [{"text": "second"}]}, + "final_response": {"role": "model", "parts": [{"text": EXPECTED_ORDER}]}, + }, + ], + } + ], + } + ) + config = EvalConfig.model_validate({"metrics": [{"metric_name": "fake_rubric_score", "threshold": 0.75}]}) + + async def baseline_agent(_query: str) -> str: + return EXPECTED_ORDER + + async def malformed_agent(query: str) -> str: + return "not json" if query == "second" else EXPECTED_ORDER + + _, _, _, baseline_results = await AgentEvaluator.evaluate_eval_set( + eval_set, call_agent=baseline_agent, eval_config=config, num_runs=1, print_detailed_results=False + ) + _, _, _, malformed_results = await AgentEvaluator.evaluate_eval_set( + eval_set, call_agent=malformed_agent, eval_config=config, num_runs=1, print_detailed_results=False + ) + + baseline = normalize_eval_results( + baseline_results, split="validation", metric_weights={"fake_rubric_score": 1.0} + )["case"] + malformed = normalize_eval_results( + malformed_results, split="validation", metric_weights={"fake_rubric_score": 1.0} + )["case"] + delta = compare_case(baseline, malformed, epsilon=1e-6, critical_case_ids=set()) + + assert "actual response: invalid JSON response" in malformed.failure_reasons + assert FailureType.FORMAT_VIOLATION in malformed.failure_types + assert delta.new_failure_types == [FailureType.FORMAT_VIOLATION] + + +@pytest.mark.asyncio +async def test_real_evaluator_missing_required_fields_produce_delta_failure_type() -> None: + register_fake_rubric_evaluator() + eval_set = EvalSet.model_validate( + { + "eval_set_id": "fake-rubric-schema", + "eval_cases": [ + { + "eval_id": "case", + "conversation": [ + { + "user_content": {"role": "user", "parts": [{"text": "schema"}]}, + "final_response": {"role": "model", "parts": [{"text": EXPECTED_ORDER}]}, + } + ], + } + ], + } + ) + config = EvalConfig.model_validate({"metrics": [{"metric_name": "fake_rubric_score", "threshold": 0.75}]}) + missing_answer = '{"route":"order_lookup","tool":"lookup_order","arguments":{"order_id":"A100"}}' + + async def baseline_agent(_query: str) -> str: + return EXPECTED_ORDER + + async def missing_field_agent(_query: str) -> str: + return missing_answer + + _, _, _, baseline_results = await AgentEvaluator.evaluate_eval_set( + eval_set, call_agent=baseline_agent, eval_config=config, num_runs=1, print_detailed_results=False + ) + _, _, _, missing_field_results = await AgentEvaluator.evaluate_eval_set( + eval_set, call_agent=missing_field_agent, eval_config=config, num_runs=1, print_detailed_results=False + ) + + baseline = normalize_eval_results( + baseline_results, split="validation", metric_weights={"fake_rubric_score": 1.0} + )["case"] + missing_field = normalize_eval_results( + missing_field_results, split="validation", metric_weights={"fake_rubric_score": 1.0} + )["case"] + delta = compare_case(baseline, missing_field, epsilon=1e-6, critical_case_ids=set()) + + assert FailureType.FORMAT_VIOLATION in missing_field.failure_types + assert delta.new_failure_types == [FailureType.FORMAT_VIOLATION] From db4437c775abde03e5e8b7fc6efc94786de629a0 Mon Sep 17 00:00:00 2001 From: shiquan_zhan Date: Sat, 11 Jul 2026 12:56:32 +0800 Subject: [PATCH 06/15] feat(eval): add offline trace attribution --- .superpowers/sdd/task-3-report.md | 51 ++++++ .../pipeline/attribution.py | 94 +++++++++++ .../eval_optimize_loop/pipeline/models.py | 18 +- .../pipeline/normalization.py | 10 +- .../eval_optimize_loop/pipeline/reporter.py | 16 +- .../eval_optimize_loop/run_pipeline.py | 65 +++++++- .../trace/trace.evalset.json | 75 +++++++++ .../trace/trace_config.json | 27 +++ .../test_attribution_and_trace.py | 157 ++++++++++++++++++ 9 files changed, 504 insertions(+), 9 deletions(-) create mode 100644 .superpowers/sdd/task-3-report.md create mode 100644 examples/optimization/eval_optimize_loop/pipeline/attribution.py create mode 100644 examples/optimization/eval_optimize_loop/trace/trace.evalset.json create mode 100644 examples/optimization/eval_optimize_loop/trace/trace_config.json create mode 100644 tests/examples/optimization/eval_optimize_loop/test_attribution_and_trace.py diff --git a/.superpowers/sdd/task-3-report.md b/.superpowers/sdd/task-3-report.md new file mode 100644 index 00000000..4d06d0c0 --- /dev/null +++ b/.superpowers/sdd/task-3-report.md @@ -0,0 +1,51 @@ +# Task 3 Report: Rule-first attribution and offline trace mode + +## Scope + +- Added deterministic `attribute_case()` classification in the example pipeline. +- Added an offline four-case trace EvalSet and trace-only CLI mode. +- Extended only example contracts and reporting to persist attribution and trace artifacts. +- No SDK public API was edited. Trace mode uses no credentials, agent inference, optimizer, network judge, or model object. + +## TDD evidence + +### RED + +Command: + +```text +python -m pytest tests/examples/optimization/eval_optimize_loop/test_attribution_and_trace.py -q +``` + +Result before implementation: `15 failed` (exit code 1). The failures were the intended missing-feature signals: `pipeline.attribution` did not exist, `run_trace_pipeline` was not exported, and the CLI rejected `--mode trace`. A subsequent added boundary test also failed as expected before the implementation treated every recorded non-timeout execution error as `EXECUTION_ERROR`. + +### GREEN + +The focused Task 3 suite passed after implementation: + +```text +16 passed +``` + +It covers all rule precedence categories, judge fallback/invalid output, structural-rule precedence over judge output, trace evaluation without `TRPC_AGENT_API_KEY`, generated report/artifact files, and the trace CLI. + +## Final verification + +```text +python -m pytest tests/examples/optimization/eval_optimize_loop/test_attribution_and_trace.py -q +# 16 passed + +python -m pytest tests/examples/optimization/eval_optimize_loop -q +# 49 passed + +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode trace --output-dir +# exit 0; wrote optimization_report.json, optimization_report.md, +# trace_raw_results.json, and trace_normalized_cases.json +``` + +`git diff --check` completed without whitespace errors. + +## Notes + +- The invalid-format trace uses a reference with `tool: none`, so no higher-priority tool-selection mismatch exists; it therefore correctly reaches `FORMAT_VIOLATION` under the required precedence. +- Test and CLI runs emit pre-existing third-party dependency warnings from `requests` and `langgraph`; they do not affect exit status. diff --git a/examples/optimization/eval_optimize_loop/pipeline/attribution.py b/examples/optimization/eval_optimize_loop/pipeline/attribution.py new file mode 100644 index 00000000..c7c213d3 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/attribution.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +from typing import Any, Callable + +from .models import CaseSnapshot, FailureAttribution, FailureType + + +def _rule(case: CaseSnapshot, failure_type: FailureType, evidence: str) -> FailureAttribution: + return FailureAttribution( + eval_id=case.eval_id, + primary_type=failure_type, + confidence=1.0, + evidence=[evidence], + source="rule", + ) + + +def _reasons(case: CaseSnapshot) -> list[str]: + return [*case.execution_errors, *case.failure_reasons, *(reason for reasons in case.metric_reasons.values() for reason in reasons)] + + +def _judge_attribution(case: CaseSnapshot, judge: Callable[..., Any]) -> FailureAttribution | None: + try: + result = judge(case) + if isinstance(result, FailureAttribution): + candidate = result + elif isinstance(result, dict): + candidate = FailureAttribution.model_validate( + {"eval_id": case.eval_id, "source": "judge", **result} + ) + else: + return None + except Exception: + return None + if not candidate.evidence: + return None + return candidate.model_copy(update={"eval_id": case.eval_id, "source": "judge"}) + + +def attribute_case(case: CaseSnapshot, *, judge: Callable[..., Any] | None = None) -> FailureAttribution: + """Classify a failed evaluation using deterministic evidence before any judge.""" + reasons = _reasons(case) + reason_text = " ".join(reasons).lower() + + if case.execution_errors: + timeout_error = next((error for error in case.execution_errors if "timeout" in error.lower()), None) + if timeout_error is not None: + return _rule(case, FailureType.TIMEOUT, timeout_error) + return _rule(case, FailureType.EXECUTION_ERROR, case.execution_errors[0]) + + if "timeout" in reason_text: + return _rule(case, FailureType.TIMEOUT, next(reason for reason in reasons if "timeout" in reason.lower())) + + execution_markers = ("execution error", "connection refused", "connection reset", "request failed", "exception") + if any(marker in reason_text for marker in execution_markers): + return _rule(case, FailureType.EXECUTION_ERROR, next(reason for reason in reasons if any(marker in reason.lower() for marker in execution_markers))) + + actual_names = [call.name for call in case.tool_calls] + expected_names = [call.name for call in case.expected_tool_calls] + if actual_names != expected_names: + return _rule(case, FailureType.TOOL_SELECTION_ERROR, f"actual tools={actual_names}; expected tools={expected_names}") + + if any(actual.arguments != expected.arguments for actual, expected in zip(case.tool_calls, case.expected_tool_calls)): + return _rule(case, FailureType.TOOL_ARGUMENT_ERROR, "tool names matched but tool arguments differed") + + tool_response_markers = ("tool response", "tool execution", "tool result") + if any(marker in reason_text for marker in tool_response_markers) and any( + marker in reason_text for marker in ("failed", "error", "empty") + ): + return _rule(case, FailureType.TOOL_EXECUTION_ERROR, next(reason for reason in reasons if any(marker in reason.lower() for marker in tool_response_markers))) + + format_markers = ("invalid json", "missing required field", "missing fake json", "fake json response") + if any(marker in reason_text for marker in format_markers): + return _rule(case, FailureType.FORMAT_VIOLATION, next(reason for reason in reasons if any(marker in reason.lower() for marker in format_markers))) + + failed_metrics = [name for name, passed in case.metric_passed.items() if not passed] + if any("knowledge" in name.lower() for name in failed_metrics): + return _rule(case, FailureType.KNOWLEDGE_RECALL_INSUFFICIENT, f"failed metrics={failed_metrics}") + if any("rubric" in name.lower() for name in failed_metrics): + return _rule(case, FailureType.LLM_RUBRIC_NOT_MET, f"failed metrics={failed_metrics}") + if any("final_response" in name.lower() or "response_match" in name.lower() for name in failed_metrics): + return _rule(case, FailureType.FINAL_RESPONSE_MISMATCH, f"failed metrics={failed_metrics}") + + if judge is not None: + attribution = _judge_attribution(case, judge) + if attribution is not None: + return attribution + return FailureAttribution( + eval_id=case.eval_id, + primary_type=FailureType.UNKNOWN, + confidence=0.0, + evidence=["no deterministic attribution rule matched"], + source="fallback", + ) diff --git a/examples/optimization/eval_optimize_loop/pipeline/models.py b/examples/optimization/eval_optimize_loop/pipeline/models.py index ebdb630d..7f50c663 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/models.py +++ b/examples/optimization/eval_optimize_loop/pipeline/models.py @@ -12,13 +12,16 @@ class StrictModel(BaseModel): class FailureType(str, Enum): + TIMEOUT = "timeout" EXECUTION_ERROR = "execution_error" + TOOL_EXECUTION_ERROR = "tool_execution_error" FINAL_RESPONSE_MISMATCH = "final_response_mismatch" TOOL_SELECTION_ERROR = "tool_selection_error" TOOL_ARGUMENT_ERROR = "tool_argument_error" FORMAT_VIOLATION = "format_violation" KNOWLEDGE_RECALL_INSUFFICIENT = "knowledge_recall_insufficient" LLM_RUBRIC_NOT_MET = "llm_rubric_not_met" + SAFETY_VIOLATION = "safety_violation" UNKNOWN = "unknown" @@ -27,6 +30,15 @@ class ToolCallSnapshot(StrictModel): arguments: dict[str, Any] = Field(default_factory=dict) +class FailureAttribution(StrictModel): + eval_id: str + primary_type: FailureType + secondary_types: list[FailureType] = Field(default_factory=list) + confidence: float = Field(ge=0.0, le=1.0) + evidence: list[str] = Field(min_length=1) + source: Literal["rule", "judge", "fallback"] + + class CaseSnapshot(StrictModel): eval_id: str split: Literal["train", "validation"] @@ -39,8 +51,10 @@ class CaseSnapshot(StrictModel): metric_passed: dict[str, bool] trace_digest: str metric_reasons: dict[str, list[str]] = Field(default_factory=dict) + execution_errors: list[str] = Field(default_factory=list) failure_reasons: list[str] = Field(default_factory=list) failure_types: list[FailureType] = Field(default_factory=list) + failure_attribution: FailureAttribution | None = None final_response: str | None = None expected_response: str | None = None tool_calls: list[ToolCallSnapshot] = Field(default_factory=list) @@ -142,7 +156,7 @@ class CandidateReport(StrictModel): class OptimizationReport(StrictModel): schema_version: str = "1.0" - mode: Literal["fake"] + mode: Literal["fake", "trace"] seed: int selected_candidate_id: str | None = None candidates: list[CandidateReport] = Field(default_factory=list) @@ -151,5 +165,5 @@ class OptimizationReport(StrictModel): source_integrity: Literal["restored", "unknown"] = "restored" @classmethod - def empty(cls, *, mode: Literal["fake"], seed: int) -> "OptimizationReport": + def empty(cls, *, mode: Literal["fake", "trace"], seed: int) -> "OptimizationReport": return cls(mode=mode, seed=seed) diff --git a/examples/optimization/eval_optimize_loop/pipeline/normalization.py b/examples/optimization/eval_optimize_loop/pipeline/normalization.py index 547b500e..11355879 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/normalization.py +++ b/examples/optimization/eval_optimize_loop/pipeline/normalization.py @@ -124,11 +124,12 @@ def normalize_eval_results( [actual_parsed for actual_parsed, _ in parsed_run_responses], [expected_parsed for _, expected_parsed in parsed_run_responses], ) + execution_errors = [run.error_message for run in runs if run.error_message] snapshots[eval_id] = CaseSnapshot( eval_id=eval_id, split=split, run_count=len(runs), passed=all(item.final_eval_status == EvalStatus.PASSED for item in runs), hard_failed=any(item.error_message for item in runs), aggregate_score=weighted / total_weight if total_weight else 0.0, metric_scores=averaged, metric_thresholds=metric_thresholds, metric_passed=metric_passed, - metric_reasons=metric_reasons, failure_reasons=failure_reasons, failure_types=failure_types, + metric_reasons=metric_reasons, execution_errors=execution_errors, failure_reasons=failure_reasons, failure_types=failure_types, final_response=actual, expected_response=expected, trace_digest="sha256:" + hashlib.sha256((actual or "").encode("utf-8")).hexdigest(), tool_calls=parsed_actual.tool_calls, @@ -145,7 +146,10 @@ def _failure_types( expected: list[FakeResponseSnapshot], ) -> list[FailureType]: failure_types: set[FailureType] = set() - if any(run.error_message for run in runs): + error_messages = [run.error_message for run in runs if run.error_message] + if any("timeout" in error.lower() for error in error_messages): + failure_types.add(FailureType.TIMEOUT) + elif error_messages: failure_types.add(FailureType.EXECUTION_ERROR) if any(item.failure_reason for item in actual + expected): failure_types.add(FailureType.FORMAT_VIOLATION) @@ -157,6 +161,8 @@ def _failure_types( failure_types.add(FailureType.TOOL_SELECTION_ERROR) if "tool arguments" in reasons: failure_types.add(FailureType.TOOL_ARGUMENT_ERROR) + if "tool response" in reasons and any(marker in reasons for marker in ("failed", "error", "empty")): + failure_types.add(FailureType.TOOL_EXECUTION_ERROR) if "refuse to guess" in reasons: failure_types.add(FailureType.KNOWLEDGE_RECALL_INSUFFICIENT) if metric_name == "final_response_avg_score": diff --git a/examples/optimization/eval_optimize_loop/pipeline/reporter.py b/examples/optimization/eval_optimize_loop/pipeline/reporter.py index 4d1f2e3c..c2c2a81d 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/reporter.py +++ b/examples/optimization/eval_optimize_loop/pipeline/reporter.py @@ -27,6 +27,20 @@ def write_reports(report: OptimizationReport, output_dir: Path) -> tuple[Path, P for name, split in (("train", report.baseline_train), ("validation", report.baseline_validation)): if split is not None: lines.append(f"| {name} | {split.pass_rate:.3f} | {split.aggregate_score:.3f} |") + baseline_failures = [ + case + for split in (report.baseline_train, report.baseline_validation) + if split is not None + for case in split.cases + if case.failure_attribution is not None + ] + if baseline_failures: + lines.extend(["", "## Baseline failure attribution", "", "| Case | Primary type | Source | Evidence |", "| --- | --- | --- | --- |"]) + for case in baseline_failures: + attribution = case.failure_attribution + lines.append( + f"| `{case.eval_id}` | `{attribution.primary_type.value}` | `{attribution.source}` | {'; '.join(attribution.evidence)} |" + ) lines.extend([ "", "## Candidates", @@ -74,7 +88,7 @@ def write_reports(report: OptimizationReport, output_dir: Path) -> tuple[Path, P "## Reproduction", "", "```text", - "python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake --output-dir ", + f"python examples/optimization/eval_optimize_loop/run_pipeline.py --mode {report.mode} --output-dir ", "```", ]) markdown_path.write_text("\n".join(lines) + "\n", encoding="utf-8") diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index 26169732..bbcfe202 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -2,6 +2,7 @@ import argparse import asyncio +import json import sys import tempfile from pathlib import Path @@ -15,15 +16,20 @@ if str(REPO_ROOT) not in sys.path: sys.path.insert(0, str(REPO_ROOT)) -from trpc_agent_sdk.evaluation import TargetPrompt +from trpc_agent_sdk.evaluation import AgentEvaluator, TargetPrompt +from trpc_agent_sdk.evaluation._eval_config import EvalConfig +from trpc_agent_sdk.evaluation._eval_set import EvalSet from examples.optimization.eval_optimize_loop.fake.fake_agent import FakeSupportAgent +from examples.optimization.eval_optimize_loop.fake.fake_judge import register_fake_rubric_evaluator from examples.optimization.eval_optimize_loop.fake.fixture_optimizer import FixtureOptimizerBackend from examples.optimization.eval_optimize_loop.pipeline.comparator import compare_case from examples.optimization.eval_optimize_loop.pipeline.config import load_pipeline_config from examples.optimization.eval_optimize_loop.pipeline.evaluator import evaluate_split from examples.optimization.eval_optimize_loop.pipeline.gate import evaluate_gate -from examples.optimization.eval_optimize_loop.pipeline.models import CandidateReport, OptimizationReport +from examples.optimization.eval_optimize_loop.pipeline.models import CandidateReport, OptimizationReport, SplitReport +from examples.optimization.eval_optimize_loop.pipeline.attribution import attribute_case +from examples.optimization.eval_optimize_loop.pipeline.normalization import normalize_eval_results from examples.optimization.eval_optimize_loop.pipeline.prompt_sandbox import PromptSandbox from examples.optimization.eval_optimize_loop.pipeline.reporter import write_reports @@ -58,12 +64,63 @@ async def run_fake_pipeline(*, output_dir: Path) -> OptimizationReport: return report +async def run_trace_pipeline(*, output_dir: Path) -> OptimizationReport: + """Evaluate recorded trace conversations without loading an agent or optimizer.""" + trace_dir = HERE / "trace" + config_payload = json.loads((trace_dir / "trace_config.json").read_text(encoding="utf-8")) + eval_config = EvalConfig.model_validate(config_payload["evaluate"]) + eval_set = EvalSet.model_validate_json((trace_dir / "trace.evalset.json").read_text(encoding="utf-8")) + register_fake_rubric_evaluator() + _, _, _, results_by_eval_id = await AgentEvaluator.evaluate_eval_set( + eval_set, + eval_config=eval_config, + num_runs=eval_config.num_runs, + print_detailed_results=False, + ) + normalized = normalize_eval_results( + results_by_eval_id, + split="validation", + metric_weights=config_payload["metric_weights"], + ) + cases = [ + case if case.passed else case.model_copy(update={"failure_attribution": attribute_case(case)}) + for case in normalized.values() + ] + report = OptimizationReport.empty(mode="trace", seed=config_payload["seed"]) + report.baseline_validation = SplitReport.from_cases(cases) + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "trace_raw_results.json").write_text( + json.dumps( + { + "raw_evaluator_ran": True, + "results_by_eval_id": { + eval_id: [result.model_dump(mode="json") for result in results] + for eval_id, results in results_by_eval_id.items() + }, + }, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + (output_dir / "trace_normalized_cases.json").write_text( + json.dumps([case.model_dump(mode="json") for case in cases], ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + write_reports(report, output_dir) + return report + + def main() -> int: parser = argparse.ArgumentParser() - parser.add_argument("--mode", choices=["fake"], default="fake") + parser.add_argument("--mode", choices=["fake", "trace", "live"], default="fake") parser.add_argument("--output-dir", type=Path, default=HERE / "runs" / "manual") args = parser.parse_args() - report = asyncio.run(run_fake_pipeline(output_dir=args.output_dir)) + if args.mode == "live": + parser.error("live mode is not implemented yet") + report = asyncio.run(run_trace_pipeline(output_dir=args.output_dir) if args.mode == "trace" else run_fake_pipeline(output_dir=args.output_dir)) print(f"Decision: {'ACCEPT' if report.selected_candidate_id else 'REJECT'}") print(f"Selected candidate: {report.selected_candidate_id or 'none'}") print(f"JSON report: {(args.output_dir / 'optimization_report.json').resolve()}") diff --git a/examples/optimization/eval_optimize_loop/trace/trace.evalset.json b/examples/optimization/eval_optimize_loop/trace/trace.evalset.json new file mode 100644 index 00000000..825af825 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/trace/trace.evalset.json @@ -0,0 +1,75 @@ +{ + "eval_set_id": "issue_91_trace", + "name": "Issue 91 offline trace attribution", + "description": "Recorded actual conversations are evaluated without agent inference.", + "eval_cases": [ + { + "eval_id": "trace_tool_selection_wrong", + "eval_mode": "trace", + "actual_conversation": [ + { + "user_content": {"role": "user", "parts": [{"text": "查询订单 A100"}]}, + "final_response": {"role": "model", "parts": [{"text": "{\"route\": \"refund_lookup\", \"tool\": \"lookup_refund\", \"arguments\": {\"order_id\": \"A100\"}, \"answer\": \"正在查询退款。\"}"}]}, + "intermediate_data": {"tool_uses": [{"id": "actual-tool", "name": "lookup_refund", "args": {"order_id": "A100"}}]} + } + ], + "conversation": [ + { + "user_content": {"role": "user", "parts": [{"text": "查询订单 A100"}]}, + "final_response": {"role": "model", "parts": [{"text": "{\"route\": \"order_lookup\", \"tool\": \"lookup_order\", \"arguments\": {\"order_id\": \"A100\"}, \"answer\": \"正在查询订单 A100。\"}"}]}, + "intermediate_data": {"tool_uses": [{"id": "expected-tool", "name": "lookup_order", "args": {"order_id": "A100"}}]} + } + ] + }, + { + "eval_id": "trace_tool_argument_wrong", + "eval_mode": "trace", + "actual_conversation": [ + { + "user_content": {"role": "user", "parts": [{"text": "查询订单 A100"}]}, + "final_response": {"role": "model", "parts": [{"text": "{\"route\": \"order_lookup\", \"tool\": \"lookup_order\", \"arguments\": {\"order_id\": \"A999\"}, \"answer\": \"正在查询订单 A999。\"}"}]}, + "intermediate_data": {"tool_uses": [{"id": "actual-tool", "name": "lookup_order", "args": {"order_id": "A999"}}]} + } + ], + "conversation": [ + { + "user_content": {"role": "user", "parts": [{"text": "查询订单 A100"}]}, + "final_response": {"role": "model", "parts": [{"text": "{\"route\": \"order_lookup\", \"tool\": \"lookup_order\", \"arguments\": {\"order_id\": \"A100\"}, \"answer\": \"正在查询订单 A100。\"}"}]}, + "intermediate_data": {"tool_uses": [{"id": "expected-tool", "name": "lookup_order", "args": {"order_id": "A100"}}]} + } + ] + }, + { + "eval_id": "trace_invalid_final_format", + "eval_mode": "trace", + "actual_conversation": [ + { + "user_content": {"role": "user", "parts": [{"text": "查询订单 A100"}]}, + "final_response": {"role": "model", "parts": [{"text": "not valid fake JSON"}]} + } + ], + "conversation": [ + { + "user_content": {"role": "user", "parts": [{"text": "查询订单 A100"}]}, + "final_response": {"role": "model", "parts": [{"text": "{\"route\": \"format_check\", \"tool\": \"none\", \"arguments\": {}, \"answer\": \"请返回 JSON。\"}"}]} + } + ] + }, + { + "eval_id": "trace_pass", + "eval_mode": "trace", + "actual_conversation": [ + { + "user_content": {"role": "user", "parts": [{"text": "查询订单 A100"}]}, + "final_response": {"role": "model", "parts": [{"text": "{\"route\": \"order_lookup\", \"tool\": \"lookup_order\", \"arguments\": {\"order_id\": \"A100\"}, \"answer\": \"正在查询订单 A100。\"}"}]} + } + ], + "conversation": [ + { + "user_content": {"role": "user", "parts": [{"text": "查询订单 A100"}]}, + "final_response": {"role": "model", "parts": [{"text": "{\"route\": \"order_lookup\", \"tool\": \"lookup_order\", \"arguments\": {\"order_id\": \"A100\"}, \"answer\": \"正在查询订单 A100。\"}"}]} + } + ] + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/trace/trace_config.json b/examples/optimization/eval_optimize_loop/trace/trace_config.json new file mode 100644 index 00000000..18e92cfe --- /dev/null +++ b/examples/optimization/eval_optimize_loop/trace/trace_config.json @@ -0,0 +1,27 @@ +{ + "evaluate": { + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact" + } + } + } + }, + { + "metric_name": "fake_rubric_score", + "threshold": 0.75 + } + ], + "num_runs": 1 + }, + "metric_weights": { + "final_response_avg_score": 0.6, + "fake_rubric_score": 0.4 + }, + "seed": 42 +} diff --git a/tests/examples/optimization/eval_optimize_loop/test_attribution_and_trace.py b/tests/examples/optimization/eval_optimize_loop/test_attribution_and_trace.py new file mode 100644 index 00000000..00844c9b --- /dev/null +++ b/tests/examples/optimization/eval_optimize_loop/test_attribution_and_trace.py @@ -0,0 +1,157 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path +from typing import Callable + +import pytest + +from examples.optimization.eval_optimize_loop.pipeline.models import CaseSnapshot, ToolCallSnapshot + + +def _case(**overrides: object) -> CaseSnapshot: + values: dict[str, object] = { + "eval_id": "case", + "split": "validation", + "run_count": 1, + "passed": False, + "hard_failed": False, + "aggregate_score": 0.0, + "metric_scores": {}, + "metric_thresholds": {}, + "metric_passed": {}, + "trace_digest": "sha256:test", + } + values.update(overrides) + return CaseSnapshot.model_validate(values) + + +def _attribute(case: CaseSnapshot, judge: Callable | None = None): + from examples.optimization.eval_optimize_loop.pipeline.attribution import attribute_case + + return attribute_case(case, judge=judge) + + +@pytest.mark.parametrize( + ("case", "expected"), + [ + (_case(failure_reasons=["request timeout after 30 seconds"]), "timeout"), + (_case(execution_errors=["backend quota exhausted"]), "execution_error"), + (_case(failure_reasons=["connection refused while executing request"]), "execution_error"), + (_case(tool_calls=[], expected_tool_calls=[ToolCallSnapshot(name="lookup_order")]), "tool_selection_error"), + (_case(tool_calls=[ToolCallSnapshot(name="lookup_order", arguments={"order_id": 7})], expected_tool_calls=[ToolCallSnapshot(name="lookup_order", arguments={"order_id": "A100"})]), "tool_argument_error"), + (_case(failure_reasons=["tool response was empty"]), "tool_execution_error"), + (_case(failure_reasons=["actual response: invalid JSON response"]), "format_violation"), + (_case(metric_passed={"knowledge_recall_score": False}), "knowledge_recall_insufficient"), + (_case(metric_passed={"fake_rubric_score": False}), "llm_rubric_not_met"), + (_case(metric_passed={"final_response_avg_score": False}), "final_response_mismatch"), + (_case(), "unknown"), + ], + ids=( + "timeout", + "recorded-execution-error", + "execution-error", + "tool-selection", + "tool-arguments", + "tool-execution", + "format", + "knowledge", + "rubric", + "final-response", + "no-rule", + ), +) +def test_rule_first_attribution_covers_each_precedence_category(case: CaseSnapshot, expected: str) -> None: + attribution = _attribute(case) + + assert attribution.primary_type.value == expected + assert attribution.evidence + + +def test_judge_is_used_only_when_no_rule_matches() -> None: + calls: list[str] = [] + + def judge(_case: CaseSnapshot) -> dict[str, object]: + calls.append("judge") + return {"primary_type": "safety_violation", "confidence": 0.8, "evidence": ["judge-only signal"]} + + attribution = _attribute(_case(), judge=judge) + + assert calls == ["judge"] + assert attribution.primary_type.value == "safety_violation" + assert attribution.source == "judge" + + +def test_invalid_judge_output_falls_back_to_unknown() -> None: + attribution = _attribute(_case(), judge=lambda _case: {}) + + assert attribution.primary_type.value == "unknown" + assert attribution.source == "fallback" + assert attribution.evidence + + +def test_structural_rule_wins_over_conflicting_judge() -> None: + calls: list[str] = [] + + def judge(_case: CaseSnapshot) -> dict[str, object]: + calls.append("judge") + return {"primary_type": "safety_violation", "confidence": 1.0, "evidence": ["incorrect override"]} + + attribution = _attribute( + _case(tool_calls=[], expected_tool_calls=[ToolCallSnapshot(name="lookup_order")]), judge=judge + ) + + assert attribution.primary_type.value == "tool_selection_error" + assert attribution.source == "rule" + assert calls == [] + + +@pytest.mark.asyncio +async def test_trace_mode_evaluates_recorded_conversations_without_a_key( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.delenv("TRPC_AGENT_API_KEY", raising=False) + from examples.optimization.eval_optimize_loop.run_pipeline import run_trace_pipeline + + report = await run_trace_pipeline(output_dir=tmp_path) + + assert report.mode == "trace" + assert report.selected_candidate_id is None + assert report.baseline_validation is not None + failures = [case for case in report.baseline_validation.cases if not case.passed] + assert {case.failure_attribution.primary_type.value for case in failures if case.failure_attribution} == { + "tool_selection_error", + "tool_argument_error", + "format_violation", + } + assert all(case.failure_attribution and case.failure_attribution.evidence for case in failures) + assert sum(case.passed for case in report.baseline_validation.cases) == 1 + assert (tmp_path / "optimization_report.json").is_file() + assert (tmp_path / "optimization_report.md").is_file() + assert (tmp_path / "trace_raw_results.json").is_file() + assert (tmp_path / "trace_normalized_cases.json").is_file() + assert json.loads((tmp_path / "trace_raw_results.json").read_text(encoding="utf-8"))["raw_evaluator_ran"] is True + + +def test_trace_cli_writes_report_paths(tmp_path: Path) -> None: + repo_root = Path(__file__).resolve().parents[4] + result = subprocess.run( + [ + sys.executable, + "examples/optimization/eval_optimize_loop/run_pipeline.py", + "--mode", + "trace", + "--output-dir", + str(tmp_path), + ], + cwd=repo_root, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + assert "JSON report:" in result.stdout + assert "Markdown report:" in result.stdout From 1af2f4604e249367d0ce1e53194d9cca5c55af85 Mon Sep 17 00:00:00 2001 From: shiquan_zhan Date: Sat, 11 Jul 2026 13:06:57 +0800 Subject: [PATCH 07/15] fix(eval): preserve trace tool evidence --- .superpowers/sdd/task-3-report.md | 27 +++++ .../pipeline/attribution.py | 20 ++++ .../eval_optimize_loop/pipeline/models.py | 8 +- .../pipeline/normalization.py | 39 ++++++- .../trace/trace.evalset.json | 26 ++++- .../trace/trace_config.json | 9 +- .../test_attribution_and_trace.py | 105 ++++++++++++++++++ 7 files changed, 227 insertions(+), 7 deletions(-) diff --git a/.superpowers/sdd/task-3-report.md b/.superpowers/sdd/task-3-report.md index 4d06d0c0..c68769b6 100644 --- a/.superpowers/sdd/task-3-report.md +++ b/.superpowers/sdd/task-3-report.md @@ -49,3 +49,30 @@ python examples/optimization/eval_optimize_loop/run_pipeline.py --mode trace --o - The invalid-format trace uses a reference with `tool: none`, so no higher-priority tool-selection mismatch exists; it therefore correctly reaches `FORMAT_VIOLATION` under the required precedence. - Test and CLI runs emit pre-existing third-party dependency warnings from `requests` and `langgraph`; they do not affect exit status. + +## P1 reviewer follow-up + +### Changes + +- Tool-call snapshots now come from each evaluator invocation's `intermediate_data.tool_uses`, rather than the fake final-response JSON. +- `ToolCallSnapshot.arguments` preserves the raw value, including invalid non-dict shapes such as `[]`, so structural comparison can classify `[]` versus `{}` as `TOOL_ARGUMENT_ERROR`. +- `CaseSnapshot.tool_responses` now preserves `intermediate_data.tool_responses` values. Explicit `error`, `failed`, failed `status`, or empty responses yield rule-first `TOOL_EXECUTION_ERROR` before final-response metrics. +- The trace EvalSet now includes a final-JSON-match / intermediate-tool-mismatch case and an explicit tool-response-error case. It evaluates trajectories locally with `tool_trajectory_avg_score`. + +### P1 RED/GREEN evidence + +Before the follow-up implementation, the three new normalizer tests failed because final fake JSON overwrote intermediate calls, `[]` was coerced to `{}`, and `CaseSnapshot` lacked `tool_responses`. The trace E2E expectation also failed before the tool-response-error fixture existed. + +After implementation: + +```text +python -m pytest tests/examples/optimization/eval_optimize_loop/test_attribution_and_trace.py -q +# 19 passed + +python -m pytest tests/examples/optimization/eval_optimize_loop -q +# 52 passed + +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode trace --output-dir +# exit 0; normalized output preserves intermediate lookup_refund versus lookup_order, +# and tool_responses.error as raw structural evidence +``` diff --git a/examples/optimization/eval_optimize_loop/pipeline/attribution.py b/examples/optimization/eval_optimize_loop/pipeline/attribution.py index c7c213d3..4c4b818f 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/attribution.py +++ b/examples/optimization/eval_optimize_loop/pipeline/attribution.py @@ -37,6 +37,22 @@ def _judge_attribution(case: CaseSnapshot, judge: Callable[..., Any]) -> Failure return candidate.model_copy(update={"eval_id": case.eval_id, "source": "judge"}) +def _tool_response_failure(case: CaseSnapshot) -> str | None: + for response in case.tool_responses: + value = response.response + if value is None or value == {}: + return f"tool response for {response.name or ''} was empty" + if isinstance(value, dict): + if value.get("error"): + return f"tool response for {response.name or ''} contained an error: {value['error']}" + if value.get("failed") is True: + return f"tool response for {response.name or ''} was failed" + status = value.get("status") + if isinstance(status, str) and status.lower() in {"error", "failed"}: + return f"tool response for {response.name or ''} had status {status}" + return None + + def attribute_case(case: CaseSnapshot, *, judge: Callable[..., Any] | None = None) -> FailureAttribution: """Classify a failed evaluation using deterministic evidence before any judge.""" reasons = _reasons(case) @@ -63,6 +79,10 @@ def attribute_case(case: CaseSnapshot, *, judge: Callable[..., Any] | None = Non if any(actual.arguments != expected.arguments for actual, expected in zip(case.tool_calls, case.expected_tool_calls)): return _rule(case, FailureType.TOOL_ARGUMENT_ERROR, "tool names matched but tool arguments differed") + response_failure = _tool_response_failure(case) + if response_failure is not None: + return _rule(case, FailureType.TOOL_EXECUTION_ERROR, response_failure) + tool_response_markers = ("tool response", "tool execution", "tool result") if any(marker in reason_text for marker in tool_response_markers) and any( marker in reason_text for marker in ("failed", "error", "empty") diff --git a/examples/optimization/eval_optimize_loop/pipeline/models.py b/examples/optimization/eval_optimize_loop/pipeline/models.py index 7f50c663..86b45caf 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/models.py +++ b/examples/optimization/eval_optimize_loop/pipeline/models.py @@ -27,7 +27,12 @@ class FailureType(str, Enum): class ToolCallSnapshot(StrictModel): name: str - arguments: dict[str, Any] = Field(default_factory=dict) + arguments: Any = Field(default_factory=dict) + + +class ToolResponseSnapshot(StrictModel): + name: str + response: Any = None class FailureAttribution(StrictModel): @@ -59,6 +64,7 @@ class CaseSnapshot(StrictModel): expected_response: str | None = None tool_calls: list[ToolCallSnapshot] = Field(default_factory=list) expected_tool_calls: list[ToolCallSnapshot] = Field(default_factory=list) + tool_responses: list[ToolResponseSnapshot] = Field(default_factory=list) class CaseDelta(StrictModel): diff --git a/examples/optimization/eval_optimize_loop/pipeline/normalization.py b/examples/optimization/eval_optimize_loop/pipeline/normalization.py index 11355879..96faf521 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/normalization.py +++ b/examples/optimization/eval_optimize_loop/pipeline/normalization.py @@ -7,8 +7,9 @@ from trpc_agent_sdk.evaluation._eval_metrics import EvalStatus from trpc_agent_sdk.evaluation._eval_result import EvalCaseResult +from trpc_agent_sdk.evaluation._eval_case import get_all_tool_calls, get_all_tool_responses -from .models import CaseSnapshot, FailureType, ToolCallSnapshot +from .models import CaseSnapshot, FailureType, ToolCallSnapshot, ToolResponseSnapshot @dataclass(frozen=True) @@ -47,6 +48,20 @@ def _text(content: object) -> str | None: return "".join(part.text or "" for part in parts) +def _tool_call_snapshot(tool_call: object) -> ToolCallSnapshot: + return ToolCallSnapshot( + name=str(getattr(tool_call, "name", "") or ""), + arguments=getattr(tool_call, "args", None), + ) + + +def _tool_response_snapshot(tool_response: object) -> ToolResponseSnapshot: + return ToolResponseSnapshot( + name=str(getattr(tool_response, "name", "") or ""), + response=getattr(tool_response, "response", None), + ) + + def normalize_eval_results( results_by_eval_id: dict[str, list[EvalCaseResult]], *, split: Literal["train", "validation"], metric_weights: dict[str, float] ) -> dict[str, CaseSnapshot]: @@ -92,6 +107,23 @@ def normalize_eval_results( expected = _text(invocation.expected_invocation.final_response) if invocation and invocation.expected_invocation else None parsed_actual = parse_fake_response(actual) parsed_expected = parse_fake_response(expected) + first_invocations = first.eval_metric_result_per_invocation + tool_calls = [ + _tool_call_snapshot(tool_call) + for run_invocation in first_invocations + for tool_call in get_all_tool_calls(run_invocation.actual_invocation.intermediate_data) + ] + expected_tool_calls = [ + _tool_call_snapshot(tool_call) + for run_invocation in first_invocations + if run_invocation.expected_invocation is not None + for tool_call in get_all_tool_calls(run_invocation.expected_invocation.intermediate_data) + ] + tool_responses = [ + _tool_response_snapshot(tool_response) + for run_invocation in first_invocations + for tool_response in get_all_tool_responses(run_invocation.actual_invocation.intermediate_data) + ] parsed_run_responses = [ ( parse_fake_response(_text(run_invocation.actual_invocation.final_response)), @@ -132,8 +164,9 @@ def normalize_eval_results( metric_reasons=metric_reasons, execution_errors=execution_errors, failure_reasons=failure_reasons, failure_types=failure_types, final_response=actual, expected_response=expected, trace_digest="sha256:" + hashlib.sha256((actual or "").encode("utf-8")).hexdigest(), - tool_calls=parsed_actual.tool_calls, - expected_tool_calls=parsed_expected.tool_calls, + tool_calls=tool_calls, + expected_tool_calls=expected_tool_calls, + tool_responses=tool_responses, ) return snapshots diff --git a/examples/optimization/eval_optimize_loop/trace/trace.evalset.json b/examples/optimization/eval_optimize_loop/trace/trace.evalset.json index 825af825..d5cd1f2b 100644 --- a/examples/optimization/eval_optimize_loop/trace/trace.evalset.json +++ b/examples/optimization/eval_optimize_loop/trace/trace.evalset.json @@ -9,7 +9,7 @@ "actual_conversation": [ { "user_content": {"role": "user", "parts": [{"text": "查询订单 A100"}]}, - "final_response": {"role": "model", "parts": [{"text": "{\"route\": \"refund_lookup\", \"tool\": \"lookup_refund\", \"arguments\": {\"order_id\": \"A100\"}, \"answer\": \"正在查询退款。\"}"}]}, + "final_response": {"role": "model", "parts": [{"text": "{\"route\": \"order_lookup\", \"tool\": \"lookup_order\", \"arguments\": {\"order_id\": \"A100\"}, \"answer\": \"正在查询订单 A100。\"}"}]}, "intermediate_data": {"tool_uses": [{"id": "actual-tool", "name": "lookup_refund", "args": {"order_id": "A100"}}]} } ], @@ -55,6 +55,30 @@ } ] }, + { + "eval_id": "trace_tool_response_error", + "eval_mode": "trace", + "actual_conversation": [ + { + "user_content": {"role": "user", "parts": [{"text": "查询订单 A100"}]}, + "final_response": {"role": "model", "parts": [{"text": "{\"route\": \"order_lookup\", \"tool\": \"lookup_order\", \"arguments\": {\"order_id\": \"A100\"}, \"answer\": \"订单服务暂时不可用。\"}"}]}, + "intermediate_data": { + "tool_uses": [{"id": "actual-tool", "name": "lookup_order", "args": {"order_id": "A100"}}], + "tool_responses": [{"id": "actual-tool", "name": "lookup_order", "response": {"error": "upstream unavailable"}}] + } + } + ], + "conversation": [ + { + "user_content": {"role": "user", "parts": [{"text": "查询订单 A100"}]}, + "final_response": {"role": "model", "parts": [{"text": "{\"route\": \"order_lookup\", \"tool\": \"lookup_order\", \"arguments\": {\"order_id\": \"A100\"}, \"answer\": \"正在查询订单 A100。\"}"}]}, + "intermediate_data": { + "tool_uses": [{"id": "expected-tool", "name": "lookup_order", "args": {"order_id": "A100"}}], + "tool_responses": [{"id": "expected-tool", "name": "lookup_order", "response": {"output": "订单 A100"}}] + } + } + ] + }, { "eval_id": "trace_pass", "eval_mode": "trace", diff --git a/examples/optimization/eval_optimize_loop/trace/trace_config.json b/examples/optimization/eval_optimize_loop/trace/trace_config.json index 18e92cfe..fcdf4c82 100644 --- a/examples/optimization/eval_optimize_loop/trace/trace_config.json +++ b/examples/optimization/eval_optimize_loop/trace/trace_config.json @@ -15,13 +15,18 @@ { "metric_name": "fake_rubric_score", "threshold": 0.75 + }, + { + "metric_name": "tool_trajectory_avg_score", + "threshold": 1.0 } ], "num_runs": 1 }, "metric_weights": { - "final_response_avg_score": 0.6, - "fake_rubric_score": 0.4 + "final_response_avg_score": 0.5, + "fake_rubric_score": 0.3, + "tool_trajectory_avg_score": 0.2 }, "seed": 42 } diff --git a/tests/examples/optimization/eval_optimize_loop/test_attribution_and_trace.py b/tests/examples/optimization/eval_optimize_loop/test_attribution_and_trace.py index 00844c9b..bee2bd4d 100644 --- a/tests/examples/optimization/eval_optimize_loop/test_attribution_and_trace.py +++ b/tests/examples/optimization/eval_optimize_loop/test_attribution_and_trace.py @@ -9,6 +9,11 @@ import pytest from examples.optimization.eval_optimize_loop.pipeline.models import CaseSnapshot, ToolCallSnapshot +from examples.optimization.eval_optimize_loop.pipeline.normalization import normalize_eval_results +from trpc_agent_sdk.evaluation._eval_case import IntermediateData, Invocation +from trpc_agent_sdk.evaluation._eval_metrics import EvalStatus +from trpc_agent_sdk.evaluation._eval_result import EvalCaseResult, EvalMetricResult, EvalMetricResultPerInvocation +from trpc_agent_sdk.types import Content, FunctionCall, FunctionResponse, Part def _case(**overrides: object) -> CaseSnapshot: @@ -34,6 +39,42 @@ def _attribute(case: CaseSnapshot, judge: Callable | None = None): return attribute_case(case, judge=judge) +def _invocation( + response: str, + *, + tool_uses: list[FunctionCall] | None = None, + tool_responses: list[FunctionResponse] | None = None, +) -> Invocation: + return Invocation( + user_content=Content(role="user", parts=[Part(text="query")]), + final_response=Content(role="model", parts=[Part(text=response)]), + intermediate_data=IntermediateData(tool_uses=tool_uses or [], tool_responses=tool_responses or []), + ) + + +def _normalization_result(actual: Invocation, expected: Invocation) -> EvalCaseResult: + metric = EvalMetricResult( + metric_name="final_response_avg_score", + threshold=1.0, + score=0.0, + eval_status=EvalStatus.FAILED, + ) + return EvalCaseResult( + eval_set_id="test", + eval_id="case", + final_eval_status=EvalStatus.FAILED, + overall_eval_metric_results=[metric], + eval_metric_result_per_invocation=[ + EvalMetricResultPerInvocation( + actual_invocation=actual, + expected_invocation=expected, + eval_metric_results=[metric], + ) + ], + session_id="test", + ) + + @pytest.mark.parametrize( ("case", "expected"), [ @@ -108,6 +149,64 @@ def judge(_case: CaseSnapshot) -> dict[str, object]: assert calls == [] +def test_normalization_uses_intermediate_tools_even_when_final_json_matches() -> None: + response = '{"route":"order_lookup","tool":"lookup_order","arguments":{"order_id":"A100"},"answer":"ok"}' + case = normalize_eval_results( + { + "case": [ + _normalization_result( + _invocation(response, tool_uses=[FunctionCall(name="lookup_refund", args={"order_id": "A100"})]), + _invocation(response, tool_uses=[FunctionCall(name="lookup_order", args={"order_id": "A100"})]), + ) + ] + }, + split="validation", + metric_weights={"final_response_avg_score": 1.0}, + )["case"] + + assert case.final_response == case.expected_response + assert [tool.name for tool in case.tool_calls] == ["lookup_refund"] + assert [tool.name for tool in case.expected_tool_calls] == ["lookup_order"] + assert _attribute(case).primary_type.value == "tool_selection_error" + + +def test_normalization_preserves_invalid_tool_argument_shape_for_attribution() -> None: + response = '{"route":"order_lookup","tool":"lookup_order","arguments":{},"answer":"ok"}' + invalid_call = FunctionCall.model_construct(name="lookup_order", args=[]) + case = normalize_eval_results( + {"case": [_normalization_result(_invocation(response, tool_uses=[invalid_call]), _invocation(response, tool_uses=[FunctionCall(name="lookup_order", args={})]))]}, + split="validation", + metric_weights={"final_response_avg_score": 1.0}, + )["case"] + + assert case.tool_calls[0].arguments == [] + assert case.expected_tool_calls[0].arguments == {} + assert _attribute(case).primary_type.value == "tool_argument_error" + + +def test_normalization_attributes_structured_tool_response_error() -> None: + response = '{"route":"order_lookup","tool":"lookup_order","arguments":{},"answer":"different"}' + case = normalize_eval_results( + { + "case": [ + _normalization_result( + _invocation( + response, + tool_uses=[FunctionCall(name="lookup_order", args={})], + tool_responses=[FunctionResponse(name="lookup_order", response={"error": "upstream unavailable"})], + ), + _invocation(response, tool_uses=[FunctionCall(name="lookup_order", args={})]), + ) + ] + }, + split="validation", + metric_weights={"final_response_avg_score": 1.0}, + )["case"] + + assert case.tool_responses[0].response == {"error": "upstream unavailable"} + assert _attribute(case).primary_type.value == "tool_execution_error" + + @pytest.mark.asyncio async def test_trace_mode_evaluates_recorded_conversations_without_a_key( monkeypatch: pytest.MonkeyPatch, tmp_path: Path @@ -125,9 +224,15 @@ async def test_trace_mode_evaluates_recorded_conversations_without_a_key( "tool_selection_error", "tool_argument_error", "format_violation", + "tool_execution_error", } assert all(case.failure_attribution and case.failure_attribution.evidence for case in failures) assert sum(case.passed for case in report.baseline_validation.cases) == 1 + trace_cases = {case.eval_id: case for case in report.baseline_validation.cases} + intermediate_wins = trace_cases["trace_tool_selection_wrong"] + assert intermediate_wins.final_response == intermediate_wins.expected_response + assert [tool.name for tool in intermediate_wins.tool_calls] == ["lookup_refund"] + assert [tool.name for tool in intermediate_wins.expected_tool_calls] == ["lookup_order"] assert (tmp_path / "optimization_report.json").is_file() assert (tmp_path / "optimization_report.md").is_file() assert (tmp_path / "trace_raw_results.json").is_file() From 8f355a6c4f0f8f5ba96141f897292f6b21b134e8 Mon Sep 17 00:00:00 2001 From: shiquan_zhan Date: Sat, 11 Jul 2026 13:11:52 +0800 Subject: [PATCH 08/15] fix(eval): retain fake json tool snapshots --- .superpowers/sdd/task-3-report.md | 20 +++++++++++++++++++ .../pipeline/normalization.py | 6 ++++-- .../eval_optimize_loop/test_fake_loop.py | 12 +++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/.superpowers/sdd/task-3-report.md b/.superpowers/sdd/task-3-report.md index c68769b6..178b2096 100644 --- a/.superpowers/sdd/task-3-report.md +++ b/.superpowers/sdd/task-3-report.md @@ -76,3 +76,23 @@ python examples/optimization/eval_optimize_loop/run_pipeline.py --mode trace --o # exit 0; normalized output preserves intermediate lookup_refund versus lookup_order, # and tool_responses.error as raw structural evidence ``` + +## P2 reviewer follow-up + +`normalize_eval_results()` now uses intermediate tool calls only when a recorded call exists. For black-box fake evaluation, where `RemoteEvalService` deliberately leaves actual `intermediate_data` empty, it falls back to the parsed fake JSON calls. The fake regression confirms the baseline actual `lookup_order` with `{}` arguments and reference `lookup_order` with `{"order_id": "A100"}` arguments both remain visible. + +Verification after the P2 change: + +```text +python -m pytest tests/examples/optimization/eval_optimize_loop/test_fake_loop.py -q -k keeps_parsed_json_tool_calls +# 1 passed + +python -m pytest tests/examples/optimization/eval_optimize_loop/test_attribution_and_trace.py -q +# 19 passed + +python -m pytest tests/examples/optimization/eval_optimize_loop -q +# 53 passed + +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode trace --output-dir +# exit 0 +``` diff --git a/examples/optimization/eval_optimize_loop/pipeline/normalization.py b/examples/optimization/eval_optimize_loop/pipeline/normalization.py index 96faf521..85f5dfae 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/normalization.py +++ b/examples/optimization/eval_optimize_loop/pipeline/normalization.py @@ -108,17 +108,19 @@ def normalize_eval_results( parsed_actual = parse_fake_response(actual) parsed_expected = parse_fake_response(expected) first_invocations = first.eval_metric_result_per_invocation - tool_calls = [ + intermediate_tool_calls = [ _tool_call_snapshot(tool_call) for run_invocation in first_invocations for tool_call in get_all_tool_calls(run_invocation.actual_invocation.intermediate_data) ] - expected_tool_calls = [ + intermediate_expected_tool_calls = [ _tool_call_snapshot(tool_call) for run_invocation in first_invocations if run_invocation.expected_invocation is not None for tool_call in get_all_tool_calls(run_invocation.expected_invocation.intermediate_data) ] + tool_calls = intermediate_tool_calls or parsed_actual.tool_calls + expected_tool_calls = intermediate_expected_tool_calls or parsed_expected.tool_calls tool_responses = [ _tool_response_snapshot(tool_response) for run_invocation in first_invocations diff --git a/tests/examples/optimization/eval_optimize_loop/test_fake_loop.py b/tests/examples/optimization/eval_optimize_loop/test_fake_loop.py index 886b7dd8..2a45d222 100644 --- a/tests/examples/optimization/eval_optimize_loop/test_fake_loop.py +++ b/tests/examples/optimization/eval_optimize_loop/test_fake_loop.py @@ -34,6 +34,18 @@ async def test_fake_loop_does_not_require_live_model_key(monkeypatch: pytest.Mon assert report.mode == "fake" +@pytest.mark.asyncio +async def test_fake_mode_keeps_parsed_json_tool_calls_when_no_intermediate_trace(tmp_path: Path) -> None: + report = await run_fake_pipeline(output_dir=tmp_path) + + assert report.baseline_train is not None + case = next(item for item in report.baseline_train.cases if item.eval_id == "train_tool_argument") + assert [(tool.name, tool.arguments) for tool in case.tool_calls] == [("lookup_order", {})] + assert [(tool.name, tool.arguments) for tool in case.expected_tool_calls] == [ + ("lookup_order", {"order_id": "A100"}) + ] + + def test_fake_cli_runs_directly_from_repository_root(tmp_path: Path) -> None: repo_root = Path(__file__).resolve().parents[4] result = subprocess.run( From 590ac10db4a3ee980cd6851c401e864c32c579b4 Mon Sep 17 00:00:00 2001 From: shiquan_zhan Date: Sat, 11 Jul 2026 13:14:47 +0800 Subject: [PATCH 09/15] fix(eval): honor empty intermediate traces --- .superpowers/sdd/task-3-report.md | 17 ++++++++++++++++ .../pipeline/normalization.py | 15 ++++++++++++-- .../test_attribution_and_trace.py | 20 +++++++++++++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/.superpowers/sdd/task-3-report.md b/.superpowers/sdd/task-3-report.md index 178b2096..72389810 100644 --- a/.superpowers/sdd/task-3-report.md +++ b/.superpowers/sdd/task-3-report.md @@ -96,3 +96,20 @@ python -m pytest tests/examples/optimization/eval_optimize_loop -q python examples/optimization/eval_optimize_loop/run_pipeline.py --mode trace --output-dir # exit 0 ``` + +## Final P1 reviewer follow-up + +Fallback now depends on whether an invocation has `intermediate_data is None`, rather than whether extracting its `tool_uses` produced an empty list. An explicit `IntermediateData(tool_uses=[])` is therefore authoritative and remains empty even if the final fake JSON claims a tool. The matching regression verifies the missing-tool structural attribution against a reference trace that contains `lookup_order`. + +Verification after this final adjustment: + +```text +python -m pytest tests/examples/optimization/eval_optimize_loop/test_attribution_and_trace.py -q +# 20 passed + +python -m pytest tests/examples/optimization/eval_optimize_loop -q +# 54 passed + +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode trace --output-dir +# exit 0 +``` diff --git a/examples/optimization/eval_optimize_loop/pipeline/normalization.py b/examples/optimization/eval_optimize_loop/pipeline/normalization.py index 85f5dfae..62f24782 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/normalization.py +++ b/examples/optimization/eval_optimize_loop/pipeline/normalization.py @@ -119,8 +119,19 @@ def normalize_eval_results( if run_invocation.expected_invocation is not None for tool_call in get_all_tool_calls(run_invocation.expected_invocation.intermediate_data) ] - tool_calls = intermediate_tool_calls or parsed_actual.tool_calls - expected_tool_calls = intermediate_expected_tool_calls or parsed_expected.tool_calls + actual_has_intermediate_data = any( + run_invocation.actual_invocation.intermediate_data is not None + for run_invocation in first_invocations + ) + expected_has_intermediate_data = any( + run_invocation.expected_invocation is not None + and run_invocation.expected_invocation.intermediate_data is not None + for run_invocation in first_invocations + ) + tool_calls = intermediate_tool_calls if actual_has_intermediate_data else parsed_actual.tool_calls + expected_tool_calls = ( + intermediate_expected_tool_calls if expected_has_intermediate_data else parsed_expected.tool_calls + ) tool_responses = [ _tool_response_snapshot(tool_response) for run_invocation in first_invocations diff --git a/tests/examples/optimization/eval_optimize_loop/test_attribution_and_trace.py b/tests/examples/optimization/eval_optimize_loop/test_attribution_and_trace.py index bee2bd4d..cf08a171 100644 --- a/tests/examples/optimization/eval_optimize_loop/test_attribution_and_trace.py +++ b/tests/examples/optimization/eval_optimize_loop/test_attribution_and_trace.py @@ -170,6 +170,26 @@ def test_normalization_uses_intermediate_tools_even_when_final_json_matches() -> assert _attribute(case).primary_type.value == "tool_selection_error" +def test_normalization_keeps_explicit_empty_intermediate_tools_over_final_json() -> None: + response = '{"route":"order_lookup","tool":"lookup_order","arguments":{"order_id":"A100"},"answer":"ok"}' + case = normalize_eval_results( + { + "case": [ + _normalization_result( + _invocation(response, tool_uses=[]), + _invocation(response, tool_uses=[FunctionCall(name="lookup_order", args={"order_id": "A100"})]), + ) + ] + }, + split="validation", + metric_weights={"final_response_avg_score": 1.0}, + )["case"] + + assert case.tool_calls == [] + assert [tool.name for tool in case.expected_tool_calls] == ["lookup_order"] + assert _attribute(case).primary_type.value == "tool_selection_error" + + def test_normalization_preserves_invalid_tool_argument_shape_for_attribution() -> None: response = '{"route":"order_lookup","tool":"lookup_order","arguments":{},"answer":"ok"}' invalid_call = FunctionCall.model_construct(name="lookup_order", args=[]) From 4fa7178a704c2c77f9634e5f8d82e226b53ccafa Mon Sep 17 00:00:00 2001 From: shiquan_zhan Date: Sat, 11 Jul 2026 13:27:47 +0800 Subject: [PATCH 10/15] feat(eval): add safe live optimizer backend --- .../eval_optimize_loop/optimizer.json | 14 ++ .../eval_optimize_loop/pipeline/models.py | 13 +- .../pipeline/optimizer_backend.py | 181 +++++++++++++++++ .../eval_optimize_loop/run_pipeline.py | 92 ++++++++- .../eval_optimize_loop/test_live_backend.py | 188 ++++++++++++++++++ 5 files changed, 482 insertions(+), 6 deletions(-) create mode 100644 examples/optimization/eval_optimize_loop/pipeline/optimizer_backend.py create mode 100644 tests/examples/optimization/eval_optimize_loop/test_live_backend.py diff --git a/examples/optimization/eval_optimize_loop/optimizer.json b/examples/optimization/eval_optimize_loop/optimizer.json index 864d8ff1..ba9dfeb3 100644 --- a/examples/optimization/eval_optimize_loop/optimizer.json +++ b/examples/optimization/eval_optimize_loop/optimizer.json @@ -6,6 +6,20 @@ ], "num_runs": 1 }, + "optimize": { + "eval_case_parallelism": 1, + "stop": {"required_metrics": "all"}, + "algorithm": { + "name": "gepa_reflective", + "seed": 42, + "reflection_lm": { + "model_name": "${TRPC_AGENT_MODEL_NAME}", + "base_url": "${TRPC_AGENT_BASE_URL}", + "api_key": "${TRPC_AGENT_API_KEY}" + }, + "max_metric_calls": 10 + } + }, "pipeline": { "reproducibility": {"seed": 42}, "gate": {"min_validation_score_delta": 0.05, "critical_case_ids": ["val_refund_critical"]} diff --git a/examples/optimization/eval_optimize_loop/pipeline/models.py b/examples/optimization/eval_optimize_loop/pipeline/models.py index 86b45caf..535744e8 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/models.py +++ b/examples/optimization/eval_optimize_loop/pipeline/models.py @@ -140,14 +140,21 @@ class PipelineSettings(StrictModel): scoring_epsilon: float = 0.000001 metric_weights: dict[str, float] = Field(default_factory=lambda: {"final_response_avg_score": 0.6, "fake_rubric_score": 0.4}) metric_floors: dict[str, float] = Field(default_factory=dict) + candidate_validation: "CandidateValidationSettings" = Field(default_factory=lambda: CandidateValidationSettings()) + + +class CandidateValidationSettings(StrictModel): + scope: Literal["best_only", "accepted_rounds", "all"] = "accepted_rounds" class CandidateRecord(StrictModel): candidate_id: str prompts: dict[str, str] - source: Literal["fixture"] = "fixture" + source: Literal["fixture", "agent_optimizer"] = "fixture" generation_cost_usd: float = 0.0 round_index: int | None = None + optimizer_accepted: bool | None = None + optimizer_reason: str = "" class CandidateReport(StrictModel): @@ -162,7 +169,7 @@ class CandidateReport(StrictModel): class OptimizationReport(StrictModel): schema_version: str = "1.0" - mode: Literal["fake", "trace"] + mode: Literal["fake", "trace", "live"] seed: int selected_candidate_id: str | None = None candidates: list[CandidateReport] = Field(default_factory=list) @@ -171,5 +178,5 @@ class OptimizationReport(StrictModel): source_integrity: Literal["restored", "unknown"] = "restored" @classmethod - def empty(cls, *, mode: Literal["fake", "trace"], seed: int) -> "OptimizationReport": + def empty(cls, *, mode: Literal["fake", "trace", "live"], seed: int) -> "OptimizationReport": return cls(mode=mode, seed=seed) diff --git a/examples/optimization/eval_optimize_loop/pipeline/optimizer_backend.py b/examples/optimization/eval_optimize_loop/pipeline/optimizer_backend.py new file mode 100644 index 00000000..6fb6831d --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/optimizer_backend.py @@ -0,0 +1,181 @@ +"""Safe, auditable adapter around the public ``AgentOptimizer`` API.""" + +from __future__ import annotations + +import json +import tempfile +from pathlib import Path +from typing import Any, Awaitable, Callable, Protocol + +from trpc_agent_sdk.evaluation import AgentOptimizer, TargetPrompt + +from ..fake.fake_agent import FakeSupportAgent +from .config import canonical_sha256 +from .models import CandidateRecord, GateDecision + + +class PipelineExecutionError(RuntimeError): + """The optimizer ran unsuccessfully, so no safe decision can be produced.""" + + +class OptimizerBackend(Protocol): + async def generate_candidates( + self, + *, + baseline_prompts: dict[str, str], + train_dataset_path: Path, + validation_dataset_path: Path, + output_dir: Path, + ) -> list[CandidateRecord]: ... + + +def _runtime_config(raw_config: dict[str, Any]) -> dict[str, Any]: + """Return the SDK-only config without ever resolving secrets from env.""" + try: + payload = {name: raw_config[name] for name in ("evaluate", "optimize")} + except KeyError as exc: + raise PipelineExecutionError(f"live optimizer config is missing {exc.args[0]!r}") from exc + + def reject_literal_secrets(value: object, path: tuple[str, ...] = ()) -> None: + if isinstance(value, dict): + for key, nested in value.items(): + normalized = "".join(character for character in str(key).lower() if character.isalnum()) + is_secret = normalized == "key" or normalized.endswith("key") or any( + marker in normalized for marker in ("token", "secret", "password", "credential") + ) + if is_secret and isinstance(nested, str) and not (nested.startswith("${") and nested.endswith("}")): + raise PipelineExecutionError(f"runtime config refuses literal secret at {'.'.join((*path, str(key)))}") + reject_literal_secrets(nested, (*path, str(key))) + elif isinstance(value, list): + for index, nested in enumerate(value): + reject_literal_secrets(nested, (*path, str(index))) + + reject_literal_secrets(payload) + return payload + + +def _full_prompt_map(candidate: object, baseline_prompts: dict[str, str]) -> dict[str, str] | None: + if not isinstance(candidate, dict) or set(candidate) != set(baseline_prompts): + return None + if not all(isinstance(value, str) for value in candidate.values()): + return None + return dict(candidate) + + +class AgentOptimizerBackend: + """Generate proposals in a disposable prompt workspace. + + The concrete agent is deliberately injected through ``call_agent_factory`` for + production adaptation. The checked-in demo uses ``FakeSupportAgent`` so the + only live dependency in this mode is the optimizer reflection model itself. + """ + + def __init__( + self, + *, + raw_config: dict[str, Any], + candidate_scope: str = "accepted_rounds", + call_agent_factory: Callable[[TargetPrompt], Callable[[str], Awaitable[str]]] | None = None, + ) -> None: + if candidate_scope not in {"best_only", "accepted_rounds", "all"}: + raise ValueError(f"unsupported candidate scope: {candidate_scope}") + self._raw_config = raw_config + self._candidate_scope = candidate_scope + self._call_agent_factory = call_agent_factory or (lambda target: FakeSupportAgent(target).call_agent) + self.audit: dict[str, object] = {"duplicate_candidate_ids": {}, "skipped_candidate_ids": []} + + async def generate_candidates( + self, + *, + baseline_prompts: dict[str, str], + train_dataset_path: Path, + validation_dataset_path: Path, + output_dir: Path, + ) -> list[CandidateRecord]: + optimizer_dir = output_dir / "optimizer" / "agent_optimizer" + optimizer_dir.mkdir(parents=True, exist_ok=True) + runtime_path = optimizer_dir / "_optimizer.runtime.json" + runtime_path.write_text(json.dumps(_runtime_config(self._raw_config), ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + with tempfile.TemporaryDirectory(prefix="trpc-agent-issue91-live-") as temporary_dir: + prompt_dir = Path(temporary_dir) + target = TargetPrompt() + for name, content in baseline_prompts.items(): + path = prompt_dir / f"{name}.md" + path.write_text(content, encoding="utf-8") + target.add_path(name, str(path)) + result = await AgentOptimizer.optimize( + config_path=str(runtime_path), + call_agent=self._call_agent_factory(target), + target_prompt=target, + train_dataset_path=str(train_dataset_path), + validation_dataset_path=str(validation_dataset_path), + output_dir=str(optimizer_dir), + update_source=False, + verbose=0, + ) + if getattr(result, "status", None) != "SUCCEEDED": + raise PipelineExecutionError(f"AgentOptimizer finished with status {getattr(result, 'status', 'unknown')}") + records = self._extract_candidates(result, baseline_prompts) + (optimizer_dir / "candidate_extraction.json").write_text( + json.dumps(self.audit, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return records + + def _extract_candidates(self, result: object, baseline_prompts: dict[str, str]) -> list[CandidateRecord]: + entries: list[tuple[str, int | None, bool | None, str | None, object]] = [] + rounds = list(getattr(result, "rounds", []) or []) + if self._candidate_scope != "best_only": + for fallback_index, round_record in enumerate(rounds, start=1): + accepted = bool(getattr(round_record, "accepted", False)) + if self._candidate_scope == "accepted_rounds" and not accepted: + continue + round_index = int(getattr(round_record, "round", fallback_index)) + entries.append((f"round-{round_index:03d}", round_index, accepted, getattr(round_record, "acceptance_reason", None), getattr(round_record, "candidate_prompts", None))) + entries.append(("best", None, True, "OptimizeResult.best_prompts", getattr(result, "best_prompts", None))) + + retained: list[CandidateRecord] = [] + seen: dict[str, CandidateRecord] = {} + duplicates: dict[str, list[str]] = {} + skipped: list[str] = [] + for candidate_id, round_index, optimizer_accepted, optimizer_reason, prompts in entries: + full_prompts = _full_prompt_map(prompts, baseline_prompts) + if full_prompts is None: + skipped.append(candidate_id) + continue + digest = canonical_sha256(full_prompts) + if digest in seen: + duplicates.setdefault(seen[digest].candidate_id, []).append(candidate_id) + continue + record = CandidateRecord( + candidate_id=candidate_id, + source="agent_optimizer", + round_index=round_index, + prompts=full_prompts, + optimizer_accepted=optimizer_accepted, + optimizer_reason=optimizer_reason or "", + ) + seen[digest] = record + retained.append(record) + self.audit = {"duplicate_candidate_ids": duplicates, "skipped_candidate_ids": skipped} + return retained + + +async def write_back_after_gate( + target_prompt: TargetPrompt, + baseline_prompts: dict[str, str], + candidate_prompts: dict[str, str], + gate: GateDecision, +) -> bool: + """Opt-in source write-back with compare-and-swap style integrity checks.""" + if not gate.accepted: + return False + current = await target_prompt.read_all() + if canonical_sha256(current) != canonical_sha256(baseline_prompts): + raise PipelineExecutionError("source baseline changed before optional write-back") + if _full_prompt_map(candidate_prompts, baseline_prompts) is None: + raise PipelineExecutionError("optional write-back requires a complete prompt map") + await target_prompt.write_all(candidate_prompts) + if canonical_sha256(await target_prompt.read_all()) != canonical_sha256(candidate_prompts): + raise PipelineExecutionError("optional write-back candidate verification failed") + return True diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index bbcfe202..1f04fe98 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -28,6 +28,7 @@ from examples.optimization.eval_optimize_loop.pipeline.evaluator import evaluate_split from examples.optimization.eval_optimize_loop.pipeline.gate import evaluate_gate from examples.optimization.eval_optimize_loop.pipeline.models import CandidateReport, OptimizationReport, SplitReport +from examples.optimization.eval_optimize_loop.pipeline.optimizer_backend import AgentOptimizerBackend, PipelineExecutionError from examples.optimization.eval_optimize_loop.pipeline.attribution import attribute_case from examples.optimization.eval_optimize_loop.pipeline.normalization import normalize_eval_results from examples.optimization.eval_optimize_loop.pipeline.prompt_sandbox import PromptSandbox @@ -113,14 +114,99 @@ async def run_trace_pipeline(*, output_dir: Path) -> OptimizationReport: return report +async def run_live_pipeline(*, output_dir: Path) -> OptimizationReport: + """Run the public optimizer API, then independently evaluate its proposals.""" + config = load_pipeline_config(HERE / "optimizer.json", mode="live") + report = OptimizationReport.empty(mode="live", seed=config.pipeline.reproducibility.seed) + source_prompt_dir = HERE / "agent" / "prompts" + source_baseline = { + "system_prompt": (source_prompt_dir / "system.md").read_text(encoding="utf-8"), + "router_prompt": (source_prompt_dir / "router.md").read_text(encoding="utf-8"), + } + with tempfile.TemporaryDirectory(prefix="trpc-agent-issue91-regression-") as temporary_dir: + prompt_dir = Path(temporary_dir) + target = TargetPrompt() + for name, content in source_baseline.items(): + path = prompt_dir / f"{name}.md" + path.write_text(content, encoding="utf-8") + target.add_path(name, str(path)) + fake_agent = FakeSupportAgent(target) + evaluate = lambda path, split: evaluate_split( + path, + call_agent=fake_agent.call_agent, + split=split, + metric_weights=config.pipeline.metric_weights, + ) + report.baseline_train = await evaluate(config.pipeline.datasets.train_path, "train") + report.baseline_validation = await evaluate(config.pipeline.datasets.validation_path, "validation") + backend = AgentOptimizerBackend( + raw_config=config.raw, + candidate_scope=config.pipeline.candidate_validation.scope, + ) + candidates = await backend.generate_candidates( + baseline_prompts=source_baseline, + train_dataset_path=config.pipeline.datasets.train_path, + validation_dataset_path=config.pipeline.datasets.validation_path, + output_dir=output_dir, + ) + for candidate in candidates: + async with PromptSandbox(target, candidate.prompts): + train = await evaluate(config.pipeline.datasets.train_path, "train") + validation = await evaluate(config.pipeline.datasets.validation_path, "validation") + deltas = [ + compare_case( + baseline_case, + next(item for item in validation.cases if item.eval_id == baseline_case.eval_id), + epsilon=config.pipeline.scoring_epsilon, + critical_case_ids=set(config.pipeline.gate.critical_case_ids), + ) + for baseline_case in report.baseline_validation.cases + ] + decision = evaluate_gate( + report.baseline_validation, + validation, + settings=config.pipeline.gate, + case_deltas=deltas, + train_score_delta=train.aggregate_score - report.baseline_train.aggregate_score, + ) + report.candidates.append( + CandidateReport( + candidate_id=candidate.candidate_id, + accepted=decision.accepted, + reasons=decision.reasons, + train=train, + validation=validation, + gate=decision, + validation_case_deltas=deltas, + ) + ) + accepted = [candidate for candidate in report.candidates if candidate.accepted] + report.selected_candidate_id = accepted[0].candidate_id if accepted else None + write_reports(report, output_dir) + return report + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--mode", choices=["fake", "trace", "live"], default="fake") parser.add_argument("--output-dir", type=Path, default=HERE / "runs" / "manual") args = parser.parse_args() - if args.mode == "live": - parser.error("live mode is not implemented yet") - report = asyncio.run(run_trace_pipeline(output_dir=args.output_dir) if args.mode == "trace" else run_fake_pipeline(output_dir=args.output_dir)) + try: + if args.mode == "live": + # Preflight before constructing the coroutine/optimizer so missing + # credentials always return the documented input-error exit code. + load_pipeline_config(HERE / "optimizer.json", mode="live") + report = asyncio.run(run_live_pipeline(output_dir=args.output_dir)) + elif args.mode == "trace": + report = asyncio.run(run_trace_pipeline(output_dir=args.output_dir)) + else: + report = asyncio.run(run_fake_pipeline(output_dir=args.output_dir)) + except ValueError as exc: + print(f"Input error: {exc}", file=sys.stderr) + return 2 + except PipelineExecutionError as exc: + print(f"Pipeline execution error: {exc}", file=sys.stderr) + return 3 print(f"Decision: {'ACCEPT' if report.selected_candidate_id else 'REJECT'}") print(f"Selected candidate: {report.selected_candidate_id or 'none'}") print(f"JSON report: {(args.output_dir / 'optimization_report.json').resolve()}") diff --git a/tests/examples/optimization/eval_optimize_loop/test_live_backend.py b/tests/examples/optimization/eval_optimize_loop/test_live_backend.py new file mode 100644 index 00000000..fb57098c --- /dev/null +++ b/tests/examples/optimization/eval_optimize_loop/test_live_backend.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from examples.optimization.eval_optimize_loop.pipeline.models import CandidateRecord, GateDecision, GateRuleResult +from examples.optimization.eval_optimize_loop.pipeline.optimizer_backend import ( + AgentOptimizerBackend, + PipelineExecutionError, + write_back_after_gate, +) +from examples.optimization.eval_optimize_loop.run_pipeline import main, run_live_pipeline +from trpc_agent_sdk.evaluation import AgentOptimizer, TargetPrompt + + +PROMPT_DIR = Path(__file__).resolve().parents[4] / "examples" / "optimization" / "eval_optimize_loop" / "agent" / "prompts" +EXAMPLE_DIR = PROMPT_DIR.parents[1] +DATASET = EXAMPLE_DIR / "train.evalset.json" + + +def _baseline() -> dict[str, str]: + return { + "system_prompt": (PROMPT_DIR / "system.md").read_text(encoding="utf-8"), + "router_prompt": (PROMPT_DIR / "router.md").read_text(encoding="utf-8"), + } + + +def _result(*, status: str = "SUCCEEDED", rounds: list[object] | None = None, best: dict[str, str] | None = None) -> SimpleNamespace: + return SimpleNamespace(status=status, best_prompts=best or _baseline(), rounds=rounds or []) + + +@pytest.mark.asyncio +async def test_live_backend_uses_sanitized_config_temporary_target_and_archives_artifacts( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + observed: dict[str, object] = {} + + async def fake_optimize(**kwargs): + observed.update(kwargs) + observed["target_read"] = await kwargs["target_prompt"].read_all() + artifact = Path(kwargs["output_dir"]) / "sdk-artifact.txt" + artifact.parent.mkdir(parents=True, exist_ok=True) + artifact.write_text("created by SDK", encoding="utf-8") + return _result() + + monkeypatch.setattr(AgentOptimizer, "optimize", fake_optimize) + backend = AgentOptimizerBackend( + raw_config={ + "evaluate": {"metrics": [], "num_runs": 1}, + "optimize": {"algorithm": {"name": "gepa_reflective", "reflection_lm": {"api_key": "${TRPC_AGENT_API_KEY}"}}}, + "pipeline": {"must_not": "reach SDK"}, + }, + candidate_scope="best_only", + ) + + candidates = await backend.generate_candidates( + baseline_prompts=_baseline(), train_dataset_path=DATASET, validation_dataset_path=EXAMPLE_DIR / "val.evalset.json", output_dir=tmp_path + ) + + assert observed["update_source"] is False + assert observed["verbose"] == 0 + assert Path(observed["output_dir"]).is_relative_to(tmp_path / "optimizer") + assert observed["target_read"] == _baseline() + runtime_config = Path(observed["config_path"]) + payload = json.loads(runtime_config.read_text(encoding="utf-8")) + assert set(payload) == {"evaluate", "optimize"} + assert payload["optimize"]["algorithm"]["reflection_lm"]["api_key"] == "${TRPC_AGENT_API_KEY}" + assert (Path(observed["output_dir"]) / "sdk-artifact.txt").is_file() + assert candidates[0].source == "agent_optimizer" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("scope", "expected_ids", "expected_duplicates", "expected_skipped"), + [ + ("best_only", ["best"], {}, []), + ("accepted_rounds", ["round-001", "best"], {"round-001": ["round-003"]}, ["round-004"]), + ("all", ["round-001", "round-002", "best"], {"round-001": ["round-003"]}, ["round-004"]), + ], +) +async def test_candidate_extraction_respects_scope_and_deduplicates( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + scope: str, + expected_ids: list[str], + expected_duplicates: dict[str, list[str]], + expected_skipped: list[str], +) -> None: + baseline = _baseline() + candidate_one = {**baseline, "system_prompt": "GENERAL_FIX"} + candidate_two = {**baseline, "system_prompt": "OVERFIT"} + rounds = [ + SimpleNamespace(round=1, accepted=True, candidate_prompts=candidate_one, acceptance_reason="better"), + SimpleNamespace(round=2, accepted=False, candidate_prompts=candidate_two, acceptance_reason="explore"), + SimpleNamespace(round=3, accepted=True, candidate_prompts=candidate_one, acceptance_reason="duplicate"), + SimpleNamespace(round=4, accepted=True, candidate_prompts={"system_prompt": "partial"}, acceptance_reason="partial"), + ] + + async def fake_optimize(**_kwargs): + return _result(rounds=rounds, best=baseline) + + monkeypatch.setattr(AgentOptimizer, "optimize", fake_optimize) + backend = AgentOptimizerBackend(raw_config={"evaluate": {}, "optimize": {}}, candidate_scope=scope) + records = await backend.generate_candidates( + baseline_prompts=baseline, train_dataset_path=DATASET, validation_dataset_path=EXAMPLE_DIR / "val.evalset.json", output_dir=tmp_path + ) + + assert [record.candidate_id for record in records] == expected_ids + assert backend.audit["duplicate_candidate_ids"] == expected_duplicates + assert backend.audit["skipped_candidate_ids"] == expected_skipped + + +def test_live_cli_missing_environment_returns_two_without_constructing_optimizer( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + for name in ("TRPC_AGENT_API_KEY", "TRPC_AGENT_BASE_URL", "TRPC_AGENT_MODEL_NAME"): + monkeypatch.delenv(name, raising=False) + + def should_not_run(*_args, **_kwargs): + raise AssertionError("optimizer must not be constructed") + + monkeypatch.setattr("examples.optimization.eval_optimize_loop.run_pipeline.run_live_pipeline", should_not_run) + monkeypatch.setattr("sys.argv", ["run_pipeline.py", "--mode", "live", "--output-dir", str(tmp_path)]) + assert main() == 2 + assert "TRPC_AGENT_API_KEY" in capsys.readouterr().err + + +@pytest.mark.asyncio +async def test_succeeded_optimizer_candidate_is_independently_evaluated_and_can_be_gate_rejected( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + for name, value in { + "TRPC_AGENT_API_KEY": "unused-in-test", + "TRPC_AGENT_BASE_URL": "https://unused.invalid", + "TRPC_AGENT_MODEL_NAME": "unused", + }.items(): + monkeypatch.setenv(name, value) + baseline = _baseline() + overfit = {**baseline, "system_prompt": "OVERFIT"} + + async def fake_optimize(**_kwargs): + return _result(best=overfit) + + monkeypatch.setattr(AgentOptimizer, "optimize", fake_optimize) + report = await run_live_pipeline(output_dir=tmp_path) + assert report.mode == "live" + assert report.candidates[0].candidate_id == "best" + assert report.candidates[0].accepted is False + assert report.candidates[0].train is not None + assert report.candidates[0].validation is not None + assert report.selected_candidate_id is None + + +@pytest.mark.asyncio +async def test_failed_optimizer_preserves_baseline_and_raises_pipeline_execution_error( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + baseline = _baseline() + + async def fake_optimize(**_kwargs): + return _result(status="FAILED", best={**baseline, "system_prompt": "OVERFIT"}) + + monkeypatch.setattr(AgentOptimizer, "optimize", fake_optimize) + backend = AgentOptimizerBackend(raw_config={"evaluate": {}, "optimize": {}}, candidate_scope="best_only") + with pytest.raises(PipelineExecutionError, match="FAILED"): + await backend.generate_candidates( + baseline_prompts=baseline, train_dataset_path=DATASET, validation_dataset_path=EXAMPLE_DIR / "val.evalset.json", output_dir=tmp_path + ) + assert _baseline() == baseline + + +@pytest.mark.asyncio +async def test_optional_write_back_requires_gate_and_detects_baseline_conflict(tmp_path: Path) -> None: + source = tmp_path / "system.md" + source.write_text("BASELINE", encoding="utf-8") + target = TargetPrompt().add_path("system_prompt", str(source)) + baseline = {"system_prompt": "BASELINE"} + candidate = {"system_prompt": "CANDIDATE"} + accepted = GateDecision(accepted=True, risk_level="low", rules=[], reasons=[]) + assert await write_back_after_gate(target, baseline, candidate, accepted) is True + assert source.read_text(encoding="utf-8") == "CANDIDATE" + source.write_text("CHANGED_EXTERNALLY", encoding="utf-8") + with pytest.raises(PipelineExecutionError, match="baseline changed"): + await write_back_after_gate(target, baseline, candidate, accepted) From 9cf5ac409b8bc7091d08250d826d6cf30ca1229e Mon Sep 17 00:00:00 2001 From: shiquan_zhan Date: Sat, 11 Jul 2026 13:34:33 +0800 Subject: [PATCH 11/15] fix(eval): harden live optimizer failure and writeback --- .../eval_optimize_loop/optimizer.json | 1 + .../eval_optimize_loop/pipeline/models.py | 1 + .../pipeline/optimizer_backend.py | 23 +++-- .../eval_optimize_loop/run_pipeline.py | 15 ++- .../eval_optimize_loop/test_live_backend.py | 94 ++++++++++++++++++- 5 files changed, 122 insertions(+), 12 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/optimizer.json b/examples/optimization/eval_optimize_loop/optimizer.json index ba9dfeb3..eaebd61c 100644 --- a/examples/optimization/eval_optimize_loop/optimizer.json +++ b/examples/optimization/eval_optimize_loop/optimizer.json @@ -22,6 +22,7 @@ }, "pipeline": { "reproducibility": {"seed": 42}, + "write_back_when_accepted": false, "gate": {"min_validation_score_delta": 0.05, "critical_case_ids": ["val_refund_critical"]} } } diff --git a/examples/optimization/eval_optimize_loop/pipeline/models.py b/examples/optimization/eval_optimize_loop/pipeline/models.py index 535744e8..725e40c7 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/models.py +++ b/examples/optimization/eval_optimize_loop/pipeline/models.py @@ -141,6 +141,7 @@ class PipelineSettings(StrictModel): metric_weights: dict[str, float] = Field(default_factory=lambda: {"final_response_avg_score": 0.6, "fake_rubric_score": 0.4}) metric_floors: dict[str, float] = Field(default_factory=dict) candidate_validation: "CandidateValidationSettings" = Field(default_factory=lambda: CandidateValidationSettings()) + write_back_when_accepted: bool = False class CandidateValidationSettings(StrictModel): diff --git a/examples/optimization/eval_optimize_loop/pipeline/optimizer_backend.py b/examples/optimization/eval_optimize_loop/pipeline/optimizer_backend.py index 6fb6831d..1af86050 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/optimizer_backend.py +++ b/examples/optimization/eval_optimize_loop/pipeline/optimizer_backend.py @@ -104,16 +104,19 @@ async def generate_candidates( path = prompt_dir / f"{name}.md" path.write_text(content, encoding="utf-8") target.add_path(name, str(path)) - result = await AgentOptimizer.optimize( - config_path=str(runtime_path), - call_agent=self._call_agent_factory(target), - target_prompt=target, - train_dataset_path=str(train_dataset_path), - validation_dataset_path=str(validation_dataset_path), - output_dir=str(optimizer_dir), - update_source=False, - verbose=0, - ) + try: + result = await AgentOptimizer.optimize( + config_path=str(runtime_path), + call_agent=self._call_agent_factory(target), + target_prompt=target, + train_dataset_path=str(train_dataset_path), + validation_dataset_path=str(validation_dataset_path), + output_dir=str(optimizer_dir), + update_source=False, + verbose=0, + ) + except Exception as exc: + raise PipelineExecutionError("AgentOptimizer.optimize failed") from exc if getattr(result, "status", None) != "SUCCEEDED": raise PipelineExecutionError(f"AgentOptimizer finished with status {getattr(result, 'status', 'unknown')}") records = self._extract_candidates(result, baseline_prompts) diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index 1f04fe98..9e6ff5da 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -28,7 +28,11 @@ from examples.optimization.eval_optimize_loop.pipeline.evaluator import evaluate_split from examples.optimization.eval_optimize_loop.pipeline.gate import evaluate_gate from examples.optimization.eval_optimize_loop.pipeline.models import CandidateReport, OptimizationReport, SplitReport -from examples.optimization.eval_optimize_loop.pipeline.optimizer_backend import AgentOptimizerBackend, PipelineExecutionError +from examples.optimization.eval_optimize_loop.pipeline.optimizer_backend import ( + AgentOptimizerBackend, + PipelineExecutionError, + write_back_after_gate, +) from examples.optimization.eval_optimize_loop.pipeline.attribution import attribute_case from examples.optimization.eval_optimize_loop.pipeline.normalization import normalize_eval_results from examples.optimization.eval_optimize_loop.pipeline.prompt_sandbox import PromptSandbox @@ -123,6 +127,9 @@ async def run_live_pipeline(*, output_dir: Path) -> OptimizationReport: "system_prompt": (source_prompt_dir / "system.md").read_text(encoding="utf-8"), "router_prompt": (source_prompt_dir / "router.md").read_text(encoding="utf-8"), } + source_target = TargetPrompt() + for name, filename in (("system_prompt", "system.md"), ("router_prompt", "router.md")): + source_target.add_path(name, str(source_prompt_dir / filename)) with tempfile.TemporaryDirectory(prefix="trpc-agent-issue91-regression-") as temporary_dir: prompt_dir = Path(temporary_dir) target = TargetPrompt() @@ -182,6 +189,12 @@ async def run_live_pipeline(*, output_dir: Path) -> OptimizationReport: ) accepted = [candidate for candidate in report.candidates if candidate.accepted] report.selected_candidate_id = accepted[0].candidate_id if accepted else None + if config.pipeline.write_back_when_accepted and report.selected_candidate_id: + selected_record = next(candidate for candidate in candidates if candidate.candidate_id == report.selected_candidate_id) + selected_report = next(candidate for candidate in accepted if candidate.candidate_id == report.selected_candidate_id) + if selected_report.gate is None: # Defensive: accepted reports always carry a GateDecision. + raise PipelineExecutionError("selected candidate is missing its gate decision") + await write_back_after_gate(source_target, source_baseline, selected_record.prompts, selected_report.gate) write_reports(report, output_dir) return report diff --git a/tests/examples/optimization/eval_optimize_loop/test_live_backend.py b/tests/examples/optimization/eval_optimize_loop/test_live_backend.py index fb57098c..80ad1280 100644 --- a/tests/examples/optimization/eval_optimize_loop/test_live_backend.py +++ b/tests/examples/optimization/eval_optimize_loop/test_live_backend.py @@ -1,6 +1,5 @@ from __future__ import annotations -import asyncio import json from pathlib import Path from types import SimpleNamespace @@ -173,6 +172,40 @@ async def fake_optimize(**_kwargs): assert _baseline() == baseline +@pytest.mark.asyncio +async def test_optimizer_exception_is_wrapped_with_original_cause(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + async def fake_optimize(**_kwargs): + raise RuntimeError("transport failed") + + monkeypatch.setattr(AgentOptimizer, "optimize", fake_optimize) + backend = AgentOptimizerBackend(raw_config={"evaluate": {}, "optimize": {}}, candidate_scope="best_only") + with pytest.raises(PipelineExecutionError, match="AgentOptimizer.optimize failed") as exc_info: + await backend.generate_candidates( + baseline_prompts=_baseline(), + train_dataset_path=DATASET, + validation_dataset_path=EXAMPLE_DIR / "val.evalset.json", + output_dir=tmp_path, + ) + assert isinstance(exc_info.value.__cause__, RuntimeError) + + +def test_live_cli_returns_three_when_optimizer_raises(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + for name, value in { + "TRPC_AGENT_API_KEY": "unused-in-test", + "TRPC_AGENT_BASE_URL": "https://unused.invalid", + "TRPC_AGENT_MODEL_NAME": "unused", + }.items(): + monkeypatch.setenv(name, value) + + async def fake_optimize(**_kwargs): + raise RuntimeError("transport failed") + + monkeypatch.setattr(AgentOptimizer, "optimize", fake_optimize) + monkeypatch.setattr("sys.argv", ["run_pipeline.py", "--mode", "live", "--output-dir", str(tmp_path)]) + assert main() == 3 + assert "AgentOptimizer.optimize failed" in capsys.readouterr().err + + @pytest.mark.asyncio async def test_optional_write_back_requires_gate_and_detects_baseline_conflict(tmp_path: Path) -> None: source = tmp_path / "system.md" @@ -186,3 +219,62 @@ async def test_optional_write_back_requires_gate_and_detects_baseline_conflict(t source.write_text("CHANGED_EXTERNALLY", encoding="utf-8") with pytest.raises(PipelineExecutionError, match="baseline changed"): await write_back_after_gate(target, baseline, candidate, accepted) + + +@pytest.mark.asyncio +async def test_live_pipeline_does_not_write_back_by_default_or_without_gate_winner( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + for name, value in { + "TRPC_AGENT_API_KEY": "unused-in-test", + "TRPC_AGENT_BASE_URL": "https://unused.invalid", + "TRPC_AGENT_MODEL_NAME": "unused", + }.items(): + monkeypatch.setenv(name, value) + general_fix = {**_baseline(), "system_prompt": "GENERAL_FIX"} + + async def fake_optimize(**_kwargs): + return _result(best=general_fix) + + async def should_not_write(*_args, **_kwargs): + raise AssertionError("write-back must remain disabled by default") + + monkeypatch.setattr(AgentOptimizer, "optimize", fake_optimize) + monkeypatch.setattr("examples.optimization.eval_optimize_loop.run_pipeline.write_back_after_gate", should_not_write) + report = await run_live_pipeline(output_dir=tmp_path) + assert report.selected_candidate_id == "best" + + +@pytest.mark.asyncio +async def test_live_pipeline_writes_only_selected_gate_winner_when_explicitly_enabled( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from examples.optimization.eval_optimize_loop.pipeline.config import load_pipeline_config + + for name, value in { + "TRPC_AGENT_API_KEY": "unused-in-test", + "TRPC_AGENT_BASE_URL": "https://unused.invalid", + "TRPC_AGENT_MODEL_NAME": "unused", + }.items(): + monkeypatch.setenv(name, value) + general_fix = {**_baseline(), "system_prompt": "GENERAL_FIX"} + + async def fake_optimize(**_kwargs): + return _result(best=general_fix) + + config = load_pipeline_config(EXAMPLE_DIR / "optimizer.json", mode="live") + enabled_pipeline = config.pipeline.model_copy(update={"write_back_when_accepted": True}) + enabled_config = config.model_copy(update={"pipeline": enabled_pipeline}) + observed: dict[str, object] = {} + + async def fake_write_back(target, baseline, candidate, gate): + observed.update({"target": target, "baseline": baseline, "candidate": candidate, "gate": gate}) + return True + + monkeypatch.setattr(AgentOptimizer, "optimize", fake_optimize) + monkeypatch.setattr("examples.optimization.eval_optimize_loop.run_pipeline.load_pipeline_config", lambda *_args, **_kwargs: enabled_config) + monkeypatch.setattr("examples.optimization.eval_optimize_loop.run_pipeline.write_back_after_gate", fake_write_back) + report = await run_live_pipeline(output_dir=tmp_path) + assert report.selected_candidate_id == "best" + assert observed["candidate"] == general_fix + assert observed["gate"].accepted is True From d84e51b384e41de07e4d2f042e81b4711c101706 Mon Sep 17 00:00:00 2001 From: shiquan_zhan Date: Sat, 11 Jul 2026 13:51:44 +0800 Subject: [PATCH 12/15] feat(eval): add auditable gate and reports --- .../optimization/eval_optimize_loop/DESIGN.md | 9 + .../optimization/eval_optimize_loop/README.md | 27 + .../eval_optimize_loop/pipeline/gate.py | 94 +- .../eval_optimize_loop/pipeline/models.py | 25 +- .../pipeline/optimizer_backend.py | 17 +- .../eval_optimize_loop/pipeline/reporter.py | 54 +- .../eval_optimize_loop/run_pipeline.py | 47 +- .../audit/candidate_reports.json | 1320 +++++++++++++++++ .../sample_output/audit/gate_decisions.json | 263 ++++ .../audit/normalized_reports.json | 1167 +++++++++++++++ .../sample_output/audit/raw_reports.json | 292 ++++ .../sample_output/environment.snapshot.json | 1 + .../sample_output/input.snapshot.json | 1 + .../sample_output/optimization_report.json | 1125 +++++++++----- .../sample_output/optimization_report.md | 39 +- .../test_report_and_gate_matrix.py | 103 ++ 16 files changed, 4217 insertions(+), 367 deletions(-) create mode 100644 examples/optimization/eval_optimize_loop/DESIGN.md create mode 100644 examples/optimization/eval_optimize_loop/README.md create mode 100644 examples/optimization/eval_optimize_loop/sample_output/audit/candidate_reports.json create mode 100644 examples/optimization/eval_optimize_loop/sample_output/audit/gate_decisions.json create mode 100644 examples/optimization/eval_optimize_loop/sample_output/audit/normalized_reports.json create mode 100644 examples/optimization/eval_optimize_loop/sample_output/audit/raw_reports.json create mode 100644 examples/optimization/eval_optimize_loop/sample_output/environment.snapshot.json create mode 100644 examples/optimization/eval_optimize_loop/sample_output/input.snapshot.json create mode 100644 tests/examples/optimization/eval_optimize_loop/test_report_and_gate_matrix.py diff --git a/examples/optimization/eval_optimize_loop/DESIGN.md b/examples/optimization/eval_optimize_loop/DESIGN.md new file mode 100644 index 00000000..0a52b875 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/DESIGN.md @@ -0,0 +1,9 @@ +# Design notes + +The loop has four boundaries. Evaluation normalizes SDK results into stable case snapshots. Attribution reads actual intermediate trace data first and uses expected conversation data only as a reference. Candidate generation is either the checked-in fixture backend or a thin live adapter around the public `AgentOptimizer` API. Acceptance is owned by the Gate, never by an optimizer round status or summary. + +`PromptSandbox` isolates every candidate: source prompt files stay untouched during evaluation. The live adapter builds a temporary target, passes an SDK-only runtime configuration with environment placeholders intact, uses `update_source=False` and `verbose=0`, and archives optimizer artifacts. It extracts complete prompt maps only, de-duplicates their canonical digest, then independently runs full train and validation evaluation for each proposal. Optional write-back remains off by default and uses a baseline/candidate digest check after Gate success. + +The Gate rejects incomplete evidence, unavailable validation deltas, new hard failures, critical or excessive regressions, metric-floor misses, train-up/validation-down overfit, configured generalization-gap, cost, duration, and tie-policy violations. Unknown cost is recorded as a warning. It ranks only independently evaluated and accepted candidates by hard failures, critical regressions, validation pass rate, validation score, cost, duration, and candidate id, making selection stable. + +Every path writes a Pydantic-readable `OptimizationReport` plus secret-free input, environment, raw, normalized, candidate, and Gate artifacts. Fake and trace runs are deterministic and suitable for regression tests. Existing third-party `LangChainPendingDeprecationWarning` and local `requests` dependency-compatibility warnings may appear during test execution; they are external warnings rather than pipeline decisions. diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md new file mode 100644 index 00000000..5cf4d899 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/README.md @@ -0,0 +1,27 @@ +# Issue #91: Auditable evaluation and prompt optimization loop + +This example demonstrates an evaluation-first prompt-optimization loop around the public `AgentOptimizer` API. It keeps the checked-in fake and trace paths deterministic and credential-free, while live mode is an opt-in adapter with independent acceptance checks. + +## Run + +From the repository root: + +```text +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake --output-dir examples/optimization/eval_optimize_loop/sample_output +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode trace --output-dir .tmp/issue-91-trace +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode live --output-dir .tmp/issue-91-live +``` + +`fake` uses checked-in candidates and a local rubric; it needs no key and makes no network calls. `trace` evaluates the checked-in recorded conversation only. `live` validates `TRPC_AGENT_API_KEY`, `TRPC_AGENT_BASE_URL`, and `TRPC_AGENT_MODEL_NAME` before constructing an optimizer. It exits with code `2` if any are absent, and does not start network work in that case. + +Live mode writes candidates to a temporary `TargetPrompt` workspace, passes only the SDK `evaluate`/`optimize` configuration to `AgentOptimizer.optimize`, and always uses `update_source=False`. Source write-back is disabled by default; it is considered only after a Gate-approved winner and guarded by prompt-digest checks. + +## Outputs + +Each run writes `optimization_report.json`, `optimization_report.md`, and secret-free audit evidence: + +- `input.snapshot.json` and `environment.snapshot.json` +- `audit/raw_reports.json` and `audit/normalized_reports.json` +- `audit/candidate_reports.json` and `audit/gate_decisions.json` + +The JSON report is Pydantic-readable as `OptimizationReport` and is the source of truth for the selected candidate. diff --git a/examples/optimization/eval_optimize_loop/pipeline/gate.py b/examples/optimization/eval_optimize_loop/pipeline/gate.py index adf4c159..5239fa61 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/gate.py +++ b/examples/optimization/eval_optimize_loop/pipeline/gate.py @@ -1,29 +1,97 @@ from __future__ import annotations -from .models import CaseDelta, GateDecision, GateRuleResult, GateSettings, SplitReport +from math import inf +from typing import Iterable + +from .models import CandidateReport, CaseDelta, GateDecision, GateRuleResult, GateSettings, SplitReport + + +def _rule(rule: str, passed: bool, actual: object, expected: object, reason: str) -> GateRuleResult: + return GateRuleResult(rule=rule, passed=passed, actual=actual, expected=expected, reason=reason) + + +def _complete(baseline: SplitReport | None, candidate: SplitReport | None) -> bool: + if baseline is None or candidate is None or not baseline.cases or not candidate.cases: + return False + baseline_ids = [case.eval_id for case in baseline.cases] + candidate_ids = [case.eval_id for case in candidate.cases] + return len(baseline_ids) == len(set(baseline_ids)) and len(candidate_ids) == len(set(candidate_ids)) and set(baseline_ids) == set(candidate_ids) def evaluate_gate( - baseline: SplitReport, - candidate: SplitReport, + baseline: SplitReport | None, + candidate: SplitReport | None, *, settings: GateSettings, case_deltas: list[CaseDelta], train_score_delta: float | None = None, + metric_floors: dict[str, float] | None = None, + generation_cost_usd: float | None = None, + duration_seconds: float | None = None, + epsilon: float = 0.000001, ) -> GateDecision: - score_delta = candidate.aggregate_score - baseline.aggregate_score - pass_rate_delta = candidate.pass_rate - baseline.pass_rate + """Apply only independently measured train/validation evidence to a proposal.""" + complete = _complete(baseline, candidate) + score_delta = candidate.aggregate_score - baseline.aggregate_score if complete and baseline and candidate else None + pass_rate_delta = candidate.pass_rate - baseline.pass_rate if complete and baseline and candidate else None new_hard_fails = sum(delta.hard_fail_added for delta in case_deltas) regressions = sum(delta.transition == "REGRESSION" for delta in case_deltas) critical_regression = any(delta.critical and delta.transition == "REGRESSION" for delta in case_deltas) - overfit = bool(settings.reject_when_train_improves_but_validation_declines and train_score_delta is not None and train_score_delta > 0 and score_delta < 0) + overfit = bool( + settings.reject_when_train_improves_but_validation_declines + and train_score_delta is not None + and train_score_delta > epsilon + and (score_delta is None or score_delta < -epsilon) + ) + generalization_gap = train_score_delta - score_delta if train_score_delta is not None and score_delta is not None else None rules = [ - GateRuleResult(rule="validation_score_improved", passed=score_delta >= settings.min_validation_score_delta, actual=score_delta, expected=settings.min_validation_score_delta, reason="validation aggregate score must improve"), - GateRuleResult(rule="validation_pass_rate_not_worse", passed=pass_rate_delta >= settings.min_validation_pass_rate_delta, actual=pass_rate_delta, expected=settings.min_validation_pass_rate_delta, reason="validation pass rate must not decline"), - GateRuleResult(rule="new_hard_fails", passed=new_hard_fails <= settings.max_new_hard_fails, actual=new_hard_fails, expected=settings.max_new_hard_fails, reason="new hard failures are not allowed"), - GateRuleResult(rule="validation_regressions", passed=regressions <= settings.max_validation_regressions, actual=regressions, expected=settings.max_validation_regressions, reason="validation regressions exceed the limit"), - GateRuleResult(rule="no_critical_regression", passed=settings.allow_critical_case_regression or not critical_regression, actual=critical_regression, expected=False, reason="critical validation cases must not regress"), - GateRuleResult(rule="no_overfit", passed=not overfit, actual=overfit, expected=False, reason="train improvement with validation decline is rejected"), + _rule("evaluation_complete", complete, None if not complete else "complete", "complete", "independent validation evaluation is incomplete"), + _rule("validation_score_delta_available", score_delta is not None, score_delta, "number", "validation aggregate score delta is required"), + _rule("validation_pass_rate_delta_available", pass_rate_delta is not None, pass_rate_delta, "number", "validation pass-rate delta is required"), + _rule("validation_score_improved", score_delta is not None and score_delta >= settings.min_validation_score_delta, score_delta, settings.min_validation_score_delta, "validation aggregate score must improve"), + _rule("validation_pass_rate_not_worse", pass_rate_delta is not None and pass_rate_delta >= settings.min_validation_pass_rate_delta, pass_rate_delta, settings.min_validation_pass_rate_delta, "validation pass rate must not decline"), + _rule("new_hard_fails", new_hard_fails <= settings.max_new_hard_fails, new_hard_fails, settings.max_new_hard_fails, "new hard failures are not allowed"), + _rule("validation_regressions", regressions <= settings.max_validation_regressions, regressions, settings.max_validation_regressions, "validation regressions exceed the limit"), + _rule("no_critical_regression", settings.allow_critical_case_regression or not critical_regression, critical_regression, False, "critical validation cases must not regress"), + _rule("no_overfit", not overfit, overfit, False, "train improvement with validation decline is rejected"), ] + for metric_name, floor in sorted((metric_floors or {}).items()): + observed = min((case.metric_scores.get(metric_name, float("-inf")) for case in candidate.cases), default=float("-inf")) if candidate else float("-inf") + rules.append(_rule(f"metric_floor:{metric_name}", observed >= floor, observed, floor, f"metric {metric_name} is below its validation floor")) + if settings.max_generalization_gap is not None: + rules.append(_rule("generalization_gap", generalization_gap is not None and generalization_gap <= settings.max_generalization_gap, generalization_gap, settings.max_generalization_gap, "train/validation generalization gap exceeds the limit")) + if settings.max_generation_cost_usd is not None: + rules.append(_rule("generation_cost_budget", generation_cost_usd is not None and generation_cost_usd <= settings.max_generation_cost_usd, generation_cost_usd, settings.max_generation_cost_usd, "generation cost exceeds the budget")) + if settings.max_duration_seconds is not None: + rules.append(_rule("duration_budget", duration_seconds is not None and duration_seconds <= settings.max_duration_seconds, duration_seconds, settings.max_duration_seconds, "generation duration exceeds the budget")) + tied = score_delta is not None and pass_rate_delta is not None and abs(score_delta) <= epsilon and abs(pass_rate_delta) <= epsilon + rules.append(_rule("tie_policy", not (settings.tie_policy == "reject" and tied), tied, False, "tie policy rejects a non-improving validation outcome")) + warnings = ["generation cost is unknown"] if generation_cost_usd is None else [] failed = [rule.reason for rule in rules if not rule.passed] - return GateDecision(accepted=not failed, risk_level="high" if critical_regression or overfit else ("medium" if failed else "low"), rules=rules, reasons=failed or ["candidate passed all independent gate rules"]) + risk_level = "high" if critical_regression or overfit else ("medium" if failed else "low") + return GateDecision(accepted=not failed, risk_level=risk_level, rules=rules, reasons=failed or ["candidate passed all independent gate rules"], warnings=warnings) + + +def select_winner(candidates: Iterable[CandidateReport]) -> str | None: + """Choose only Gate-approved, independently evaluated records with a stable key.""" + eligible = [candidate for candidate in candidates if candidate.accepted and candidate.independently_evaluated and candidate.train and candidate.validation] + if not eligible: + return None + + def rank(candidate: CandidateReport) -> tuple[float, ...] | tuple[float, float, float, float, float, float, str]: + gate = candidate.gate + rules = gate.rules if gate else [] + failures = {rule.rule: rule.actual for rule in rules} + hard_fails = float(failures.get("new_hard_fails", 0) or 0) + critical = 1.0 if failures.get("no_critical_regression") is True else 0.0 + return ( + hard_fails, + critical, + -candidate.validation.pass_rate, + -candidate.validation.aggregate_score, + candidate.generation_cost_usd if candidate.generation_cost_usd is not None else inf, + candidate.duration_seconds if candidate.duration_seconds is not None else inf, + candidate.candidate_id, + ) + + return min(eligible, key=rank).candidate_id diff --git a/examples/optimization/eval_optimize_loop/pipeline/models.py b/examples/optimization/eval_optimize_loop/pipeline/models.py index 725e40c7..79c0943c 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/models.py +++ b/examples/optimization/eval_optimize_loop/pipeline/models.py @@ -106,6 +106,9 @@ class GateSettings(StrictModel): critical_case_ids: list[str] = Field(default_factory=list) allow_critical_case_regression: bool = False reject_when_train_improves_but_validation_declines: bool = True + max_generalization_gap: float | None = None + max_generation_cost_usd: float | None = None + max_duration_seconds: float | None = None tie_policy: Literal["reject"] = "reject" @@ -122,6 +125,7 @@ class GateDecision(StrictModel): risk_level: Literal["low", "medium", "high"] rules: list[GateRuleResult] reasons: list[str] + warnings: list[str] = Field(default_factory=list) class ReproducibilitySettings(StrictModel): @@ -152,7 +156,8 @@ class CandidateRecord(StrictModel): candidate_id: str prompts: dict[str, str] source: Literal["fixture", "agent_optimizer"] = "fixture" - generation_cost_usd: float = 0.0 + generation_cost_usd: float | None = None + duration_seconds: float | None = None round_index: int | None = None optimizer_accepted: bool | None = None optimizer_reason: str = "" @@ -166,6 +171,20 @@ class CandidateReport(StrictModel): validation: SplitReport | None = None gate: GateDecision | None = None validation_case_deltas: list[CaseDelta] = Field(default_factory=list) + independently_evaluated: bool = False + source: Literal["fixture", "agent_optimizer"] = "fixture" + generation_cost_usd: float | None = None + duration_seconds: float | None = None + + +class AuditReferences(StrictModel): + config_snapshot_path: Path = Path("input.snapshot.json") + input_snapshot_path: Path = Path("input.snapshot.json") + environment_snapshot_path: Path = Path("environment.snapshot.json") + raw_reports_path: Path = Path("audit/raw_reports.json") + normalized_reports_path: Path = Path("audit/normalized_reports.json") + candidate_reports_path: Path = Path("audit/candidate_reports.json") + gate_decisions_path: Path = Path("audit/gate_decisions.json") class OptimizationReport(StrictModel): @@ -177,6 +196,10 @@ class OptimizationReport(StrictModel): baseline_train: SplitReport | None = None baseline_validation: SplitReport | None = None source_integrity: Literal["restored", "unknown"] = "restored" + run_metadata: dict[str, Any] = Field(default_factory=dict) + audit_references: AuditReferences = Field(default_factory=AuditReferences) + attribution_summary: dict[str, int] = Field(default_factory=dict) + warnings: list[str] = Field(default_factory=list) @classmethod def empty(cls, *, mode: Literal["fake", "trace", "live"], seed: int) -> "OptimizationReport": diff --git a/examples/optimization/eval_optimize_loop/pipeline/optimizer_backend.py b/examples/optimization/eval_optimize_loop/pipeline/optimizer_backend.py index 1af86050..adb71524 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/optimizer_backend.py +++ b/examples/optimization/eval_optimize_loop/pipeline/optimizer_backend.py @@ -126,7 +126,7 @@ async def generate_candidates( return records def _extract_candidates(self, result: object, baseline_prompts: dict[str, str]) -> list[CandidateRecord]: - entries: list[tuple[str, int | None, bool | None, str | None, object]] = [] + entries: list[tuple[str, int | None, bool | None, str | None, object, float | None, float | None]] = [] rounds = list(getattr(result, "rounds", []) or []) if self._candidate_scope != "best_only": for fallback_index, round_record in enumerate(rounds, start=1): @@ -134,14 +134,21 @@ def _extract_candidates(self, result: object, baseline_prompts: dict[str, str]) if self._candidate_scope == "accepted_rounds" and not accepted: continue round_index = int(getattr(round_record, "round", fallback_index)) - entries.append((f"round-{round_index:03d}", round_index, accepted, getattr(round_record, "acceptance_reason", None), getattr(round_record, "candidate_prompts", None))) - entries.append(("best", None, True, "OptimizeResult.best_prompts", getattr(result, "best_prompts", None))) + entries.append(( + f"round-{round_index:03d}", round_index, accepted, getattr(round_record, "acceptance_reason", None), + getattr(round_record, "candidate_prompts", None), getattr(round_record, "generation_cost_usd", getattr(round_record, "cost_usd", None)), + getattr(round_record, "duration_seconds", None), + )) + entries.append(( + "best", None, True, "OptimizeResult.best_prompts", getattr(result, "best_prompts", None), + getattr(result, "generation_cost_usd", getattr(result, "cost_usd", None)), getattr(result, "duration_seconds", None), + )) retained: list[CandidateRecord] = [] seen: dict[str, CandidateRecord] = {} duplicates: dict[str, list[str]] = {} skipped: list[str] = [] - for candidate_id, round_index, optimizer_accepted, optimizer_reason, prompts in entries: + for candidate_id, round_index, optimizer_accepted, optimizer_reason, prompts, generation_cost_usd, duration_seconds in entries: full_prompts = _full_prompt_map(prompts, baseline_prompts) if full_prompts is None: skipped.append(candidate_id) @@ -157,6 +164,8 @@ def _extract_candidates(self, result: object, baseline_prompts: dict[str, str]) prompts=full_prompts, optimizer_accepted=optimizer_accepted, optimizer_reason=optimizer_reason or "", + generation_cost_usd=generation_cost_usd, + duration_seconds=duration_seconds, ) seen[digest] = record retained.append(record) diff --git a/examples/optimization/eval_optimize_loop/pipeline/reporter.py b/examples/optimization/eval_optimize_loop/pipeline/reporter.py index c2c2a81d..48c5a3af 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/reporter.py +++ b/examples/optimization/eval_optimize_loop/pipeline/reporter.py @@ -3,14 +3,63 @@ import json from pathlib import Path +from pydantic import BaseModel + +from .config import sanitize_config from .models import OptimizationReport +def _json_safe(value: object) -> object: + if isinstance(value, BaseModel): + return _json_safe(value.model_dump(mode="json")) + if isinstance(value, Path): + return str(value) + if isinstance(value, dict): + return {str(key): _json_safe(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_json_safe(item) for item in value] + return value + + +def _write_json(path: Path, payload: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(sanitize_config(_json_safe(payload)), ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def write_audit_artifacts(report: OptimizationReport, output_dir: Path) -> None: + """Persist normalized, secret-free evidence used by the Gate decision.""" + audit = output_dir / "audit" + refs = report.audit_references + _write_json(output_dir / refs.raw_reports_path, {"baseline_train": report.baseline_train, "baseline_validation": report.baseline_validation}) + _write_json( + output_dir / refs.normalized_reports_path, + { + "baseline_train": report.baseline_train, + "baseline_validation": report.baseline_validation, + "candidate_train_validation": [ + {"candidate_id": candidate.candidate_id, "train": candidate.train, "validation": candidate.validation} + for candidate in report.candidates + ], + }, + ) + _write_json(output_dir / refs.candidate_reports_path, report.candidates) + _write_json(output_dir / refs.gate_decisions_path, {"selected_candidate_id": report.selected_candidate_id, "decisions": [{"candidate_id": candidate.candidate_id, "gate": candidate.gate} for candidate in report.candidates]}) + # The run functions may write richer snapshots before this call. Keep a + # deterministic placeholder so every mode has the documented references. + config_path = output_dir / refs.config_snapshot_path + if not config_path.exists(): + _write_json(config_path, {"mode": report.mode, "seed": report.seed, "run_metadata": report.run_metadata}) + environment_path = output_dir / refs.environment_snapshot_path + if not environment_path.exists(): + _write_json(environment_path, {"mode": report.mode, "seed": report.seed}) + + def write_reports(report: OptimizationReport, output_dir: Path) -> tuple[Path, Path]: output_dir.mkdir(parents=True, exist_ok=True) + write_audit_artifacts(report, output_dir) json_path = output_dir / "optimization_report.json" markdown_path = output_dir / "optimization_report.md" - json_path.write_text(json.dumps(report.model_dump(mode="json"), ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + json_path.write_text(json.dumps(sanitize_config(report.model_dump(mode="json")), ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") lines = [ "# Optimization Report", "", @@ -18,6 +67,7 @@ def write_reports(report: OptimizationReport, output_dir: Path) -> tuple[Path, P f"- Seed: `{report.seed}`", f"- Selected candidate: `{report.selected_candidate_id or 'none'}`", f"- Source integrity: `{report.source_integrity}`", + f"- Audit evidence: `{report.audit_references.gate_decisions_path}`", "", "## Baseline", "", @@ -72,6 +122,8 @@ def write_reports(report: OptimizationReport, output_dir: Path) -> tuple[Path, P if candidate.gate: for rule in candidate.gate.rules: section.append(f"| `{rule.rule}` | {'yes' if rule.passed else 'no'} | `{rule.actual}` | `{rule.expected}` |") + if candidate.gate.warnings: + section.append(f"\nWarnings: {'; '.join(candidate.gate.warnings)}") candidate_sections.extend(section) lines.extend([ "", diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index 9e6ff5da..2626349b 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -24,9 +24,10 @@ from examples.optimization.eval_optimize_loop.fake.fake_judge import register_fake_rubric_evaluator from examples.optimization.eval_optimize_loop.fake.fixture_optimizer import FixtureOptimizerBackend from examples.optimization.eval_optimize_loop.pipeline.comparator import compare_case -from examples.optimization.eval_optimize_loop.pipeline.config import load_pipeline_config +from examples.optimization.eval_optimize_loop.pipeline.config import load_pipeline_config, sanitize_config from examples.optimization.eval_optimize_loop.pipeline.evaluator import evaluate_split from examples.optimization.eval_optimize_loop.pipeline.gate import evaluate_gate +from examples.optimization.eval_optimize_loop.pipeline.gate import select_winner from examples.optimization.eval_optimize_loop.pipeline.models import CandidateReport, OptimizationReport, SplitReport from examples.optimization.eval_optimize_loop.pipeline.optimizer_backend import ( AgentOptimizerBackend, @@ -37,12 +38,22 @@ from examples.optimization.eval_optimize_loop.pipeline.normalization import normalize_eval_results from examples.optimization.eval_optimize_loop.pipeline.prompt_sandbox import PromptSandbox from examples.optimization.eval_optimize_loop.pipeline.reporter import write_reports - +from examples.optimization.eval_optimize_loop.pipeline.audit import write_environment_snapshot, write_input_snapshot +def _attribution_summary(*splits: SplitReport | None) -> dict[str, int]: + summary: dict[str, int] = {} + for split in splits: + if split is not None: + for case in split.cases: + if case.failure_attribution is not None: + key = case.failure_attribution.primary_type.value + summary[key] = summary.get(key, 0) + 1 + return dict(sorted(summary.items())) async def run_fake_pipeline(*, output_dir: Path) -> OptimizationReport: config = load_pipeline_config(HERE / "optimizer.json", mode="fake") report = OptimizationReport.empty(mode="fake", seed=config.pipeline.reproducibility.seed) + report.run_metadata = {"evaluation_source": "fixture", "independent_candidate_evaluation": True} with tempfile.TemporaryDirectory(prefix="trpc-agent-issue91-") as temporary_dir: prompt_dir = Path(temporary_dir) source_prompt_dir = HERE / "agent" / "prompts" @@ -50,6 +61,9 @@ async def run_fake_pipeline(*, output_dir: Path) -> OptimizationReport: (prompt_dir / prompt_name).write_text((source_prompt_dir / prompt_name).read_text(encoding="utf-8"), encoding="utf-8") target = TargetPrompt().add_path("system_prompt", str(prompt_dir / "system.md")).add_path("router_prompt", str(prompt_dir / "router.md")) + output_dir.mkdir(parents=True, exist_ok=True) + write_input_snapshot(config, target, output_dir) + write_environment_snapshot("fake", report.seed, output_dir) fake_agent = FakeSupportAgent(target) evaluate = lambda path, split: evaluate_split(path, call_agent=fake_agent.call_agent, split=split, metric_weights=config.pipeline.metric_weights) baseline_train = await evaluate(HERE / "train.evalset.json", "train") @@ -61,10 +75,11 @@ async def run_fake_pipeline(*, output_dir: Path) -> OptimizationReport: train = await evaluate(HERE / "train.evalset.json", "train") validation = await evaluate(HERE / "val.evalset.json", "validation") deltas = [compare_case(base, next(item for item in validation.cases if item.eval_id == base.eval_id), epsilon=config.pipeline.scoring_epsilon, critical_case_ids=set(config.pipeline.gate.critical_case_ids)) for base in baseline_validation.cases] - decision = evaluate_gate(baseline_validation, validation, settings=config.pipeline.gate, case_deltas=deltas, train_score_delta=train.aggregate_score - baseline_train.aggregate_score) - report.candidates.append(CandidateReport(candidate_id=candidate.candidate_id, accepted=decision.accepted, reasons=decision.reasons, train=train, validation=validation, gate=decision, validation_case_deltas=deltas)) - accepted = [candidate for candidate in report.candidates if candidate.accepted] - report.selected_candidate_id = accepted[0].candidate_id if accepted else None + decision = evaluate_gate(baseline_validation, validation, settings=config.pipeline.gate, case_deltas=deltas, train_score_delta=train.aggregate_score - baseline_train.aggregate_score, metric_floors=config.pipeline.metric_floors, generation_cost_usd=candidate.generation_cost_usd, duration_seconds=candidate.duration_seconds, epsilon=config.pipeline.scoring_epsilon) + report.candidates.append(CandidateReport(candidate_id=candidate.candidate_id, accepted=decision.accepted, reasons=decision.reasons, train=train, validation=validation, gate=decision, validation_case_deltas=deltas, independently_evaluated=True, source=candidate.source, generation_cost_usd=candidate.generation_cost_usd, duration_seconds=candidate.duration_seconds)) + report.selected_candidate_id = select_winner(report.candidates) + report.warnings = sorted({warning for candidate in report.candidates if candidate.gate for warning in candidate.gate.warnings}) + report.attribution_summary = _attribution_summary(report.baseline_train, report.baseline_validation) write_reports(report, output_dir) return report @@ -92,8 +107,12 @@ async def run_trace_pipeline(*, output_dir: Path) -> OptimizationReport: for case in normalized.values() ] report = OptimizationReport.empty(mode="trace", seed=config_payload["seed"]) + report.run_metadata = {"evaluation_source": "recorded_trace", "independent_candidate_evaluation": False} report.baseline_validation = SplitReport.from_cases(cases) + report.attribution_summary = _attribution_summary(report.baseline_validation) output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "input.snapshot.json").write_text(json.dumps(sanitize_config({"trace_config": config_payload, "trace_evalset": "trace/trace.evalset.json"}), ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + write_environment_snapshot("trace", report.seed, output_dir) (output_dir / "trace_raw_results.json").write_text( json.dumps( { @@ -122,6 +141,7 @@ async def run_live_pipeline(*, output_dir: Path) -> OptimizationReport: """Run the public optimizer API, then independently evaluate its proposals.""" config = load_pipeline_config(HERE / "optimizer.json", mode="live") report = OptimizationReport.empty(mode="live", seed=config.pipeline.reproducibility.seed) + report.run_metadata = {"evaluation_source": "independent_full_train_validation", "independent_candidate_evaluation": True, "write_back_default": False} source_prompt_dir = HERE / "agent" / "prompts" source_baseline = { "system_prompt": (source_prompt_dir / "system.md").read_text(encoding="utf-8"), @@ -137,6 +157,9 @@ async def run_live_pipeline(*, output_dir: Path) -> OptimizationReport: path = prompt_dir / f"{name}.md" path.write_text(content, encoding="utf-8") target.add_path(name, str(path)) + output_dir.mkdir(parents=True, exist_ok=True) + write_input_snapshot(config, target, output_dir) + write_environment_snapshot("live", report.seed, output_dir) fake_agent = FakeSupportAgent(target) evaluate = lambda path, split: evaluate_split( path, @@ -175,6 +198,10 @@ async def run_live_pipeline(*, output_dir: Path) -> OptimizationReport: settings=config.pipeline.gate, case_deltas=deltas, train_score_delta=train.aggregate_score - report.baseline_train.aggregate_score, + metric_floors=config.pipeline.metric_floors, + generation_cost_usd=candidate.generation_cost_usd, + duration_seconds=candidate.duration_seconds, + epsilon=config.pipeline.scoring_epsilon, ) report.candidates.append( CandidateReport( @@ -185,10 +212,16 @@ async def run_live_pipeline(*, output_dir: Path) -> OptimizationReport: validation=validation, gate=decision, validation_case_deltas=deltas, + independently_evaluated=True, + source=candidate.source, + generation_cost_usd=candidate.generation_cost_usd, + duration_seconds=candidate.duration_seconds, ) ) accepted = [candidate for candidate in report.candidates if candidate.accepted] - report.selected_candidate_id = accepted[0].candidate_id if accepted else None + report.selected_candidate_id = select_winner(report.candidates) + report.warnings = sorted({warning for candidate in report.candidates if candidate.gate for warning in candidate.gate.warnings}) + report.attribution_summary = _attribution_summary(report.baseline_train, report.baseline_validation) if config.pipeline.write_back_when_accepted and report.selected_candidate_id: selected_record = next(candidate for candidate in candidates if candidate.candidate_id == report.selected_candidate_id) selected_report = next(candidate for candidate in accepted if candidate.candidate_id == report.selected_candidate_id) diff --git a/examples/optimization/eval_optimize_loop/sample_output/audit/candidate_reports.json b/examples/optimization/eval_optimize_loop/sample_output/audit/candidate_reports.json new file mode 100644 index 00000000..4389bc28 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/audit/candidate_reports.json @@ -0,0 +1,1320 @@ +[ + { + "accepted": true, + "candidate_id": "candidate_general_fix", + "duration_seconds": null, + "gate": { + "accepted": true, + "reasons": [ + "candidate passed all independent gate rules" + ], + "risk_level": "low", + "rules": [ + { + "actual": "complete", + "expected": "complete", + "passed": true, + "reason": "independent validation evaluation is incomplete", + "rule": "evaluation_complete" + }, + { + "actual": 0.33333333333333337, + "expected": "number", + "passed": true, + "reason": "validation aggregate score delta is required", + "rule": "validation_score_delta_available" + }, + { + "actual": 0.33333333333333337, + "expected": "number", + "passed": true, + "reason": "validation pass-rate delta is required", + "rule": "validation_pass_rate_delta_available" + }, + { + "actual": 0.33333333333333337, + "expected": 0.05, + "passed": true, + "reason": "validation aggregate score must improve", + "rule": "validation_score_improved" + }, + { + "actual": 0.33333333333333337, + "expected": 0.0, + "passed": true, + "reason": "validation pass rate must not decline", + "rule": "validation_pass_rate_not_worse" + }, + { + "actual": 0, + "expected": 0, + "passed": true, + "reason": "new hard failures are not allowed", + "rule": "new_hard_fails" + }, + { + "actual": 0, + "expected": 0, + "passed": true, + "reason": "validation regressions exceed the limit", + "rule": "validation_regressions" + }, + { + "actual": false, + "expected": false, + "passed": true, + "reason": "critical validation cases must not regress", + "rule": "no_critical_regression" + }, + { + "actual": false, + "expected": false, + "passed": true, + "reason": "train improvement with validation decline is rejected", + "rule": "no_overfit" + }, + { + "actual": false, + "expected": false, + "passed": true, + "reason": "tie policy rejects a non-improving validation outcome", + "rule": "tie_policy" + } + ], + "warnings": [ + "generation cost is unknown" + ] + }, + "generation_cost_usd": null, + "independently_evaluated": true, + "reasons": [ + "candidate passed all independent gate rules" + ], + "source": "fixture", + "train": { + "aggregate_score": 0.7666666666666666, + "cases": [ + { + "aggregate_score": 1.0, + "eval_id": "train_tool_argument", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:5da8db545d2c7b26c93058b6619ab49dd59c1348f6b032f05c451c2709b66e6e" + }, + { + "aggregate_score": 0.30000000000000004, + "eval_id": "train_missing_knowledge", + "execution_errors": [], + "expected_response": "{\"answer\": \"未提供该政策，不能猜测。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", + "expected_tool_calls": [], + "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "failure_types": [ + "final_response_mismatch" + ], + "final_response": "{\"answer\": \"我不确定。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "unknown knowledge must explicitly refuse to guess" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [], + "tool_responses": [], + "trace_digest": "sha256:3f059c7504becc8cb98f605431f3c3714a0e70b4e62931895b888bba4ffcf9ed" + }, + { + "aggregate_score": 1.0, + "eval_id": "train_json_format", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:5da8db545d2c7b26c93058b6619ab49dd59c1348f6b032f05c451c2709b66e6e" + } + ], + "pass_rate": 0.6666666666666666 + }, + "validation": { + "aggregate_score": 1.0, + "cases": [ + { + "aggregate_score": 1.0, + "eval_id": "val_refund_critical", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "amount": "12", + "currency": "USD", + "order_id": "R900" + }, + "name": "refund_order" + } + ], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [ + { + "arguments": { + "amount": "12", + "currency": "USD", + "order_id": "R900" + }, + "name": "refund_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:19bb632d52c6f72bc177991367b8e146fd76b0fb3a68bc8ee0622be9fa4df8a8" + }, + { + "aggregate_score": 1.0, + "eval_id": "val_stable_faq", + "execution_errors": [], + "expected_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "expected_tool_calls": [], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [], + "tool_responses": [], + "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" + }, + { + "aggregate_score": 1.0, + "eval_id": "val_json_generalization", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:99d82b8a08b7eeeda5b5f41d08a3afc1c9f4f22879605c257b78458c4ef9e561" + } + ], + "pass_rate": 1.0 + }, + "validation_case_deltas": [ + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_passed": true, + "candidate_score": 1.0, + "critical": true, + "eval_id": "val_refund_critical", + "hard_fail_added": false, + "metric_deltas": { + "fake_rubric_score": 0.0, + "final_response_avg_score": 0.0 + }, + "new_failure_types": [], + "resolved_failure_types": [], + "score_delta": 0.0, + "transition": "UNCHANGED" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_passed": true, + "candidate_score": 1.0, + "critical": false, + "eval_id": "val_stable_faq", + "hard_fail_added": false, + "metric_deltas": { + "fake_rubric_score": 0.0, + "final_response_avg_score": 0.0 + }, + "new_failure_types": [], + "resolved_failure_types": [], + "score_delta": 0.0, + "transition": "UNCHANGED" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_passed": true, + "candidate_score": 1.0, + "critical": false, + "eval_id": "val_json_generalization", + "hard_fail_added": false, + "metric_deltas": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "new_failure_types": [], + "resolved_failure_types": [ + "final_response_mismatch", + "format_violation" + ], + "score_delta": 1.0, + "transition": "NEW_PASS" + } + ] + }, + { + "accepted": false, + "candidate_id": "candidate_noop", + "duration_seconds": null, + "gate": { + "accepted": false, + "reasons": [ + "validation aggregate score must improve", + "tie policy rejects a non-improving validation outcome" + ], + "risk_level": "medium", + "rules": [ + { + "actual": "complete", + "expected": "complete", + "passed": true, + "reason": "independent validation evaluation is incomplete", + "rule": "evaluation_complete" + }, + { + "actual": 0.0, + "expected": "number", + "passed": true, + "reason": "validation aggregate score delta is required", + "rule": "validation_score_delta_available" + }, + { + "actual": 0.0, + "expected": "number", + "passed": true, + "reason": "validation pass-rate delta is required", + "rule": "validation_pass_rate_delta_available" + }, + { + "actual": 0.0, + "expected": 0.05, + "passed": false, + "reason": "validation aggregate score must improve", + "rule": "validation_score_improved" + }, + { + "actual": 0.0, + "expected": 0.0, + "passed": true, + "reason": "validation pass rate must not decline", + "rule": "validation_pass_rate_not_worse" + }, + { + "actual": 0, + "expected": 0, + "passed": true, + "reason": "new hard failures are not allowed", + "rule": "new_hard_fails" + }, + { + "actual": 0, + "expected": 0, + "passed": true, + "reason": "validation regressions exceed the limit", + "rule": "validation_regressions" + }, + { + "actual": false, + "expected": false, + "passed": true, + "reason": "critical validation cases must not regress", + "rule": "no_critical_regression" + }, + { + "actual": false, + "expected": false, + "passed": true, + "reason": "train improvement with validation decline is rejected", + "rule": "no_overfit" + }, + { + "actual": true, + "expected": false, + "passed": false, + "reason": "tie policy rejects a non-improving validation outcome", + "rule": "tie_policy" + } + ], + "warnings": [ + "generation cost is unknown" + ] + }, + "generation_cost_usd": null, + "independently_evaluated": true, + "reasons": [ + "validation aggregate score must improve", + "tie policy rejects a non-improving validation outcome" + ], + "source": "fixture", + "train": { + "aggregate_score": 0.20000000000000004, + "cases": [ + { + "aggregate_score": 0.30000000000000004, + "eval_id": "train_tool_argument", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "failure_types": [ + "final_response_mismatch" + ], + "final_response": "{\"answer\": \"正在查询订单。\", \"arguments\": {}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "required tool arguments do not match expected response" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": {}, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:f086f4b22284734f1462eee58a4d204a31361c26d67449e5b23d292aaf2642bf" + }, + { + "aggregate_score": 0.30000000000000004, + "eval_id": "train_missing_knowledge", + "execution_errors": [], + "expected_response": "{\"answer\": \"未提供该政策，不能猜测。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", + "expected_tool_calls": [], + "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "failure_types": [ + "final_response_mismatch" + ], + "final_response": "{\"answer\": \"我不确定。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "unknown knowledge must explicitly refuse to guess" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [], + "tool_responses": [], + "trace_digest": "sha256:3f059c7504becc8cb98f605431f3c3714a0e70b4e62931895b888bba4ffcf9ed" + }, + { + "aggregate_score": 0.0, + "eval_id": "train_json_format", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0", + "metric fake_rubric_score did not meet threshold 0.75", + "invalid JSON response", + "actual response: invalid JSON response" + ], + "failure_types": [ + "final_response_mismatch", + "format_violation" + ], + "final_response": "订单 A100 正在查询", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": false, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "invalid JSON response" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.0, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [], + "tool_responses": [], + "trace_digest": "sha256:65fdc3d48fd18eb37b2aa5138ee27435257ecbd0ef582c523a42673d87b04b77" + } + ], + "pass_rate": 0.0 + }, + "validation": { + "aggregate_score": 0.6666666666666666, + "cases": [ + { + "aggregate_score": 1.0, + "eval_id": "val_refund_critical", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "amount": "12", + "currency": "USD", + "order_id": "R900" + }, + "name": "refund_order" + } + ], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [ + { + "arguments": { + "amount": "12", + "currency": "USD", + "order_id": "R900" + }, + "name": "refund_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:19bb632d52c6f72bc177991367b8e146fd76b0fb3a68bc8ee0622be9fa4df8a8" + }, + { + "aggregate_score": 1.0, + "eval_id": "val_stable_faq", + "execution_errors": [], + "expected_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "expected_tool_calls": [], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [], + "tool_responses": [], + "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" + }, + { + "aggregate_score": 0.0, + "eval_id": "val_json_generalization", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0", + "metric fake_rubric_score did not meet threshold 0.75", + "invalid JSON response", + "actual response: invalid JSON response" + ], + "failure_types": [ + "final_response_mismatch", + "format_violation" + ], + "final_response": "订单 B200 正在查询", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": false, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "invalid JSON response" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.0, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "validation", + "tool_calls": [], + "tool_responses": [], + "trace_digest": "sha256:8684ddee9351120c1bf48affbd280c5b73835b2c1d58d04682622b577c37a455" + } + ], + "pass_rate": 0.6666666666666666 + }, + "validation_case_deltas": [ + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_passed": true, + "candidate_score": 1.0, + "critical": true, + "eval_id": "val_refund_critical", + "hard_fail_added": false, + "metric_deltas": { + "fake_rubric_score": 0.0, + "final_response_avg_score": 0.0 + }, + "new_failure_types": [], + "resolved_failure_types": [], + "score_delta": 0.0, + "transition": "UNCHANGED" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_passed": true, + "candidate_score": 1.0, + "critical": false, + "eval_id": "val_stable_faq", + "hard_fail_added": false, + "metric_deltas": { + "fake_rubric_score": 0.0, + "final_response_avg_score": 0.0 + }, + "new_failure_types": [], + "resolved_failure_types": [], + "score_delta": 0.0, + "transition": "UNCHANGED" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_passed": false, + "candidate_score": 0.0, + "critical": false, + "eval_id": "val_json_generalization", + "hard_fail_added": false, + "metric_deltas": { + "fake_rubric_score": 0.0, + "final_response_avg_score": 0.0 + }, + "new_failure_types": [], + "resolved_failure_types": [], + "score_delta": 0.0, + "transition": "UNCHANGED" + } + ] + }, + { + "accepted": false, + "candidate_id": "candidate_overfit", + "duration_seconds": null, + "gate": { + "accepted": false, + "reasons": [ + "new hard failures are not allowed", + "validation regressions exceed the limit", + "critical validation cases must not regress" + ], + "risk_level": "high", + "rules": [ + { + "actual": "complete", + "expected": "complete", + "passed": true, + "reason": "independent validation evaluation is incomplete", + "rule": "evaluation_complete" + }, + { + "actual": 0.06666666666666676, + "expected": "number", + "passed": true, + "reason": "validation aggregate score delta is required", + "rule": "validation_score_delta_available" + }, + { + "actual": 0.0, + "expected": "number", + "passed": true, + "reason": "validation pass-rate delta is required", + "rule": "validation_pass_rate_delta_available" + }, + { + "actual": 0.06666666666666676, + "expected": 0.05, + "passed": true, + "reason": "validation aggregate score must improve", + "rule": "validation_score_improved" + }, + { + "actual": 0.0, + "expected": 0.0, + "passed": true, + "reason": "validation pass rate must not decline", + "rule": "validation_pass_rate_not_worse" + }, + { + "actual": 1, + "expected": 0, + "passed": false, + "reason": "new hard failures are not allowed", + "rule": "new_hard_fails" + }, + { + "actual": 1, + "expected": 0, + "passed": false, + "reason": "validation regressions exceed the limit", + "rule": "validation_regressions" + }, + { + "actual": true, + "expected": false, + "passed": false, + "reason": "critical validation cases must not regress", + "rule": "no_critical_regression" + }, + { + "actual": false, + "expected": false, + "passed": true, + "reason": "train improvement with validation decline is rejected", + "rule": "no_overfit" + }, + { + "actual": false, + "expected": false, + "passed": true, + "reason": "tie policy rejects a non-improving validation outcome", + "rule": "tie_policy" + } + ], + "warnings": [ + "generation cost is unknown" + ] + }, + "generation_cost_usd": null, + "independently_evaluated": true, + "reasons": [ + "new hard failures are not allowed", + "validation regressions exceed the limit", + "critical validation cases must not regress" + ], + "source": "fixture", + "train": { + "aggregate_score": 1.0, + "cases": [ + { + "aggregate_score": 1.0, + "eval_id": "train_tool_argument", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:5da8db545d2c7b26c93058b6619ab49dd59c1348f6b032f05c451c2709b66e6e" + }, + { + "aggregate_score": 1.0, + "eval_id": "train_missing_knowledge", + "execution_errors": [], + "expected_response": "{\"answer\": \"未提供该政策，不能猜测。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", + "expected_tool_calls": [], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"未提供该政策，不能猜测。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "train", + "tool_calls": [], + "tool_responses": [], + "trace_digest": "sha256:22d90b34cb10f26e6a207ed065e82b862b238989d23eda1b8dbd8d4072208e0f" + }, + { + "aggregate_score": 1.0, + "eval_id": "train_json_format", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:5da8db545d2c7b26c93058b6619ab49dd59c1348f6b032f05c451c2709b66e6e" + } + ], + "pass_rate": 1.0 + }, + "validation": { + "aggregate_score": 0.7333333333333334, + "cases": [ + { + "aggregate_score": 0.2, + "eval_id": "val_refund_critical", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "amount": "12", + "currency": "USD", + "order_id": "R900" + }, + "name": "refund_order" + } + ], + "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0", + "metric fake_rubric_score did not meet threshold 0.75", + "route or tool does not match expected response; required tool arguments do not match expected response" + ], + "failure_types": [ + "final_response_mismatch", + "tool_argument_error", + "tool_selection_error" + ], + "final_response": "{\"answer\": \"正在查询订单 R900。\", \"arguments\": {\"order_id\": \"R900\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": false, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "route or tool does not match expected response; required tool arguments do not match expected response" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.5, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "validation", + "tool_calls": [ + { + "arguments": { + "order_id": "R900" + }, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:254559dde2e80e22b22f6c4cb0d37c61bbd5a8200e19d46232c76f1b0a5d8604" + }, + { + "aggregate_score": 1.0, + "eval_id": "val_stable_faq", + "execution_errors": [], + "expected_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "expected_tool_calls": [], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [], + "tool_responses": [], + "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" + }, + { + "aggregate_score": 1.0, + "eval_id": "val_json_generalization", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:99d82b8a08b7eeeda5b5f41d08a3afc1c9f4f22879605c257b78458c4ef9e561" + } + ], + "pass_rate": 0.6666666666666666 + }, + "validation_case_deltas": [ + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_passed": false, + "candidate_score": 0.2, + "critical": true, + "eval_id": "val_refund_critical", + "hard_fail_added": true, + "metric_deltas": { + "fake_rubric_score": -0.5, + "final_response_avg_score": -1.0 + }, + "new_failure_types": [ + "final_response_mismatch", + "tool_argument_error", + "tool_selection_error" + ], + "resolved_failure_types": [], + "score_delta": -0.8, + "transition": "REGRESSION" + }, + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_passed": true, + "candidate_score": 1.0, + "critical": false, + "eval_id": "val_stable_faq", + "hard_fail_added": false, + "metric_deltas": { + "fake_rubric_score": 0.0, + "final_response_avg_score": 0.0 + }, + "new_failure_types": [], + "resolved_failure_types": [], + "score_delta": 0.0, + "transition": "UNCHANGED" + }, + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_passed": true, + "candidate_score": 1.0, + "critical": false, + "eval_id": "val_json_generalization", + "hard_fail_added": false, + "metric_deltas": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "new_failure_types": [], + "resolved_failure_types": [ + "final_response_mismatch", + "format_violation" + ], + "score_delta": 1.0, + "transition": "NEW_PASS" + } + ] + } +] diff --git a/examples/optimization/eval_optimize_loop/sample_output/audit/gate_decisions.json b/examples/optimization/eval_optimize_loop/sample_output/audit/gate_decisions.json new file mode 100644 index 00000000..5b314869 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/audit/gate_decisions.json @@ -0,0 +1,263 @@ +{ + "decisions": [ + { + "candidate_id": "candidate_general_fix", + "gate": { + "accepted": true, + "reasons": [ + "candidate passed all independent gate rules" + ], + "risk_level": "low", + "rules": [ + { + "actual": "complete", + "expected": "complete", + "passed": true, + "reason": "independent validation evaluation is incomplete", + "rule": "evaluation_complete" + }, + { + "actual": 0.33333333333333337, + "expected": "number", + "passed": true, + "reason": "validation aggregate score delta is required", + "rule": "validation_score_delta_available" + }, + { + "actual": 0.33333333333333337, + "expected": "number", + "passed": true, + "reason": "validation pass-rate delta is required", + "rule": "validation_pass_rate_delta_available" + }, + { + "actual": 0.33333333333333337, + "expected": 0.05, + "passed": true, + "reason": "validation aggregate score must improve", + "rule": "validation_score_improved" + }, + { + "actual": 0.33333333333333337, + "expected": 0.0, + "passed": true, + "reason": "validation pass rate must not decline", + "rule": "validation_pass_rate_not_worse" + }, + { + "actual": 0, + "expected": 0, + "passed": true, + "reason": "new hard failures are not allowed", + "rule": "new_hard_fails" + }, + { + "actual": 0, + "expected": 0, + "passed": true, + "reason": "validation regressions exceed the limit", + "rule": "validation_regressions" + }, + { + "actual": false, + "expected": false, + "passed": true, + "reason": "critical validation cases must not regress", + "rule": "no_critical_regression" + }, + { + "actual": false, + "expected": false, + "passed": true, + "reason": "train improvement with validation decline is rejected", + "rule": "no_overfit" + }, + { + "actual": false, + "expected": false, + "passed": true, + "reason": "tie policy rejects a non-improving validation outcome", + "rule": "tie_policy" + } + ], + "warnings": [ + "generation cost is unknown" + ] + } + }, + { + "candidate_id": "candidate_noop", + "gate": { + "accepted": false, + "reasons": [ + "validation aggregate score must improve", + "tie policy rejects a non-improving validation outcome" + ], + "risk_level": "medium", + "rules": [ + { + "actual": "complete", + "expected": "complete", + "passed": true, + "reason": "independent validation evaluation is incomplete", + "rule": "evaluation_complete" + }, + { + "actual": 0.0, + "expected": "number", + "passed": true, + "reason": "validation aggregate score delta is required", + "rule": "validation_score_delta_available" + }, + { + "actual": 0.0, + "expected": "number", + "passed": true, + "reason": "validation pass-rate delta is required", + "rule": "validation_pass_rate_delta_available" + }, + { + "actual": 0.0, + "expected": 0.05, + "passed": false, + "reason": "validation aggregate score must improve", + "rule": "validation_score_improved" + }, + { + "actual": 0.0, + "expected": 0.0, + "passed": true, + "reason": "validation pass rate must not decline", + "rule": "validation_pass_rate_not_worse" + }, + { + "actual": 0, + "expected": 0, + "passed": true, + "reason": "new hard failures are not allowed", + "rule": "new_hard_fails" + }, + { + "actual": 0, + "expected": 0, + "passed": true, + "reason": "validation regressions exceed the limit", + "rule": "validation_regressions" + }, + { + "actual": false, + "expected": false, + "passed": true, + "reason": "critical validation cases must not regress", + "rule": "no_critical_regression" + }, + { + "actual": false, + "expected": false, + "passed": true, + "reason": "train improvement with validation decline is rejected", + "rule": "no_overfit" + }, + { + "actual": true, + "expected": false, + "passed": false, + "reason": "tie policy rejects a non-improving validation outcome", + "rule": "tie_policy" + } + ], + "warnings": [ + "generation cost is unknown" + ] + } + }, + { + "candidate_id": "candidate_overfit", + "gate": { + "accepted": false, + "reasons": [ + "new hard failures are not allowed", + "validation regressions exceed the limit", + "critical validation cases must not regress" + ], + "risk_level": "high", + "rules": [ + { + "actual": "complete", + "expected": "complete", + "passed": true, + "reason": "independent validation evaluation is incomplete", + "rule": "evaluation_complete" + }, + { + "actual": 0.06666666666666676, + "expected": "number", + "passed": true, + "reason": "validation aggregate score delta is required", + "rule": "validation_score_delta_available" + }, + { + "actual": 0.0, + "expected": "number", + "passed": true, + "reason": "validation pass-rate delta is required", + "rule": "validation_pass_rate_delta_available" + }, + { + "actual": 0.06666666666666676, + "expected": 0.05, + "passed": true, + "reason": "validation aggregate score must improve", + "rule": "validation_score_improved" + }, + { + "actual": 0.0, + "expected": 0.0, + "passed": true, + "reason": "validation pass rate must not decline", + "rule": "validation_pass_rate_not_worse" + }, + { + "actual": 1, + "expected": 0, + "passed": false, + "reason": "new hard failures are not allowed", + "rule": "new_hard_fails" + }, + { + "actual": 1, + "expected": 0, + "passed": false, + "reason": "validation regressions exceed the limit", + "rule": "validation_regressions" + }, + { + "actual": true, + "expected": false, + "passed": false, + "reason": "critical validation cases must not regress", + "rule": "no_critical_regression" + }, + { + "actual": false, + "expected": false, + "passed": true, + "reason": "train improvement with validation decline is rejected", + "rule": "no_overfit" + }, + { + "actual": false, + "expected": false, + "passed": true, + "reason": "tie policy rejects a non-improving validation outcome", + "rule": "tie_policy" + } + ], + "warnings": [ + "generation cost is unknown" + ] + } + } + ], + "selected_candidate_id": "candidate_general_fix" +} diff --git a/examples/optimization/eval_optimize_loop/sample_output/audit/normalized_reports.json b/examples/optimization/eval_optimize_loop/sample_output/audit/normalized_reports.json new file mode 100644 index 00000000..aa02dbb2 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/audit/normalized_reports.json @@ -0,0 +1,1167 @@ +{ + "baseline_train": { + "aggregate_score": 0.20000000000000004, + "cases": [ + { + "aggregate_score": 0.30000000000000004, + "eval_id": "train_tool_argument", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "failure_types": [ + "final_response_mismatch" + ], + "final_response": "{\"answer\": \"正在查询订单。\", \"arguments\": {}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "required tool arguments do not match expected response" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": {}, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:f086f4b22284734f1462eee58a4d204a31361c26d67449e5b23d292aaf2642bf" + }, + { + "aggregate_score": 0.30000000000000004, + "eval_id": "train_missing_knowledge", + "execution_errors": [], + "expected_response": "{\"answer\": \"未提供该政策，不能猜测。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", + "expected_tool_calls": [], + "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "failure_types": [ + "final_response_mismatch" + ], + "final_response": "{\"answer\": \"我不确定。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "unknown knowledge must explicitly refuse to guess" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [], + "tool_responses": [], + "trace_digest": "sha256:3f059c7504becc8cb98f605431f3c3714a0e70b4e62931895b888bba4ffcf9ed" + }, + { + "aggregate_score": 0.0, + "eval_id": "train_json_format", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0", + "metric fake_rubric_score did not meet threshold 0.75", + "invalid JSON response", + "actual response: invalid JSON response" + ], + "failure_types": [ + "final_response_mismatch", + "format_violation" + ], + "final_response": "订单 A100 正在查询", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": false, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "invalid JSON response" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.0, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [], + "tool_responses": [], + "trace_digest": "sha256:65fdc3d48fd18eb37b2aa5138ee27435257ecbd0ef582c523a42673d87b04b77" + } + ], + "pass_rate": 0.0 + }, + "baseline_validation": { + "aggregate_score": 0.6666666666666666, + "cases": [ + { + "aggregate_score": 1.0, + "eval_id": "val_refund_critical", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "amount": "12", + "currency": "USD", + "order_id": "R900" + }, + "name": "refund_order" + } + ], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [ + { + "arguments": { + "amount": "12", + "currency": "USD", + "order_id": "R900" + }, + "name": "refund_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:19bb632d52c6f72bc177991367b8e146fd76b0fb3a68bc8ee0622be9fa4df8a8" + }, + { + "aggregate_score": 1.0, + "eval_id": "val_stable_faq", + "execution_errors": [], + "expected_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "expected_tool_calls": [], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [], + "tool_responses": [], + "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" + }, + { + "aggregate_score": 0.0, + "eval_id": "val_json_generalization", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0", + "metric fake_rubric_score did not meet threshold 0.75", + "invalid JSON response", + "actual response: invalid JSON response" + ], + "failure_types": [ + "final_response_mismatch", + "format_violation" + ], + "final_response": "订单 B200 正在查询", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": false, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "invalid JSON response" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.0, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "validation", + "tool_calls": [], + "tool_responses": [], + "trace_digest": "sha256:8684ddee9351120c1bf48affbd280c5b73835b2c1d58d04682622b577c37a455" + } + ], + "pass_rate": 0.6666666666666666 + }, + "candidate_train_validation": [ + { + "candidate_id": "candidate_general_fix", + "train": { + "aggregate_score": 0.7666666666666666, + "cases": [ + { + "aggregate_score": 1.0, + "eval_id": "train_tool_argument", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:5da8db545d2c7b26c93058b6619ab49dd59c1348f6b032f05c451c2709b66e6e" + }, + { + "aggregate_score": 0.30000000000000004, + "eval_id": "train_missing_knowledge", + "execution_errors": [], + "expected_response": "{\"answer\": \"未提供该政策，不能猜测。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", + "expected_tool_calls": [], + "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "failure_types": [ + "final_response_mismatch" + ], + "final_response": "{\"answer\": \"我不确定。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "unknown knowledge must explicitly refuse to guess" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [], + "tool_responses": [], + "trace_digest": "sha256:3f059c7504becc8cb98f605431f3c3714a0e70b4e62931895b888bba4ffcf9ed" + }, + { + "aggregate_score": 1.0, + "eval_id": "train_json_format", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:5da8db545d2c7b26c93058b6619ab49dd59c1348f6b032f05c451c2709b66e6e" + } + ], + "pass_rate": 0.6666666666666666 + }, + "validation": { + "aggregate_score": 1.0, + "cases": [ + { + "aggregate_score": 1.0, + "eval_id": "val_refund_critical", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "amount": "12", + "currency": "USD", + "order_id": "R900" + }, + "name": "refund_order" + } + ], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [ + { + "arguments": { + "amount": "12", + "currency": "USD", + "order_id": "R900" + }, + "name": "refund_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:19bb632d52c6f72bc177991367b8e146fd76b0fb3a68bc8ee0622be9fa4df8a8" + }, + { + "aggregate_score": 1.0, + "eval_id": "val_stable_faq", + "execution_errors": [], + "expected_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "expected_tool_calls": [], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [], + "tool_responses": [], + "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" + }, + { + "aggregate_score": 1.0, + "eval_id": "val_json_generalization", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:99d82b8a08b7eeeda5b5f41d08a3afc1c9f4f22879605c257b78458c4ef9e561" + } + ], + "pass_rate": 1.0 + } + }, + { + "candidate_id": "candidate_noop", + "train": { + "aggregate_score": 0.20000000000000004, + "cases": [ + { + "aggregate_score": 0.30000000000000004, + "eval_id": "train_tool_argument", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "failure_types": [ + "final_response_mismatch" + ], + "final_response": "{\"answer\": \"正在查询订单。\", \"arguments\": {}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "required tool arguments do not match expected response" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": {}, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:f086f4b22284734f1462eee58a4d204a31361c26d67449e5b23d292aaf2642bf" + }, + { + "aggregate_score": 0.30000000000000004, + "eval_id": "train_missing_knowledge", + "execution_errors": [], + "expected_response": "{\"answer\": \"未提供该政策，不能猜测。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", + "expected_tool_calls": [], + "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "failure_types": [ + "final_response_mismatch" + ], + "final_response": "{\"answer\": \"我不确定。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "unknown knowledge must explicitly refuse to guess" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [], + "tool_responses": [], + "trace_digest": "sha256:3f059c7504becc8cb98f605431f3c3714a0e70b4e62931895b888bba4ffcf9ed" + }, + { + "aggregate_score": 0.0, + "eval_id": "train_json_format", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0", + "metric fake_rubric_score did not meet threshold 0.75", + "invalid JSON response", + "actual response: invalid JSON response" + ], + "failure_types": [ + "final_response_mismatch", + "format_violation" + ], + "final_response": "订单 A100 正在查询", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": false, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "invalid JSON response" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.0, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [], + "tool_responses": [], + "trace_digest": "sha256:65fdc3d48fd18eb37b2aa5138ee27435257ecbd0ef582c523a42673d87b04b77" + } + ], + "pass_rate": 0.0 + }, + "validation": { + "aggregate_score": 0.6666666666666666, + "cases": [ + { + "aggregate_score": 1.0, + "eval_id": "val_refund_critical", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "amount": "12", + "currency": "USD", + "order_id": "R900" + }, + "name": "refund_order" + } + ], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [ + { + "arguments": { + "amount": "12", + "currency": "USD", + "order_id": "R900" + }, + "name": "refund_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:19bb632d52c6f72bc177991367b8e146fd76b0fb3a68bc8ee0622be9fa4df8a8" + }, + { + "aggregate_score": 1.0, + "eval_id": "val_stable_faq", + "execution_errors": [], + "expected_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "expected_tool_calls": [], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [], + "tool_responses": [], + "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" + }, + { + "aggregate_score": 0.0, + "eval_id": "val_json_generalization", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0", + "metric fake_rubric_score did not meet threshold 0.75", + "invalid JSON response", + "actual response: invalid JSON response" + ], + "failure_types": [ + "final_response_mismatch", + "format_violation" + ], + "final_response": "订单 B200 正在查询", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": false, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "invalid JSON response" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.0, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "validation", + "tool_calls": [], + "tool_responses": [], + "trace_digest": "sha256:8684ddee9351120c1bf48affbd280c5b73835b2c1d58d04682622b577c37a455" + } + ], + "pass_rate": 0.6666666666666666 + } + }, + { + "candidate_id": "candidate_overfit", + "train": { + "aggregate_score": 1.0, + "cases": [ + { + "aggregate_score": 1.0, + "eval_id": "train_tool_argument", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:5da8db545d2c7b26c93058b6619ab49dd59c1348f6b032f05c451c2709b66e6e" + }, + { + "aggregate_score": 1.0, + "eval_id": "train_missing_knowledge", + "execution_errors": [], + "expected_response": "{\"answer\": \"未提供该政策，不能猜测。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", + "expected_tool_calls": [], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"未提供该政策，不能猜测。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "train", + "tool_calls": [], + "tool_responses": [], + "trace_digest": "sha256:22d90b34cb10f26e6a207ed065e82b862b238989d23eda1b8dbd8d4072208e0f" + }, + { + "aggregate_score": 1.0, + "eval_id": "train_json_format", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:5da8db545d2c7b26c93058b6619ab49dd59c1348f6b032f05c451c2709b66e6e" + } + ], + "pass_rate": 1.0 + }, + "validation": { + "aggregate_score": 0.7333333333333334, + "cases": [ + { + "aggregate_score": 0.2, + "eval_id": "val_refund_critical", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "amount": "12", + "currency": "USD", + "order_id": "R900" + }, + "name": "refund_order" + } + ], + "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0", + "metric fake_rubric_score did not meet threshold 0.75", + "route or tool does not match expected response; required tool arguments do not match expected response" + ], + "failure_types": [ + "final_response_mismatch", + "tool_argument_error", + "tool_selection_error" + ], + "final_response": "{\"answer\": \"正在查询订单 R900。\", \"arguments\": {\"order_id\": \"R900\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": false, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "route or tool does not match expected response; required tool arguments do not match expected response" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.5, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "validation", + "tool_calls": [ + { + "arguments": { + "order_id": "R900" + }, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:254559dde2e80e22b22f6c4cb0d37c61bbd5a8200e19d46232c76f1b0a5d8604" + }, + { + "aggregate_score": 1.0, + "eval_id": "val_stable_faq", + "execution_errors": [], + "expected_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "expected_tool_calls": [], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [], + "tool_responses": [], + "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" + }, + { + "aggregate_score": 1.0, + "eval_id": "val_json_generalization", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:99d82b8a08b7eeeda5b5f41d08a3afc1c9f4f22879605c257b78458c4ef9e561" + } + ], + "pass_rate": 0.6666666666666666 + } + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/sample_output/audit/raw_reports.json b/examples/optimization/eval_optimize_loop/sample_output/audit/raw_reports.json new file mode 100644 index 00000000..0105f68c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/audit/raw_reports.json @@ -0,0 +1,292 @@ +{ + "baseline_train": { + "aggregate_score": 0.20000000000000004, + "cases": [ + { + "aggregate_score": 0.30000000000000004, + "eval_id": "train_tool_argument", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "failure_types": [ + "final_response_mismatch" + ], + "final_response": "{\"answer\": \"正在查询订单。\", \"arguments\": {}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "required tool arguments do not match expected response" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": {}, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:f086f4b22284734f1462eee58a4d204a31361c26d67449e5b23d292aaf2642bf" + }, + { + "aggregate_score": 0.30000000000000004, + "eval_id": "train_missing_knowledge", + "execution_errors": [], + "expected_response": "{\"answer\": \"未提供该政策，不能猜测。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", + "expected_tool_calls": [], + "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "failure_types": [ + "final_response_mismatch" + ], + "final_response": "{\"answer\": \"我不确定。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "unknown knowledge must explicitly refuse to guess" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [], + "tool_responses": [], + "trace_digest": "sha256:3f059c7504becc8cb98f605431f3c3714a0e70b4e62931895b888bba4ffcf9ed" + }, + { + "aggregate_score": 0.0, + "eval_id": "train_json_format", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0", + "metric fake_rubric_score did not meet threshold 0.75", + "invalid JSON response", + "actual response: invalid JSON response" + ], + "failure_types": [ + "final_response_mismatch", + "format_violation" + ], + "final_response": "订单 A100 正在查询", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": false, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "invalid JSON response" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.0, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [], + "tool_responses": [], + "trace_digest": "sha256:65fdc3d48fd18eb37b2aa5138ee27435257ecbd0ef582c523a42673d87b04b77" + } + ], + "pass_rate": 0.0 + }, + "baseline_validation": { + "aggregate_score": 0.6666666666666666, + "cases": [ + { + "aggregate_score": 1.0, + "eval_id": "val_refund_critical", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "amount": "12", + "currency": "USD", + "order_id": "R900" + }, + "name": "refund_order" + } + ], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [ + { + "arguments": { + "amount": "12", + "currency": "USD", + "order_id": "R900" + }, + "name": "refund_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:19bb632d52c6f72bc177991367b8e146fd76b0fb3a68bc8ee0622be9fa4df8a8" + }, + { + "aggregate_score": 1.0, + "eval_id": "val_stable_faq", + "execution_errors": [], + "expected_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "expected_tool_calls": [], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [], + "tool_responses": [], + "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" + }, + { + "aggregate_score": 0.0, + "eval_id": "val_json_generalization", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0", + "metric fake_rubric_score did not meet threshold 0.75", + "invalid JSON response", + "actual response: invalid JSON response" + ], + "failure_types": [ + "final_response_mismatch", + "format_violation" + ], + "final_response": "订单 B200 正在查询", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": false, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "invalid JSON response" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.0, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "validation", + "tool_calls": [], + "tool_responses": [], + "trace_digest": "sha256:8684ddee9351120c1bf48affbd280c5b73835b2c1d58d04682622b577c37a455" + } + ], + "pass_rate": 0.6666666666666666 + } +} diff --git a/examples/optimization/eval_optimize_loop/sample_output/environment.snapshot.json b/examples/optimization/eval_optimize_loop/sample_output/environment.snapshot.json new file mode 100644 index 00000000..8f671e94 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/environment.snapshot.json @@ -0,0 +1 @@ +{"dependencies":{"pydantic":"2.11.3","pytest":"8.4.2"},"mode":"fake","platform":"Windows-11-10.0.26200-SP0","python_version":"3.12.5","sdk_commit":"9cf5ac409b8bc7091d08250d826d6cf30ca1229e","sdk_version":"1.1.11","seed":42} diff --git a/examples/optimization/eval_optimize_loop/sample_output/input.snapshot.json b/examples/optimization/eval_optimize_loop/sample_output/input.snapshot.json new file mode 100644 index 00000000..16de8249 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/input.snapshot.json @@ -0,0 +1 @@ +{"config":{"evaluate":{"metrics":[{"criterion":{"final_response":{"text":{"match":"exact"}}},"metric_name":"final_response_avg_score","threshold":1.0},{"metric_name":"fake_rubric_score","threshold":0.75}],"num_runs":1},"optimize":{"algorithm":{"max_metric_calls":10,"name":"gepa_reflective","reflection_lm":{"api_key":"***REDACTED***","base_url":"${TRPC_AGENT_BASE_URL}","model_name":"${TRPC_AGENT_MODEL_NAME}"},"seed":42},"eval_case_parallelism":1,"stop":{"required_metrics":"all"}},"pipeline":{"gate":{"critical_case_ids":["val_refund_critical"],"min_validation_score_delta":0.05},"reproducibility":{"seed":42},"write_back_when_accepted":false}},"digests":{"config_digest":"sha256:a7abcd7250f870b3bfc620c9760bc312cb0da0abe805094200e04c425f0b0090","prompt_digest":"sha256:17f9e974f8c43579bb992408b7f904c4523a0adbd40748a03c4636777ddb6cb9","train_dataset_digest":"sha256:a4e41e0da27d8533d6c02b11191f55a0be951fa65e77cca052d919a003ee7d1f","validation_dataset_digest":"sha256:5abab3c917b8248b6a0ebae536be4cfe6a668b7e659379f75fcb8f0d3e4a5c7d"}} diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json index b38dc8ff..6c208381 100644 --- a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json +++ b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json @@ -1,36 +1,111 @@ { + "attribution_summary": {}, + "audit_references": { + "candidate_reports_path": "audit\\candidate_reports.json", + "config_snapshot_path": "input.snapshot.json", + "environment_snapshot_path": "environment.snapshot.json", + "gate_decisions_path": "audit\\gate_decisions.json", + "input_snapshot_path": "input.snapshot.json", + "normalized_reports_path": "audit\\normalized_reports.json", + "raw_reports_path": "audit\\raw_reports.json" + }, "baseline_train": { - "aggregate_score": 0.0, + "aggregate_score": 0.20000000000000004, "cases": [ { - "aggregate_score": 0.0, + "aggregate_score": 0.30000000000000004, + "eval_id": "train_tool_argument", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "failure_types": [ + "final_response_mismatch" + ], + "final_response": "{\"answer\": \"正在查询订单。\", \"arguments\": {}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "required tool arguments do not match expected response" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": {}, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:f086f4b22284734f1462eee58a4d204a31361c26d67449e5b23d292aaf2642bf" + }, + { + "aggregate_score": 0.30000000000000004, "eval_id": "train_missing_knowledge", + "execution_errors": [], "expected_response": "{\"answer\": \"未提供该政策，不能猜测。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", "expected_tool_calls": [], + "failure_attribution": null, "failure_reasons": [ "metric final_response_avg_score did not meet threshold 1.0" ], + "failure_types": [ + "final_response_mismatch" + ], "final_response": "{\"answer\": \"我不确定。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", "hard_failed": false, "metric_passed": { + "fake_rubric_score": true, "final_response_avg_score": false }, - "metric_reasons": {}, + "metric_reasons": { + "fake_rubric_score": [ + "unknown knowledge must explicitly refuse to guess" + ] + }, "metric_scores": { + "fake_rubric_score": 0.75, "final_response_avg_score": 0.0 }, "metric_thresholds": { + "fake_rubric_score": 0.75, "final_response_avg_score": 1.0 }, "passed": false, "run_count": 1, "split": "train", "tool_calls": [], + "tool_responses": [], "trace_digest": "sha256:3f059c7504becc8cb98f605431f3c3714a0e70b4e62931895b888bba4ffcf9ed" }, { "aggregate_score": 0.0, "eval_id": "train_json_format", + "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ { @@ -40,98 +115,141 @@ "name": "lookup_order" } ], + "failure_attribution": null, "failure_reasons": [ - "metric final_response_avg_score did not meet threshold 1.0" + "metric final_response_avg_score did not meet threshold 1.0", + "metric fake_rubric_score did not meet threshold 0.75", + "invalid JSON response", + "actual response: invalid JSON response" + ], + "failure_types": [ + "final_response_mismatch", + "format_violation" ], "final_response": "订单 A100 正在查询", "hard_failed": false, "metric_passed": { + "fake_rubric_score": false, "final_response_avg_score": false }, - "metric_reasons": {}, + "metric_reasons": { + "fake_rubric_score": [ + "invalid JSON response" + ] + }, "metric_scores": { + "fake_rubric_score": 0.0, "final_response_avg_score": 0.0 }, "metric_thresholds": { + "fake_rubric_score": 0.75, "final_response_avg_score": 1.0 }, "passed": false, "run_count": 1, "split": "train", "tool_calls": [], + "tool_responses": [], "trace_digest": "sha256:65fdc3d48fd18eb37b2aa5138ee27435257ecbd0ef582c523a42673d87b04b77" - }, + } + ], + "pass_rate": 0.0 + }, + "baseline_validation": { + "aggregate_score": 0.6666666666666666, + "cases": [ { - "aggregate_score": 0.0, - "eval_id": "train_tool_argument", - "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "aggregate_score": 1.0, + "eval_id": "val_refund_critical", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", "expected_tool_calls": [ { "arguments": { - "order_id": "A100" + "amount": "12", + "currency": "USD", + "order_id": "R900" }, - "name": "lookup_order" + "name": "refund_order" } ], - "failure_reasons": [ - "metric final_response_avg_score did not meet threshold 1.0" - ], - "final_response": "{\"answer\": \"正在查询订单。\", \"arguments\": {}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", "hard_failed": false, "metric_passed": { - "final_response_avg_score": false + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] }, - "metric_reasons": {}, "metric_scores": { - "final_response_avg_score": 0.0 + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 }, "metric_thresholds": { + "fake_rubric_score": 0.75, "final_response_avg_score": 1.0 }, - "passed": false, + "passed": true, "run_count": 1, - "split": "train", + "split": "validation", "tool_calls": [ { - "arguments": {}, - "name": "lookup_order" + "arguments": { + "amount": "12", + "currency": "USD", + "order_id": "R900" + }, + "name": "refund_order" } ], - "trace_digest": "sha256:f086f4b22284734f1462eee58a4d204a31361c26d67449e5b23d292aaf2642bf" - } - ], - "pass_rate": 0.0 - }, - "baseline_validation": { - "aggregate_score": 0.6666666666666666, - "cases": [ + "tool_responses": [], + "trace_digest": "sha256:19bb632d52c6f72bc177991367b8e146fd76b0fb3a68bc8ee0622be9fa4df8a8" + }, { "aggregate_score": 1.0, "eval_id": "val_stable_faq", + "execution_errors": [], "expected_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", "expected_tool_calls": [], + "failure_attribution": null, "failure_reasons": [], + "failure_types": [], "final_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", "hard_failed": false, "metric_passed": { + "fake_rubric_score": true, "final_response_avg_score": true }, - "metric_reasons": {}, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, "metric_scores": { + "fake_rubric_score": 1.0, "final_response_avg_score": 1.0 }, "metric_thresholds": { + "fake_rubric_score": 0.75, "final_response_avg_score": 1.0 }, "passed": true, "run_count": 1, "split": "validation", "tool_calls": [], + "tool_responses": [], "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" }, { "aggregate_score": 0.0, "eval_id": "val_json_generalization", + "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ { @@ -141,68 +259,42 @@ "name": "lookup_order" } ], + "failure_attribution": null, "failure_reasons": [ - "metric final_response_avg_score did not meet threshold 1.0" + "metric final_response_avg_score did not meet threshold 1.0", + "metric fake_rubric_score did not meet threshold 0.75", + "invalid JSON response", + "actual response: invalid JSON response" + ], + "failure_types": [ + "final_response_mismatch", + "format_violation" ], "final_response": "订单 B200 正在查询", "hard_failed": false, "metric_passed": { + "fake_rubric_score": false, "final_response_avg_score": false }, - "metric_reasons": {}, + "metric_reasons": { + "fake_rubric_score": [ + "invalid JSON response" + ] + }, "metric_scores": { + "fake_rubric_score": 0.0, "final_response_avg_score": 0.0 }, "metric_thresholds": { + "fake_rubric_score": 0.75, "final_response_avg_score": 1.0 }, "passed": false, "run_count": 1, "split": "validation", "tool_calls": [], + "tool_responses": [], "trace_digest": "sha256:8684ddee9351120c1bf48affbd280c5b73835b2c1d58d04682622b577c37a455" - }, - { - "aggregate_score": 1.0, - "eval_id": "val_refund_critical", - "expected_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", - "expected_tool_calls": [ - { - "arguments": { - "amount": "12", - "currency": "USD", - "order_id": "R900" - }, - "name": "refund_order" - } - ], - "failure_reasons": [], - "final_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", - "hard_failed": false, - "metric_passed": { - "final_response_avg_score": true - }, - "metric_reasons": {}, - "metric_scores": { - "final_response_avg_score": 1.0 - }, - "metric_thresholds": { - "final_response_avg_score": 1.0 - }, - "passed": true, - "run_count": 1, - "split": "validation", - "tool_calls": [ - { - "arguments": { - "amount": "12", - "currency": "USD", - "order_id": "R900" - }, - "name": "refund_order" - } - ], - "trace_digest": "sha256:19bb632d52c6f72bc177991367b8e146fd76b0fb3a68bc8ee0622be9fa4df8a8" } ], "pass_rate": 0.6666666666666666 @@ -211,6 +303,7 @@ { "accepted": true, "candidate_id": "candidate_general_fix", + "duration_seconds": null, "gate": { "accepted": true, "reasons": [ @@ -218,6 +311,27 @@ ], "risk_level": "low", "rules": [ + { + "actual": "complete", + "expected": "complete", + "passed": true, + "reason": "independent validation evaluation is incomplete", + "rule": "evaluation_complete" + }, + { + "actual": 0.33333333333333337, + "expected": "number", + "passed": true, + "reason": "validation aggregate score delta is required", + "rule": "validation_score_delta_available" + }, + { + "actual": 0.33333333333333337, + "expected": "number", + "passed": true, + "reason": "validation pass-rate delta is required", + "rule": "validation_pass_rate_delta_available" + }, { "actual": 0.33333333333333337, "expected": 0.05, @@ -259,44 +373,120 @@ "passed": true, "reason": "train improvement with validation decline is rejected", "rule": "no_overfit" + }, + { + "actual": false, + "expected": false, + "passed": true, + "reason": "tie policy rejects a non-improving validation outcome", + "rule": "tie_policy" } + ], + "warnings": [ + "generation cost is unknown" ] }, + "generation_cost_usd": null, + "independently_evaluated": true, "reasons": [ "candidate passed all independent gate rules" ], + "source": "fixture", "train": { - "aggregate_score": 0.6666666666666666, + "aggregate_score": 0.7666666666666666, "cases": [ { - "aggregate_score": 0.0, + "aggregate_score": 1.0, + "eval_id": "train_tool_argument", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:5da8db545d2c7b26c93058b6619ab49dd59c1348f6b032f05c451c2709b66e6e" + }, + { + "aggregate_score": 0.30000000000000004, "eval_id": "train_missing_knowledge", + "execution_errors": [], "expected_response": "{\"answer\": \"未提供该政策，不能猜测。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", "expected_tool_calls": [], + "failure_attribution": null, "failure_reasons": [ "metric final_response_avg_score did not meet threshold 1.0" ], + "failure_types": [ + "final_response_mismatch" + ], "final_response": "{\"answer\": \"我不确定。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", "hard_failed": false, "metric_passed": { + "fake_rubric_score": true, "final_response_avg_score": false }, - "metric_reasons": {}, + "metric_reasons": { + "fake_rubric_score": [ + "unknown knowledge must explicitly refuse to guess" + ] + }, "metric_scores": { + "fake_rubric_score": 0.75, "final_response_avg_score": 0.0 }, "metric_thresholds": { + "fake_rubric_score": 0.75, "final_response_avg_score": 1.0 }, "passed": false, "run_count": 1, "split": "train", "tool_calls": [], + "tool_responses": [], "trace_digest": "sha256:3f059c7504becc8cb98f605431f3c3714a0e70b4e62931895b888bba4ffcf9ed" }, { "aggregate_score": 1.0, "eval_id": "train_json_format", + "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ { @@ -306,17 +496,26 @@ "name": "lookup_order" } ], + "failure_attribution": null, "failure_reasons": [], + "failure_types": [], "final_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "hard_failed": false, "metric_passed": { + "fake_rubric_score": true, "final_response_avg_score": true }, - "metric_reasons": {}, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, "metric_scores": { + "fake_rubric_score": 1.0, "final_response_avg_score": 1.0 }, "metric_thresholds": { + "fake_rubric_score": 0.75, "final_response_avg_score": 1.0 }, "passed": true, @@ -330,79 +529,107 @@ "name": "lookup_order" } ], + "tool_responses": [], "trace_digest": "sha256:5da8db545d2c7b26c93058b6619ab49dd59c1348f6b032f05c451c2709b66e6e" - }, + } + ], + "pass_rate": 0.6666666666666666 + }, + "validation": { + "aggregate_score": 1.0, + "cases": [ { "aggregate_score": 1.0, - "eval_id": "train_tool_argument", - "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "eval_id": "val_refund_critical", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", "expected_tool_calls": [ { "arguments": { - "order_id": "A100" + "amount": "12", + "currency": "USD", + "order_id": "R900" }, - "name": "lookup_order" + "name": "refund_order" } ], + "failure_attribution": null, "failure_reasons": [], - "final_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "failure_types": [], + "final_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", "hard_failed": false, "metric_passed": { + "fake_rubric_score": true, "final_response_avg_score": true }, - "metric_reasons": {}, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, "metric_scores": { + "fake_rubric_score": 1.0, "final_response_avg_score": 1.0 }, "metric_thresholds": { + "fake_rubric_score": 0.75, "final_response_avg_score": 1.0 }, "passed": true, "run_count": 1, - "split": "train", + "split": "validation", "tool_calls": [ { "arguments": { - "order_id": "A100" + "amount": "12", + "currency": "USD", + "order_id": "R900" }, - "name": "lookup_order" + "name": "refund_order" } ], - "trace_digest": "sha256:5da8db545d2c7b26c93058b6619ab49dd59c1348f6b032f05c451c2709b66e6e" - } - ], - "pass_rate": 0.6666666666666666 - }, - "validation": { - "aggregate_score": 1.0, - "cases": [ + "tool_responses": [], + "trace_digest": "sha256:19bb632d52c6f72bc177991367b8e146fd76b0fb3a68bc8ee0622be9fa4df8a8" + }, { "aggregate_score": 1.0, "eval_id": "val_stable_faq", + "execution_errors": [], "expected_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", "expected_tool_calls": [], + "failure_attribution": null, "failure_reasons": [], + "failure_types": [], "final_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", "hard_failed": false, "metric_passed": { + "fake_rubric_score": true, "final_response_avg_score": true }, - "metric_reasons": {}, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, "metric_scores": { + "fake_rubric_score": 1.0, "final_response_avg_score": 1.0 }, "metric_thresholds": { + "fake_rubric_score": 0.75, "final_response_avg_score": 1.0 }, "passed": true, "run_count": 1, "split": "validation", "tool_calls": [], + "tool_responses": [], "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" }, { "aggregate_score": 1.0, "eval_id": "val_json_generalization", + "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ { @@ -412,17 +639,26 @@ "name": "lookup_order" } ], + "failure_attribution": null, "failure_reasons": [], + "failure_types": [], "final_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "hard_failed": false, "metric_passed": { + "fake_rubric_score": true, "final_response_avg_score": true }, - "metric_reasons": {}, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, "metric_scores": { + "fake_rubric_score": 1.0, "final_response_avg_score": 1.0 }, "metric_thresholds": { + "fake_rubric_score": 0.75, "final_response_avg_score": 1.0 }, "passed": true, @@ -436,54 +672,30 @@ "name": "lookup_order" } ], + "tool_responses": [], "trace_digest": "sha256:99d82b8a08b7eeeda5b5f41d08a3afc1c9f4f22879605c257b78458c4ef9e561" - }, - { - "aggregate_score": 1.0, - "eval_id": "val_refund_critical", - "expected_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", - "expected_tool_calls": [ - { - "arguments": { - "amount": "12", - "currency": "USD", - "order_id": "R900" - }, - "name": "refund_order" - } - ], - "failure_reasons": [], - "final_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", - "hard_failed": false, - "metric_passed": { - "final_response_avg_score": true - }, - "metric_reasons": {}, - "metric_scores": { - "final_response_avg_score": 1.0 - }, - "metric_thresholds": { - "final_response_avg_score": 1.0 - }, - "passed": true, - "run_count": 1, - "split": "validation", - "tool_calls": [ - { - "arguments": { - "amount": "12", - "currency": "USD", - "order_id": "R900" - }, - "name": "refund_order" - } - ], - "trace_digest": "sha256:19bb632d52c6f72bc177991367b8e146fd76b0fb3a68bc8ee0622be9fa4df8a8" } ], "pass_rate": 1.0 }, "validation_case_deltas": [ + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_passed": true, + "candidate_score": 1.0, + "critical": true, + "eval_id": "val_refund_critical", + "hard_fail_added": false, + "metric_deltas": { + "fake_rubric_score": 0.0, + "final_response_avg_score": 0.0 + }, + "new_failure_types": [], + "resolved_failure_types": [], + "score_delta": 0.0, + "transition": "UNCHANGED" + }, { "baseline_passed": true, "baseline_score": 1.0, @@ -493,8 +705,11 @@ "eval_id": "val_stable_faq", "hard_fail_added": false, "metric_deltas": { + "fake_rubric_score": 0.0, "final_response_avg_score": 0.0 }, + "new_failure_types": [], + "resolved_failure_types": [], "score_delta": 0.0, "transition": "UNCHANGED" }, @@ -507,37 +722,52 @@ "eval_id": "val_json_generalization", "hard_fail_added": false, "metric_deltas": { + "fake_rubric_score": 1.0, "final_response_avg_score": 1.0 }, + "new_failure_types": [], + "resolved_failure_types": [ + "final_response_mismatch", + "format_violation" + ], "score_delta": 1.0, "transition": "NEW_PASS" - }, - { - "baseline_passed": true, - "baseline_score": 1.0, - "candidate_passed": true, - "candidate_score": 1.0, - "critical": true, - "eval_id": "val_refund_critical", - "hard_fail_added": false, - "metric_deltas": { - "final_response_avg_score": 0.0 - }, - "score_delta": 0.0, - "transition": "UNCHANGED" } ] }, { "accepted": false, "candidate_id": "candidate_noop", + "duration_seconds": null, "gate": { "accepted": false, "reasons": [ - "validation aggregate score must improve" + "validation aggregate score must improve", + "tie policy rejects a non-improving validation outcome" ], "risk_level": "medium", "rules": [ + { + "actual": "complete", + "expected": "complete", + "passed": true, + "reason": "independent validation evaluation is incomplete", + "rule": "evaluation_complete" + }, + { + "actual": 0.0, + "expected": "number", + "passed": true, + "reason": "validation aggregate score delta is required", + "rule": "validation_score_delta_available" + }, + { + "actual": 0.0, + "expected": "number", + "passed": true, + "reason": "validation pass-rate delta is required", + "rule": "validation_pass_rate_delta_available" + }, { "actual": 0.0, "expected": 0.05, @@ -579,44 +809,123 @@ "passed": true, "reason": "train improvement with validation decline is rejected", "rule": "no_overfit" + }, + { + "actual": true, + "expected": false, + "passed": false, + "reason": "tie policy rejects a non-improving validation outcome", + "rule": "tie_policy" } + ], + "warnings": [ + "generation cost is unknown" ] }, + "generation_cost_usd": null, + "independently_evaluated": true, "reasons": [ - "validation aggregate score must improve" + "validation aggregate score must improve", + "tie policy rejects a non-improving validation outcome" ], + "source": "fixture", "train": { - "aggregate_score": 0.0, + "aggregate_score": 0.20000000000000004, "cases": [ { - "aggregate_score": 0.0, + "aggregate_score": 0.30000000000000004, + "eval_id": "train_tool_argument", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "failure_types": [ + "final_response_mismatch" + ], + "final_response": "{\"answer\": \"正在查询订单。\", \"arguments\": {}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "required tool arguments do not match expected response" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": {}, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:f086f4b22284734f1462eee58a4d204a31361c26d67449e5b23d292aaf2642bf" + }, + { + "aggregate_score": 0.30000000000000004, "eval_id": "train_missing_knowledge", + "execution_errors": [], "expected_response": "{\"answer\": \"未提供该政策，不能猜测。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", "expected_tool_calls": [], + "failure_attribution": null, "failure_reasons": [ "metric final_response_avg_score did not meet threshold 1.0" ], + "failure_types": [ + "final_response_mismatch" + ], "final_response": "{\"answer\": \"我不确定。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", "hard_failed": false, "metric_passed": { + "fake_rubric_score": true, "final_response_avg_score": false }, - "metric_reasons": {}, + "metric_reasons": { + "fake_rubric_score": [ + "unknown knowledge must explicitly refuse to guess" + ] + }, "metric_scores": { + "fake_rubric_score": 0.75, "final_response_avg_score": 0.0 }, "metric_thresholds": { + "fake_rubric_score": 0.75, "final_response_avg_score": 1.0 }, "passed": false, "run_count": 1, "split": "train", "tool_calls": [], + "tool_responses": [], "trace_digest": "sha256:3f059c7504becc8cb98f605431f3c3714a0e70b4e62931895b888bba4ffcf9ed" }, { "aggregate_score": 0.0, "eval_id": "train_json_format", + "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ { @@ -626,98 +935,141 @@ "name": "lookup_order" } ], + "failure_attribution": null, "failure_reasons": [ - "metric final_response_avg_score did not meet threshold 1.0" + "metric final_response_avg_score did not meet threshold 1.0", + "metric fake_rubric_score did not meet threshold 0.75", + "invalid JSON response", + "actual response: invalid JSON response" + ], + "failure_types": [ + "final_response_mismatch", + "format_violation" ], "final_response": "订单 A100 正在查询", "hard_failed": false, "metric_passed": { + "fake_rubric_score": false, "final_response_avg_score": false }, - "metric_reasons": {}, + "metric_reasons": { + "fake_rubric_score": [ + "invalid JSON response" + ] + }, "metric_scores": { + "fake_rubric_score": 0.0, "final_response_avg_score": 0.0 }, "metric_thresholds": { + "fake_rubric_score": 0.75, "final_response_avg_score": 1.0 }, "passed": false, "run_count": 1, "split": "train", "tool_calls": [], + "tool_responses": [], "trace_digest": "sha256:65fdc3d48fd18eb37b2aa5138ee27435257ecbd0ef582c523a42673d87b04b77" - }, + } + ], + "pass_rate": 0.0 + }, + "validation": { + "aggregate_score": 0.6666666666666666, + "cases": [ { - "aggregate_score": 0.0, - "eval_id": "train_tool_argument", - "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "aggregate_score": 1.0, + "eval_id": "val_refund_critical", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", "expected_tool_calls": [ { "arguments": { - "order_id": "A100" + "amount": "12", + "currency": "USD", + "order_id": "R900" }, - "name": "lookup_order" + "name": "refund_order" } ], - "failure_reasons": [ - "metric final_response_avg_score did not meet threshold 1.0" - ], - "final_response": "{\"answer\": \"正在查询订单。\", \"arguments\": {}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", "hard_failed": false, "metric_passed": { - "final_response_avg_score": false + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] }, - "metric_reasons": {}, "metric_scores": { - "final_response_avg_score": 0.0 + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 }, "metric_thresholds": { + "fake_rubric_score": 0.75, "final_response_avg_score": 1.0 }, - "passed": false, + "passed": true, "run_count": 1, - "split": "train", + "split": "validation", "tool_calls": [ { - "arguments": {}, - "name": "lookup_order" + "arguments": { + "amount": "12", + "currency": "USD", + "order_id": "R900" + }, + "name": "refund_order" } ], - "trace_digest": "sha256:f086f4b22284734f1462eee58a4d204a31361c26d67449e5b23d292aaf2642bf" - } - ], - "pass_rate": 0.0 - }, - "validation": { - "aggregate_score": 0.6666666666666666, - "cases": [ + "tool_responses": [], + "trace_digest": "sha256:19bb632d52c6f72bc177991367b8e146fd76b0fb3a68bc8ee0622be9fa4df8a8" + }, { "aggregate_score": 1.0, "eval_id": "val_stable_faq", + "execution_errors": [], "expected_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", "expected_tool_calls": [], + "failure_attribution": null, "failure_reasons": [], + "failure_types": [], "final_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", "hard_failed": false, "metric_passed": { + "fake_rubric_score": true, "final_response_avg_score": true }, - "metric_reasons": {}, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, "metric_scores": { + "fake_rubric_score": 1.0, "final_response_avg_score": 1.0 }, "metric_thresholds": { + "fake_rubric_score": 0.75, "final_response_avg_score": 1.0 }, "passed": true, "run_count": 1, "split": "validation", "tool_calls": [], + "tool_responses": [], "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" }, { "aggregate_score": 0.0, "eval_id": "val_json_generalization", + "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ { @@ -727,68 +1079,42 @@ "name": "lookup_order" } ], + "failure_attribution": null, "failure_reasons": [ - "metric final_response_avg_score did not meet threshold 1.0" + "metric final_response_avg_score did not meet threshold 1.0", + "metric fake_rubric_score did not meet threshold 0.75", + "invalid JSON response", + "actual response: invalid JSON response" + ], + "failure_types": [ + "final_response_mismatch", + "format_violation" ], "final_response": "订单 B200 正在查询", "hard_failed": false, "metric_passed": { + "fake_rubric_score": false, "final_response_avg_score": false }, - "metric_reasons": {}, + "metric_reasons": { + "fake_rubric_score": [ + "invalid JSON response" + ] + }, "metric_scores": { + "fake_rubric_score": 0.0, "final_response_avg_score": 0.0 }, "metric_thresholds": { + "fake_rubric_score": 0.75, "final_response_avg_score": 1.0 }, "passed": false, "run_count": 1, "split": "validation", "tool_calls": [], + "tool_responses": [], "trace_digest": "sha256:8684ddee9351120c1bf48affbd280c5b73835b2c1d58d04682622b577c37a455" - }, - { - "aggregate_score": 1.0, - "eval_id": "val_refund_critical", - "expected_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", - "expected_tool_calls": [ - { - "arguments": { - "amount": "12", - "currency": "USD", - "order_id": "R900" - }, - "name": "refund_order" - } - ], - "failure_reasons": [], - "final_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", - "hard_failed": false, - "metric_passed": { - "final_response_avg_score": true - }, - "metric_reasons": {}, - "metric_scores": { - "final_response_avg_score": 1.0 - }, - "metric_thresholds": { - "final_response_avg_score": 1.0 - }, - "passed": true, - "run_count": 1, - "split": "validation", - "tool_calls": [ - { - "arguments": { - "amount": "12", - "currency": "USD", - "order_id": "R900" - }, - "name": "refund_order" - } - ], - "trace_digest": "sha256:19bb632d52c6f72bc177991367b8e146fd76b0fb3a68bc8ee0622be9fa4df8a8" } ], "pass_rate": 0.6666666666666666 @@ -799,40 +1125,49 @@ "baseline_score": 1.0, "candidate_passed": true, "candidate_score": 1.0, - "critical": false, - "eval_id": "val_stable_faq", + "critical": true, + "eval_id": "val_refund_critical", "hard_fail_added": false, "metric_deltas": { + "fake_rubric_score": 0.0, "final_response_avg_score": 0.0 }, + "new_failure_types": [], + "resolved_failure_types": [], "score_delta": 0.0, "transition": "UNCHANGED" }, { - "baseline_passed": false, - "baseline_score": 0.0, - "candidate_passed": false, - "candidate_score": 0.0, + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_passed": true, + "candidate_score": 1.0, "critical": false, - "eval_id": "val_json_generalization", + "eval_id": "val_stable_faq", "hard_fail_added": false, "metric_deltas": { + "fake_rubric_score": 0.0, "final_response_avg_score": 0.0 }, + "new_failure_types": [], + "resolved_failure_types": [], "score_delta": 0.0, "transition": "UNCHANGED" }, { - "baseline_passed": true, - "baseline_score": 1.0, - "candidate_passed": true, - "candidate_score": 1.0, - "critical": true, - "eval_id": "val_refund_critical", + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_passed": false, + "candidate_score": 0.0, + "critical": false, + "eval_id": "val_json_generalization", "hard_fail_added": false, "metric_deltas": { + "fake_rubric_score": 0.0, "final_response_avg_score": 0.0 }, + "new_failure_types": [], + "resolved_failure_types": [], "score_delta": 0.0, "transition": "UNCHANGED" } @@ -841,20 +1176,41 @@ { "accepted": false, "candidate_id": "candidate_overfit", + "duration_seconds": null, "gate": { "accepted": false, "reasons": [ - "validation aggregate score must improve", "new hard failures are not allowed", "validation regressions exceed the limit", "critical validation cases must not regress" ], "risk_level": "high", "rules": [ + { + "actual": "complete", + "expected": "complete", + "passed": true, + "reason": "independent validation evaluation is incomplete", + "rule": "evaluation_complete" + }, + { + "actual": 0.06666666666666676, + "expected": "number", + "passed": true, + "reason": "validation aggregate score delta is required", + "rule": "validation_score_delta_available" + }, { "actual": 0.0, + "expected": "number", + "passed": true, + "reason": "validation pass-rate delta is required", + "rule": "validation_pass_rate_delta_available" + }, + { + "actual": 0.06666666666666676, "expected": 0.05, - "passed": false, + "passed": true, "reason": "validation aggregate score must improve", "rule": "validation_score_improved" }, @@ -892,45 +1248,118 @@ "passed": true, "reason": "train improvement with validation decline is rejected", "rule": "no_overfit" + }, + { + "actual": false, + "expected": false, + "passed": true, + "reason": "tie policy rejects a non-improving validation outcome", + "rule": "tie_policy" } + ], + "warnings": [ + "generation cost is unknown" ] }, + "generation_cost_usd": null, + "independently_evaluated": true, "reasons": [ - "validation aggregate score must improve", "new hard failures are not allowed", "validation regressions exceed the limit", "critical validation cases must not regress" ], + "source": "fixture", "train": { "aggregate_score": 1.0, "cases": [ + { + "aggregate_score": 1.0, + "eval_id": "train_tool_argument", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": { + "order_id": "A100" + }, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:5da8db545d2c7b26c93058b6619ab49dd59c1348f6b032f05c451c2709b66e6e" + }, { "aggregate_score": 1.0, "eval_id": "train_missing_knowledge", + "execution_errors": [], "expected_response": "{\"answer\": \"未提供该政策，不能猜测。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", "expected_tool_calls": [], + "failure_attribution": null, "failure_reasons": [], + "failure_types": [], "final_response": "{\"answer\": \"未提供该政策，不能猜测。\", \"arguments\": {}, \"route\": \"knowledge_gap\", \"tool\": \"none\"}", "hard_failed": false, "metric_passed": { + "fake_rubric_score": true, "final_response_avg_score": true }, - "metric_reasons": {}, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, "metric_scores": { + "fake_rubric_score": 1.0, "final_response_avg_score": 1.0 }, "metric_thresholds": { + "fake_rubric_score": 0.75, "final_response_avg_score": 1.0 }, "passed": true, "run_count": 1, "split": "train", "tool_calls": [], + "tool_responses": [], "trace_digest": "sha256:22d90b34cb10f26e6a207ed065e82b862b238989d23eda1b8dbd8d4072208e0f" }, { "aggregate_score": 1.0, "eval_id": "train_json_format", + "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ { @@ -940,17 +1369,26 @@ "name": "lookup_order" } ], + "failure_attribution": null, "failure_reasons": [], + "failure_types": [], "final_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "hard_failed": false, "metric_passed": { + "fake_rubric_score": true, "final_response_avg_score": true }, - "metric_reasons": {}, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, "metric_scores": { + "fake_rubric_score": 1.0, "final_response_avg_score": 1.0 }, "metric_thresholds": { + "fake_rubric_score": 0.75, "final_response_avg_score": 1.0 }, "passed": true, @@ -964,79 +1402,113 @@ "name": "lookup_order" } ], + "tool_responses": [], "trace_digest": "sha256:5da8db545d2c7b26c93058b6619ab49dd59c1348f6b032f05c451c2709b66e6e" - }, + } + ], + "pass_rate": 1.0 + }, + "validation": { + "aggregate_score": 0.7333333333333334, + "cases": [ { - "aggregate_score": 1.0, - "eval_id": "train_tool_argument", - "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "aggregate_score": 0.2, + "eval_id": "val_refund_critical", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", "expected_tool_calls": [ { "arguments": { - "order_id": "A100" + "amount": "12", + "currency": "USD", + "order_id": "R900" }, - "name": "lookup_order" + "name": "refund_order" } ], - "failure_reasons": [], - "final_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0", + "metric fake_rubric_score did not meet threshold 0.75", + "route or tool does not match expected response; required tool arguments do not match expected response" + ], + "failure_types": [ + "final_response_mismatch", + "tool_argument_error", + "tool_selection_error" + ], + "final_response": "{\"answer\": \"正在查询订单 R900。\", \"arguments\": {\"order_id\": \"R900\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "hard_failed": false, "metric_passed": { - "final_response_avg_score": true + "fake_rubric_score": false, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "route or tool does not match expected response; required tool arguments do not match expected response" + ] }, - "metric_reasons": {}, "metric_scores": { - "final_response_avg_score": 1.0 + "fake_rubric_score": 0.5, + "final_response_avg_score": 0.0 }, "metric_thresholds": { + "fake_rubric_score": 0.75, "final_response_avg_score": 1.0 }, - "passed": true, + "passed": false, "run_count": 1, - "split": "train", + "split": "validation", "tool_calls": [ { "arguments": { - "order_id": "A100" + "order_id": "R900" }, "name": "lookup_order" } ], - "trace_digest": "sha256:5da8db545d2c7b26c93058b6619ab49dd59c1348f6b032f05c451c2709b66e6e" - } - ], - "pass_rate": 1.0 - }, - "validation": { - "aggregate_score": 0.6666666666666666, - "cases": [ + "tool_responses": [], + "trace_digest": "sha256:254559dde2e80e22b22f6c4cb0d37c61bbd5a8200e19d46232c76f1b0a5d8604" + }, { "aggregate_score": 1.0, "eval_id": "val_stable_faq", + "execution_errors": [], "expected_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", "expected_tool_calls": [], + "failure_attribution": null, "failure_reasons": [], + "failure_types": [], "final_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", "hard_failed": false, "metric_passed": { + "fake_rubric_score": true, "final_response_avg_score": true }, - "metric_reasons": {}, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, "metric_scores": { + "fake_rubric_score": 1.0, "final_response_avg_score": 1.0 }, "metric_thresholds": { + "fake_rubric_score": 0.75, "final_response_avg_score": 1.0 }, "passed": true, "run_count": 1, "split": "validation", "tool_calls": [], + "tool_responses": [], "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" }, { "aggregate_score": 1.0, "eval_id": "val_json_generalization", + "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ { @@ -1046,17 +1518,26 @@ "name": "lookup_order" } ], + "failure_attribution": null, "failure_reasons": [], + "failure_types": [], "final_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "hard_failed": false, "metric_passed": { + "fake_rubric_score": true, "final_response_avg_score": true }, - "metric_reasons": {}, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, "metric_scores": { + "fake_rubric_score": 1.0, "final_response_avg_score": 1.0 }, "metric_thresholds": { + "fake_rubric_score": 0.75, "final_response_avg_score": 1.0 }, "passed": true, @@ -1070,54 +1551,34 @@ "name": "lookup_order" } ], + "tool_responses": [], "trace_digest": "sha256:99d82b8a08b7eeeda5b5f41d08a3afc1c9f4f22879605c257b78458c4ef9e561" - }, - { - "aggregate_score": 0.0, - "eval_id": "val_refund_critical", - "expected_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", - "expected_tool_calls": [ - { - "arguments": { - "amount": "12", - "currency": "USD", - "order_id": "R900" - }, - "name": "refund_order" - } - ], - "failure_reasons": [ - "metric final_response_avg_score did not meet threshold 1.0" - ], - "final_response": "{\"answer\": \"正在查询订单 R900。\", \"arguments\": {\"order_id\": \"R900\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", - "hard_failed": false, - "metric_passed": { - "final_response_avg_score": false - }, - "metric_reasons": {}, - "metric_scores": { - "final_response_avg_score": 0.0 - }, - "metric_thresholds": { - "final_response_avg_score": 1.0 - }, - "passed": false, - "run_count": 1, - "split": "validation", - "tool_calls": [ - { - "arguments": { - "order_id": "R900" - }, - "name": "lookup_order" - } - ], - "trace_digest": "sha256:254559dde2e80e22b22f6c4cb0d37c61bbd5a8200e19d46232c76f1b0a5d8604" } ], "pass_rate": 0.6666666666666666 }, "validation_case_deltas": [ + { + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_passed": false, + "candidate_score": 0.2, + "critical": true, + "eval_id": "val_refund_critical", + "hard_fail_added": true, + "metric_deltas": { + "fake_rubric_score": -0.5, + "final_response_avg_score": -1.0 + }, + "new_failure_types": [ + "final_response_mismatch", + "tool_argument_error", + "tool_selection_error" + ], + "resolved_failure_types": [], + "score_delta": -0.8, + "transition": "REGRESSION" + }, { "baseline_passed": true, "baseline_score": 1.0, @@ -1127,8 +1588,11 @@ "eval_id": "val_stable_faq", "hard_fail_added": false, "metric_deltas": { + "fake_rubric_score": 0.0, "final_response_avg_score": 0.0 }, + "new_failure_types": [], + "resolved_failure_types": [], "score_delta": 0.0, "transition": "UNCHANGED" }, @@ -1141,31 +1605,30 @@ "eval_id": "val_json_generalization", "hard_fail_added": false, "metric_deltas": { + "fake_rubric_score": 1.0, "final_response_avg_score": 1.0 }, + "new_failure_types": [], + "resolved_failure_types": [ + "final_response_mismatch", + "format_violation" + ], "score_delta": 1.0, "transition": "NEW_PASS" - }, - { - "baseline_passed": true, - "baseline_score": 1.0, - "candidate_passed": false, - "candidate_score": 0.0, - "critical": true, - "eval_id": "val_refund_critical", - "hard_fail_added": true, - "metric_deltas": { - "final_response_avg_score": -1.0 - }, - "score_delta": -1.0, - "transition": "REGRESSION" } ] } ], "mode": "fake", + "run_metadata": { + "evaluation_source": "fixture", + "independent_candidate_evaluation": true + }, "schema_version": "1.0", "seed": 42, "selected_candidate_id": "candidate_general_fix", - "source_integrity": "restored" + "source_integrity": "restored", + "warnings": [ + "generation cost is unknown" + ] } diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md index fe283fab..0ba8f17c 100644 --- a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md +++ b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md @@ -4,21 +4,22 @@ - Seed: `42` - Selected candidate: `candidate_general_fix` - Source integrity: `restored` +- Audit evidence: `audit\gate_decisions.json` ## Baseline | Split | Pass rate | Aggregate score | | --- | ---: | ---: | -| train | 0.000 | 0.000 | +| train | 0.000 | 0.200 | | validation | 0.667 | 0.667 | ## Candidates | Candidate | Decision | Train score | Validation score | | --- | --- | ---: | ---: | -| `candidate_general_fix` | ACCEPT | 0.667 | 1.000 | -| `candidate_noop` | REJECT | 0.000 | 0.667 | -| `candidate_overfit` | REJECT | 1.000 | 0.667 | +| `candidate_general_fix` | ACCEPT | 0.767 | 1.000 | +| `candidate_noop` | REJECT | 0.200 | 0.667 | +| `candidate_overfit` | REJECT | 1.000 | 0.733 | ## Validation deltas @@ -37,68 +38,86 @@ Reasons: candidate passed all independent gate rules | Case | Transition | Score delta | Critical | | --- | --- | ---: | --- | +| `val_refund_critical` | UNCHANGED | +0.000 | yes | | `val_stable_faq` | UNCHANGED | +0.000 | no | | `val_json_generalization` | NEW_PASS | +1.000 | no | -| `val_refund_critical` | UNCHANGED | +0.000 | yes | #### Gate rules | Rule | Passed | Actual | Expected | | --- | --- | ---: | ---: | +| `evaluation_complete` | yes | `complete` | `complete` | +| `validation_score_delta_available` | yes | `0.33333333333333337` | `number` | +| `validation_pass_rate_delta_available` | yes | `0.33333333333333337` | `number` | | `validation_score_improved` | yes | `0.33333333333333337` | `0.05` | | `validation_pass_rate_not_worse` | yes | `0.33333333333333337` | `0.0` | | `new_hard_fails` | yes | `0` | `0` | | `validation_regressions` | yes | `0` | `0` | | `no_critical_regression` | yes | `False` | `False` | | `no_overfit` | yes | `False` | `False` | +| `tie_policy` | yes | `False` | `False` | + +Warnings: generation cost is unknown ### `candidate_noop` Decision: **REJECT** -Reasons: validation aggregate score must improve +Reasons: validation aggregate score must improve; tie policy rejects a non-improving validation outcome #### Validation deltas | Case | Transition | Score delta | Critical | | --- | --- | ---: | --- | +| `val_refund_critical` | UNCHANGED | +0.000 | yes | | `val_stable_faq` | UNCHANGED | +0.000 | no | | `val_json_generalization` | UNCHANGED | +0.000 | no | -| `val_refund_critical` | UNCHANGED | +0.000 | yes | #### Gate rules | Rule | Passed | Actual | Expected | | --- | --- | ---: | ---: | +| `evaluation_complete` | yes | `complete` | `complete` | +| `validation_score_delta_available` | yes | `0.0` | `number` | +| `validation_pass_rate_delta_available` | yes | `0.0` | `number` | | `validation_score_improved` | no | `0.0` | `0.05` | | `validation_pass_rate_not_worse` | yes | `0.0` | `0.0` | | `new_hard_fails` | yes | `0` | `0` | | `validation_regressions` | yes | `0` | `0` | | `no_critical_regression` | yes | `False` | `False` | | `no_overfit` | yes | `False` | `False` | +| `tie_policy` | no | `True` | `False` | + +Warnings: generation cost is unknown ### `candidate_overfit` Decision: **REJECT** -Reasons: validation aggregate score must improve; new hard failures are not allowed; validation regressions exceed the limit; critical validation cases must not regress +Reasons: new hard failures are not allowed; validation regressions exceed the limit; critical validation cases must not regress #### Validation deltas | Case | Transition | Score delta | Critical | | --- | --- | ---: | --- | +| `val_refund_critical` | REGRESSION | -0.800 | yes | | `val_stable_faq` | UNCHANGED | +0.000 | no | | `val_json_generalization` | NEW_PASS | +1.000 | no | -| `val_refund_critical` | REGRESSION | -1.000 | yes | #### Gate rules | Rule | Passed | Actual | Expected | | --- | --- | ---: | ---: | -| `validation_score_improved` | no | `0.0` | `0.05` | +| `evaluation_complete` | yes | `complete` | `complete` | +| `validation_score_delta_available` | yes | `0.06666666666666676` | `number` | +| `validation_pass_rate_delta_available` | yes | `0.0` | `number` | +| `validation_score_improved` | yes | `0.06666666666666676` | `0.05` | | `validation_pass_rate_not_worse` | yes | `0.0` | `0.0` | | `new_hard_fails` | no | `1` | `0` | | `validation_regressions` | no | `1` | `0` | | `no_critical_regression` | no | `True` | `False` | | `no_overfit` | yes | `False` | `False` | +| `tie_policy` | yes | `False` | `False` | + +Warnings: generation cost is unknown ## Reproduction ```text diff --git a/tests/examples/optimization/eval_optimize_loop/test_report_and_gate_matrix.py b/tests/examples/optimization/eval_optimize_loop/test_report_and_gate_matrix.py new file mode 100644 index 00000000..95744fe2 --- /dev/null +++ b/tests/examples/optimization/eval_optimize_loop/test_report_and_gate_matrix.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop.pipeline.gate import evaluate_gate, select_winner +from examples.optimization.eval_optimize_loop.pipeline.models import ( + CandidateReport, + CaseDelta, + CaseSnapshot, + GateSettings, + OptimizationReport, + SplitReport, +) +from examples.optimization.eval_optimize_loop.run_pipeline import run_fake_pipeline, run_trace_pipeline + + +def _case(eval_id: str = "validation-1", *, passed: bool = True, score: float = 1.0, hard_failed: bool = False, metric: float = 1.0) -> CaseSnapshot: + return CaseSnapshot( + eval_id=eval_id, split="validation", run_count=1, passed=passed, hard_failed=hard_failed, + aggregate_score=score, metric_scores={"quality": metric}, metric_thresholds={"quality": 0.0}, + metric_passed={"quality": metric >= 0.0}, trace_digest="sha256:test", + ) + + +def _delta(*, transition: str = "IMPROVED", hard_fail_added: bool = False, critical: bool = False) -> CaseDelta: + return CaseDelta( + eval_id="validation-1", baseline_passed=True, candidate_passed=transition != "REGRESSION", + transition=transition, baseline_score=0.5, candidate_score=1.0 if transition != "REGRESSION" else 0.0, + score_delta=0.5 if transition != "REGRESSION" else -0.5, metric_deltas={"quality": 0.5}, + critical=critical, hard_fail_added=hard_fail_added, + ) + + +def _decision(**kwargs): + baseline = kwargs.pop("baseline", SplitReport.from_cases([_case(score=0.5, metric=0.5)])) + candidate = kwargs.pop("candidate", SplitReport.from_cases([_case(score=1.0, metric=1.0)])) + settings = GateSettings(min_validation_score_delta=0.0, **kwargs.pop("settings", {})) + return evaluate_gate(baseline, candidate, settings=settings, case_deltas=kwargs.pop("case_deltas", [_delta()]), train_score_delta=kwargs.pop("train_score_delta", 0.1), **kwargs) + + +@pytest.mark.parametrize( + ("kwargs", "failed_rule"), + [ + ({"candidate": SplitReport.from_cases([])}, "evaluation_complete"), + ({"case_deltas": [_delta(hard_fail_added=True)]}, "new_hard_fails"), + ({"case_deltas": [_delta(transition="REGRESSION", critical=True)]}, "no_critical_regression"), + ({"case_deltas": [_delta(transition="REGRESSION")]}, "validation_regressions"), + ({"metric_floors": {"quality": 1.1}}, "metric_floor:quality"), + ({"train_score_delta": 0.1, "candidate": SplitReport.from_cases([_case(score=0.4)])}, "no_overfit"), + ({"train_score_delta": 0.9, "settings": {"max_generalization_gap": 0.1}}, "generalization_gap"), + ({"generation_cost_usd": 2.0, "settings": {"max_generation_cost_usd": 1.0}}, "generation_cost_budget"), + ({"duration_seconds": 2.0, "settings": {"max_duration_seconds": 1.0}}, "duration_budget"), + ], +) +def test_gate_matrix_rejects_each_hard_constraint(kwargs: dict[str, object], failed_rule: str) -> None: + decision = _decision(**kwargs) + assert decision.accepted is False + assert next(rule for rule in decision.rules if rule.rule == failed_rule).passed is False + + +def test_gate_records_unknown_cost_warning_and_rejects_configured_tie() -> None: + decision = _decision(generation_cost_usd=None) + assert decision.accepted is True + assert "generation cost is unknown" in decision.warnings + baseline = SplitReport.from_cases([_case(score=1.0)]) + candidate = SplitReport.from_cases([_case(score=1.0)]) + tie = evaluate_gate(baseline, candidate, settings=GateSettings(min_validation_score_delta=0.0), case_deltas=[_delta(transition="UNCHANGED")]) + assert tie.accepted is False + assert next(rule for rule in tie.rules if rule.rule == "tie_policy").passed is False + + +def test_winner_selection_is_stable_and_requires_independent_evaluation() -> None: + train = SplitReport.from_cases([_case(eval_id="train-1")]) + validation = SplitReport.from_cases([_case()]) + accepted = CandidateReport(candidate_id="z", accepted=True, train=train, validation=validation, independently_evaluated=True, generation_cost_usd=1.0, duration_seconds=2.0) + also_accepted = accepted.model_copy(update={"candidate_id": "a"}) + unverified = accepted.model_copy(update={"candidate_id": "unverified", "independently_evaluated": False}) + assert select_winner([accepted, also_accepted, unverified]) == "a" + + +@pytest.mark.asyncio +async def test_fake_report_round_trip_has_secret_free_audit_evidence(tmp_path: Path) -> None: + report = await run_fake_pipeline(output_dir=tmp_path) + loaded = OptimizationReport.model_validate_json((tmp_path / "optimization_report.json").read_text(encoding="utf-8")) + assert loaded.selected_candidate_id == report.selected_candidate_id + for path in loaded.audit_references.model_dump().values(): + assert (tmp_path / path).is_file() + serialized = (tmp_path / "optimization_report.json").read_text(encoding="utf-8").lower() + assert "unused-in-test" not in serialized + assert "api_key" not in serialized + + +@pytest.mark.asyncio +async def test_fake_and_trace_outputs_are_deterministic_and_auditable(tmp_path: Path) -> None: + first = await run_fake_pipeline(output_dir=tmp_path / "first") + second = await run_fake_pipeline(output_dir=tmp_path / "second") + assert first.model_dump(mode="json") == second.model_dump(mode="json") + trace = await run_trace_pipeline(output_dir=tmp_path / "trace") + assert trace.audit_references.raw_reports_path == Path("audit/raw_reports.json") + assert json.loads((tmp_path / "trace" / "audit" / "normalized_reports.json").read_text(encoding="utf-8")) From b8f0f1105c2f51946cd87d7947dff58c41dc27d5 Mon Sep 17 00:00:00 2001 From: shiquan_zhan Date: Sat, 11 Jul 2026 14:44:49 +0800 Subject: [PATCH 13/15] fix(eval): harden audit gate integrity --- .../eval_optimize_loop/pipeline/evaluator.py | 3 +- .../eval_optimize_loop/pipeline/gate.py | 22 +- .../eval_optimize_loop/pipeline/reporter.py | 37 +- .../eval_optimize_loop/run_pipeline.py | 58 +- .../audit/candidate_reports.json | 508 +++++++------ .../sample_output/audit/gate_decisions.json | 66 +- .../audit/normalized_reports.json | 490 ++++++------- .../sample_output/audit/raw_reports.json | 154 ++-- .../sample_output/environment.snapshot.json | 2 +- .../sample_output/optimization_report.json | 666 ++++++++++-------- .../sample_output/optimization_report.md | 15 +- .../test_attribution_and_trace.py | 22 + .../test_report_and_gate_matrix.py | 42 ++ 13 files changed, 1176 insertions(+), 909 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/pipeline/evaluator.py b/examples/optimization/eval_optimize_loop/pipeline/evaluator.py index 0e6cae1e..a53663c8 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/evaluator.py +++ b/examples/optimization/eval_optimize_loop/pipeline/evaluator.py @@ -22,4 +22,5 @@ async def evaluate_split( _, _, _, results_by_eval_id = await AgentEvaluator.evaluate_eval_set( eval_set, call_agent=call_agent, eval_config=config, num_runs=config.num_runs, print_detailed_results=False ) - return SplitReport.from_cases(list(normalize_eval_results(results_by_eval_id, split=split, metric_weights=metric_weights).values())) + normalized = normalize_eval_results(results_by_eval_id, split=split, metric_weights=metric_weights) + return SplitReport.from_cases([case for _, case in sorted(normalized.items())]) diff --git a/examples/optimization/eval_optimize_loop/pipeline/gate.py b/examples/optimization/eval_optimize_loop/pipeline/gate.py index 5239fa61..7e355662 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/gate.py +++ b/examples/optimization/eval_optimize_loop/pipeline/gate.py @@ -32,11 +32,18 @@ def evaluate_gate( ) -> GateDecision: """Apply only independently measured train/validation evidence to a proposal.""" complete = _complete(baseline, candidate) + expected_delta_ids = {case.eval_id for case in baseline.cases} if complete and baseline else set() + actual_delta_ids = [delta.eval_id for delta in case_deltas] + deltas_complete = ( + complete + and len(actual_delta_ids) == len(set(actual_delta_ids)) + and set(actual_delta_ids) == expected_delta_ids + ) score_delta = candidate.aggregate_score - baseline.aggregate_score if complete and baseline and candidate else None pass_rate_delta = candidate.pass_rate - baseline.pass_rate if complete and baseline and candidate else None new_hard_fails = sum(delta.hard_fail_added for delta in case_deltas) regressions = sum(delta.transition == "REGRESSION" for delta in case_deltas) - critical_regression = any(delta.critical and delta.transition == "REGRESSION" for delta in case_deltas) + critical_regressions = sum(delta.critical and delta.transition == "REGRESSION" for delta in case_deltas) overfit = bool( settings.reject_when_train_improves_but_validation_declines and train_score_delta is not None @@ -46,13 +53,20 @@ def evaluate_gate( generalization_gap = train_score_delta - score_delta if train_score_delta is not None and score_delta is not None else None rules = [ _rule("evaluation_complete", complete, None if not complete else "complete", "complete", "independent validation evaluation is incomplete"), + _rule( + "validation_case_deltas_complete", + deltas_complete, + {"expected": sorted(expected_delta_ids), "actual": sorted(actual_delta_ids)}, + "one unique delta per validation case", + "validation case deltas are incomplete, duplicated, or do not match evaluated validation cases", + ), _rule("validation_score_delta_available", score_delta is not None, score_delta, "number", "validation aggregate score delta is required"), _rule("validation_pass_rate_delta_available", pass_rate_delta is not None, pass_rate_delta, "number", "validation pass-rate delta is required"), _rule("validation_score_improved", score_delta is not None and score_delta >= settings.min_validation_score_delta, score_delta, settings.min_validation_score_delta, "validation aggregate score must improve"), _rule("validation_pass_rate_not_worse", pass_rate_delta is not None and pass_rate_delta >= settings.min_validation_pass_rate_delta, pass_rate_delta, settings.min_validation_pass_rate_delta, "validation pass rate must not decline"), _rule("new_hard_fails", new_hard_fails <= settings.max_new_hard_fails, new_hard_fails, settings.max_new_hard_fails, "new hard failures are not allowed"), _rule("validation_regressions", regressions <= settings.max_validation_regressions, regressions, settings.max_validation_regressions, "validation regressions exceed the limit"), - _rule("no_critical_regression", settings.allow_critical_case_regression or not critical_regression, critical_regression, False, "critical validation cases must not regress"), + _rule("no_critical_regression", settings.allow_critical_case_regression or critical_regressions == 0, critical_regressions, 0, "critical validation cases must not regress"), _rule("no_overfit", not overfit, overfit, False, "train improvement with validation decline is rejected"), ] for metric_name, floor in sorted((metric_floors or {}).items()): @@ -68,7 +82,7 @@ def evaluate_gate( rules.append(_rule("tie_policy", not (settings.tie_policy == "reject" and tied), tied, False, "tie policy rejects a non-improving validation outcome")) warnings = ["generation cost is unknown"] if generation_cost_usd is None else [] failed = [rule.reason for rule in rules if not rule.passed] - risk_level = "high" if critical_regression or overfit else ("medium" if failed else "low") + risk_level = "high" if critical_regressions or overfit else ("medium" if failed else "low") return GateDecision(accepted=not failed, risk_level=risk_level, rules=rules, reasons=failed or ["candidate passed all independent gate rules"], warnings=warnings) @@ -83,7 +97,7 @@ def rank(candidate: CandidateReport) -> tuple[float, ...] | tuple[float, float, rules = gate.rules if gate else [] failures = {rule.rule: rule.actual for rule in rules} hard_fails = float(failures.get("new_hard_fails", 0) or 0) - critical = 1.0 if failures.get("no_critical_regression") is True else 0.0 + critical = float(failures.get("no_critical_regression", 0) or 0) return ( hard_fails, critical, diff --git a/examples/optimization/eval_optimize_loop/pipeline/reporter.py b/examples/optimization/eval_optimize_loop/pipeline/reporter.py index 48c5a3af..9718634b 100644 --- a/examples/optimization/eval_optimize_loop/pipeline/reporter.py +++ b/examples/optimization/eval_optimize_loop/pipeline/reporter.py @@ -21,19 +21,28 @@ def _json_safe(value: object) -> object: return value -def _write_json(path: Path, payload: object) -> None: +def write_secret_free_json(path: Path, payload: object) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(sanitize_config(_json_safe(payload)), ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") -def write_audit_artifacts(report: OptimizationReport, output_dir: Path) -> None: +def write_audit_artifacts( + report: OptimizationReport, + output_dir: Path, + *, + raw_payload: object | None = None, + normalized_payload: object | None = None, +) -> None: """Persist normalized, secret-free evidence used by the Gate decision.""" audit = output_dir / "audit" refs = report.audit_references - _write_json(output_dir / refs.raw_reports_path, {"baseline_train": report.baseline_train, "baseline_validation": report.baseline_validation}) - _write_json( + write_secret_free_json( + output_dir / refs.raw_reports_path, + raw_payload if raw_payload is not None else {"baseline_train": report.baseline_train, "baseline_validation": report.baseline_validation}, + ) + write_secret_free_json( output_dir / refs.normalized_reports_path, - { + normalized_payload if normalized_payload is not None else { "baseline_train": report.baseline_train, "baseline_validation": report.baseline_validation, "candidate_train_validation": [ @@ -42,21 +51,27 @@ def write_audit_artifacts(report: OptimizationReport, output_dir: Path) -> None: ], }, ) - _write_json(output_dir / refs.candidate_reports_path, report.candidates) - _write_json(output_dir / refs.gate_decisions_path, {"selected_candidate_id": report.selected_candidate_id, "decisions": [{"candidate_id": candidate.candidate_id, "gate": candidate.gate} for candidate in report.candidates]}) + write_secret_free_json(output_dir / refs.candidate_reports_path, report.candidates) + write_secret_free_json(output_dir / refs.gate_decisions_path, {"selected_candidate_id": report.selected_candidate_id, "decisions": [{"candidate_id": candidate.candidate_id, "gate": candidate.gate} for candidate in report.candidates]}) # The run functions may write richer snapshots before this call. Keep a # deterministic placeholder so every mode has the documented references. config_path = output_dir / refs.config_snapshot_path if not config_path.exists(): - _write_json(config_path, {"mode": report.mode, "seed": report.seed, "run_metadata": report.run_metadata}) + write_secret_free_json(config_path, {"mode": report.mode, "seed": report.seed, "run_metadata": report.run_metadata}) environment_path = output_dir / refs.environment_snapshot_path if not environment_path.exists(): - _write_json(environment_path, {"mode": report.mode, "seed": report.seed}) + write_secret_free_json(environment_path, {"mode": report.mode, "seed": report.seed}) -def write_reports(report: OptimizationReport, output_dir: Path) -> tuple[Path, Path]: +def write_reports( + report: OptimizationReport, + output_dir: Path, + *, + raw_payload: object | None = None, + normalized_payload: object | None = None, +) -> tuple[Path, Path]: output_dir.mkdir(parents=True, exist_ok=True) - write_audit_artifacts(report, output_dir) + write_audit_artifacts(report, output_dir, raw_payload=raw_payload, normalized_payload=normalized_payload) json_path = output_dir / "optimization_report.json" markdown_path = output_dir / "optimization_report.md" json_path.write_text(json.dumps(sanitize_config(report.model_dump(mode="json")), ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py index 2626349b..7b52b0a1 100644 --- a/examples/optimization/eval_optimize_loop/run_pipeline.py +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -24,7 +24,7 @@ from examples.optimization.eval_optimize_loop.fake.fake_judge import register_fake_rubric_evaluator from examples.optimization.eval_optimize_loop.fake.fixture_optimizer import FixtureOptimizerBackend from examples.optimization.eval_optimize_loop.pipeline.comparator import compare_case -from examples.optimization.eval_optimize_loop.pipeline.config import load_pipeline_config, sanitize_config +from examples.optimization.eval_optimize_loop.pipeline.config import load_pipeline_config from examples.optimization.eval_optimize_loop.pipeline.evaluator import evaluate_split from examples.optimization.eval_optimize_loop.pipeline.gate import evaluate_gate from examples.optimization.eval_optimize_loop.pipeline.gate import select_winner @@ -37,7 +37,7 @@ from examples.optimization.eval_optimize_loop.pipeline.attribution import attribute_case from examples.optimization.eval_optimize_loop.pipeline.normalization import normalize_eval_results from examples.optimization.eval_optimize_loop.pipeline.prompt_sandbox import PromptSandbox -from examples.optimization.eval_optimize_loop.pipeline.reporter import write_reports +from examples.optimization.eval_optimize_loop.pipeline.reporter import write_reports, write_secret_free_json from examples.optimization.eval_optimize_loop.pipeline.audit import write_environment_snapshot, write_input_snapshot def _attribution_summary(*splits: SplitReport | None) -> dict[str, int]: summary: dict[str, int] = {} @@ -50,6 +50,30 @@ def _attribution_summary(*splits: SplitReport | None) -> dict[str, int]: return dict(sorted(summary.items())) +def _stable_trace_projection(results_by_eval_id: dict[str, list[object]]) -> dict[str, object]: + """Keep recorded evaluator evidence while removing per-run identifiers.""" + ephemeral_keys = {"sessionid", "timestamp", "createdat", "updatedat"} + + def project(value: object) -> object: + if isinstance(value, dict): + return { + key: project(item) + for key, item in value.items() + if "".join(character for character in str(key).lower() if character.isalnum()) not in ephemeral_keys + } + if isinstance(value, list): + return [project(item) for item in value] + return value + + return { + "raw_evaluator_ran": True, + "results_by_eval_id": { + eval_id: project([result.model_dump(mode="json") for result in results]) + for eval_id, results in sorted(results_by_eval_id.items()) + }, + } + + async def run_fake_pipeline(*, output_dir: Path) -> OptimizationReport: config = load_pipeline_config(HERE / "optimizer.json", mode="fake") report = OptimizationReport.empty(mode="fake", seed=config.pipeline.reproducibility.seed) @@ -104,36 +128,20 @@ async def run_trace_pipeline(*, output_dir: Path) -> OptimizationReport: ) cases = [ case if case.passed else case.model_copy(update={"failure_attribution": attribute_case(case)}) - for case in normalized.values() + for _, case in sorted(normalized.items()) ] report = OptimizationReport.empty(mode="trace", seed=config_payload["seed"]) report.run_metadata = {"evaluation_source": "recorded_trace", "independent_candidate_evaluation": False} report.baseline_validation = SplitReport.from_cases(cases) report.attribution_summary = _attribution_summary(report.baseline_validation) output_dir.mkdir(parents=True, exist_ok=True) - (output_dir / "input.snapshot.json").write_text(json.dumps(sanitize_config({"trace_config": config_payload, "trace_evalset": "trace/trace.evalset.json"}), ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + write_secret_free_json(output_dir / "input.snapshot.json", {"trace_config": config_payload, "trace_evalset": "trace/trace.evalset.json"}) write_environment_snapshot("trace", report.seed, output_dir) - (output_dir / "trace_raw_results.json").write_text( - json.dumps( - { - "raw_evaluator_ran": True, - "results_by_eval_id": { - eval_id: [result.model_dump(mode="json") for result in results] - for eval_id, results in results_by_eval_id.items() - }, - }, - ensure_ascii=False, - indent=2, - sort_keys=True, - ) - + "\n", - encoding="utf-8", - ) - (output_dir / "trace_normalized_cases.json").write_text( - json.dumps([case.model_dump(mode="json") for case in cases], ensure_ascii=False, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - write_reports(report, output_dir) + raw_projection = _stable_trace_projection(results_by_eval_id) + normalized_projection = {"cases": [case.model_dump(mode="json") for case in cases]} + write_secret_free_json(output_dir / "trace_raw_results.json", raw_projection) + write_secret_free_json(output_dir / "trace_normalized_cases.json", normalized_projection) + write_reports(report, output_dir, raw_payload=raw_projection, normalized_payload=normalized_projection) return report diff --git a/examples/optimization/eval_optimize_loop/sample_output/audit/candidate_reports.json b/examples/optimization/eval_optimize_loop/sample_output/audit/candidate_reports.json index 4389bc28..4e68cbef 100644 --- a/examples/optimization/eval_optimize_loop/sample_output/audit/candidate_reports.json +++ b/examples/optimization/eval_optimize_loop/sample_output/audit/candidate_reports.json @@ -17,6 +17,24 @@ "reason": "independent validation evaluation is incomplete", "rule": "evaluation_complete" }, + { + "actual": { + "actual": [ + "val_json_generalization", + "val_refund_critical", + "val_stable_faq" + ], + "expected": [ + "val_json_generalization", + "val_refund_critical", + "val_stable_faq" + ] + }, + "expected": "one unique delta per validation case", + "passed": true, + "reason": "validation case deltas are incomplete, duplicated, or do not match evaluated validation cases", + "rule": "validation_case_deltas_complete" + }, { "actual": 0.33333333333333337, "expected": "number", @@ -60,8 +78,8 @@ "rule": "validation_regressions" }, { - "actual": false, - "expected": false, + "actual": 0, + "expected": 0, "passed": true, "reason": "critical validation cases must not regress", "rule": "no_critical_regression" @@ -96,7 +114,7 @@ "cases": [ { "aggregate_score": 1.0, - "eval_id": "train_tool_argument", + "eval_id": "train_json_format", "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ @@ -184,7 +202,7 @@ }, { "aggregate_score": 1.0, - "eval_id": "train_json_format", + "eval_id": "train_tool_argument", "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ @@ -239,23 +257,21 @@ "cases": [ { "aggregate_score": 1.0, - "eval_id": "val_refund_critical", + "eval_id": "val_json_generalization", "execution_errors": [], - "expected_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ { "arguments": { - "amount": "12", - "currency": "USD", - "order_id": "R900" + "order_id": "B200" }, - "name": "refund_order" + "name": "lookup_order" } ], "failure_attribution": null, "failure_reasons": [], "failure_types": [], - "final_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "final_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "hard_failed": false, "metric_passed": { "fake_rubric_score": true, @@ -280,26 +296,33 @@ "tool_calls": [ { "arguments": { - "amount": "12", - "currency": "USD", - "order_id": "R900" + "order_id": "B200" }, - "name": "refund_order" + "name": "lookup_order" } ], "tool_responses": [], - "trace_digest": "sha256:19bb632d52c6f72bc177991367b8e146fd76b0fb3a68bc8ee0622be9fa4df8a8" + "trace_digest": "sha256:99d82b8a08b7eeeda5b5f41d08a3afc1c9f4f22879605c257b78458c4ef9e561" }, { "aggregate_score": 1.0, - "eval_id": "val_stable_faq", + "eval_id": "val_refund_critical", "execution_errors": [], - "expected_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", - "expected_tool_calls": [], + "expected_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "amount": "12", + "currency": "USD", + "order_id": "R900" + }, + "name": "refund_order" + } + ], "failure_attribution": null, "failure_reasons": [], "failure_types": [], - "final_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "final_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", "hard_failed": false, "metric_passed": { "fake_rubric_score": true, @@ -321,27 +344,29 @@ "passed": true, "run_count": 1, "split": "validation", - "tool_calls": [], - "tool_responses": [], - "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" - }, - { - "aggregate_score": 1.0, - "eval_id": "val_json_generalization", - "execution_errors": [], - "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", - "expected_tool_calls": [ + "tool_calls": [ { "arguments": { - "order_id": "B200" + "amount": "12", + "currency": "USD", + "order_id": "R900" }, - "name": "lookup_order" + "name": "refund_order" } ], + "tool_responses": [], + "trace_digest": "sha256:19bb632d52c6f72bc177991367b8e146fd76b0fb3a68bc8ee0622be9fa4df8a8" + }, + { + "aggregate_score": 1.0, + "eval_id": "val_stable_faq", + "execution_errors": [], + "expected_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "expected_tool_calls": [], "failure_attribution": null, "failure_reasons": [], "failure_types": [], - "final_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "final_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", "hard_failed": false, "metric_passed": { "fake_rubric_score": true, @@ -363,21 +388,34 @@ "passed": true, "run_count": 1, "split": "validation", - "tool_calls": [ - { - "arguments": { - "order_id": "B200" - }, - "name": "lookup_order" - } - ], + "tool_calls": [], "tool_responses": [], - "trace_digest": "sha256:99d82b8a08b7eeeda5b5f41d08a3afc1c9f4f22879605c257b78458c4ef9e561" + "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" } ], "pass_rate": 1.0 }, "validation_case_deltas": [ + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_passed": true, + "candidate_score": 1.0, + "critical": false, + "eval_id": "val_json_generalization", + "hard_fail_added": false, + "metric_deltas": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "new_failure_types": [], + "resolved_failure_types": [ + "final_response_mismatch", + "format_violation" + ], + "score_delta": 1.0, + "transition": "NEW_PASS" + }, { "baseline_passed": true, "baseline_score": 1.0, @@ -411,26 +449,6 @@ "resolved_failure_types": [], "score_delta": 0.0, "transition": "UNCHANGED" - }, - { - "baseline_passed": false, - "baseline_score": 0.0, - "candidate_passed": true, - "candidate_score": 1.0, - "critical": false, - "eval_id": "val_json_generalization", - "hard_fail_added": false, - "metric_deltas": { - "fake_rubric_score": 1.0, - "final_response_avg_score": 1.0 - }, - "new_failure_types": [], - "resolved_failure_types": [ - "final_response_mismatch", - "format_violation" - ], - "score_delta": 1.0, - "transition": "NEW_PASS" } ] }, @@ -453,6 +471,24 @@ "reason": "independent validation evaluation is incomplete", "rule": "evaluation_complete" }, + { + "actual": { + "actual": [ + "val_json_generalization", + "val_refund_critical", + "val_stable_faq" + ], + "expected": [ + "val_json_generalization", + "val_refund_critical", + "val_stable_faq" + ] + }, + "expected": "one unique delta per validation case", + "passed": true, + "reason": "validation case deltas are incomplete, duplicated, or do not match evaluated validation cases", + "rule": "validation_case_deltas_complete" + }, { "actual": 0.0, "expected": "number", @@ -496,8 +532,8 @@ "rule": "validation_regressions" }, { - "actual": false, - "expected": false, + "actual": 0, + "expected": 0, "passed": true, "reason": "critical validation cases must not regress", "rule": "no_critical_regression" @@ -532,8 +568,8 @@ "aggregate_score": 0.20000000000000004, "cases": [ { - "aggregate_score": 0.30000000000000004, - "eval_id": "train_tool_argument", + "aggregate_score": 0.0, + "eval_id": "train_json_format", "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ @@ -546,24 +582,28 @@ ], "failure_attribution": null, "failure_reasons": [ - "metric final_response_avg_score did not meet threshold 1.0" + "metric final_response_avg_score did not meet threshold 1.0", + "metric fake_rubric_score did not meet threshold 0.75", + "invalid JSON response", + "actual response: invalid JSON response" ], "failure_types": [ - "final_response_mismatch" + "final_response_mismatch", + "format_violation" ], - "final_response": "{\"answer\": \"正在查询订单。\", \"arguments\": {}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "final_response": "订单 A100 正在查询", "hard_failed": false, "metric_passed": { - "fake_rubric_score": true, + "fake_rubric_score": false, "final_response_avg_score": false }, "metric_reasons": { "fake_rubric_score": [ - "required tool arguments do not match expected response" + "invalid JSON response" ] }, "metric_scores": { - "fake_rubric_score": 0.75, + "fake_rubric_score": 0.0, "final_response_avg_score": 0.0 }, "metric_thresholds": { @@ -573,14 +613,9 @@ "passed": false, "run_count": 1, "split": "train", - "tool_calls": [ - { - "arguments": {}, - "name": "lookup_order" - } - ], + "tool_calls": [], "tool_responses": [], - "trace_digest": "sha256:f086f4b22284734f1462eee58a4d204a31361c26d67449e5b23d292aaf2642bf" + "trace_digest": "sha256:65fdc3d48fd18eb37b2aa5138ee27435257ecbd0ef582c523a42673d87b04b77" }, { "aggregate_score": 0.30000000000000004, @@ -622,8 +657,8 @@ "trace_digest": "sha256:3f059c7504becc8cb98f605431f3c3714a0e70b4e62931895b888bba4ffcf9ed" }, { - "aggregate_score": 0.0, - "eval_id": "train_json_format", + "aggregate_score": 0.30000000000000004, + "eval_id": "train_tool_argument", "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ @@ -635,6 +670,63 @@ } ], "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "failure_types": [ + "final_response_mismatch" + ], + "final_response": "{\"answer\": \"正在查询订单。\", \"arguments\": {}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "required tool arguments do not match expected response" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": {}, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:f086f4b22284734f1462eee58a4d204a31361c26d67449e5b23d292aaf2642bf" + } + ], + "pass_rate": 0.0 + }, + "validation": { + "aggregate_score": 0.6666666666666666, + "cases": [ + { + "aggregate_score": 0.0, + "eval_id": "val_json_generalization", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, "failure_reasons": [ "metric final_response_avg_score did not meet threshold 1.0", "metric fake_rubric_score did not meet threshold 0.75", @@ -645,7 +737,7 @@ "final_response_mismatch", "format_violation" ], - "final_response": "订单 A100 正在查询", + "final_response": "订单 B200 正在查询", "hard_failed": false, "metric_passed": { "fake_rubric_score": false, @@ -666,17 +758,11 @@ }, "passed": false, "run_count": 1, - "split": "train", + "split": "validation", "tool_calls": [], "tool_responses": [], - "trace_digest": "sha256:65fdc3d48fd18eb37b2aa5138ee27435257ecbd0ef582c523a42673d87b04b77" - } - ], - "pass_rate": 0.0 - }, - "validation": { - "aggregate_score": 0.6666666666666666, - "cases": [ + "trace_digest": "sha256:8684ddee9351120c1bf48affbd280c5b73835b2c1d58d04682622b577c37a455" + }, { "aggregate_score": 1.0, "eval_id": "val_refund_critical", @@ -764,68 +850,18 @@ "tool_calls": [], "tool_responses": [], "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" - }, - { - "aggregate_score": 0.0, - "eval_id": "val_json_generalization", - "execution_errors": [], - "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", - "expected_tool_calls": [ - { - "arguments": { - "order_id": "B200" - }, - "name": "lookup_order" - } - ], - "failure_attribution": null, - "failure_reasons": [ - "metric final_response_avg_score did not meet threshold 1.0", - "metric fake_rubric_score did not meet threshold 0.75", - "invalid JSON response", - "actual response: invalid JSON response" - ], - "failure_types": [ - "final_response_mismatch", - "format_violation" - ], - "final_response": "订单 B200 正在查询", - "hard_failed": false, - "metric_passed": { - "fake_rubric_score": false, - "final_response_avg_score": false - }, - "metric_reasons": { - "fake_rubric_score": [ - "invalid JSON response" - ] - }, - "metric_scores": { - "fake_rubric_score": 0.0, - "final_response_avg_score": 0.0 - }, - "metric_thresholds": { - "fake_rubric_score": 0.75, - "final_response_avg_score": 1.0 - }, - "passed": false, - "run_count": 1, - "split": "validation", - "tool_calls": [], - "tool_responses": [], - "trace_digest": "sha256:8684ddee9351120c1bf48affbd280c5b73835b2c1d58d04682622b577c37a455" } ], "pass_rate": 0.6666666666666666 }, "validation_case_deltas": [ { - "baseline_passed": true, - "baseline_score": 1.0, - "candidate_passed": true, - "candidate_score": 1.0, - "critical": true, - "eval_id": "val_refund_critical", + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_passed": false, + "candidate_score": 0.0, + "critical": false, + "eval_id": "val_json_generalization", "hard_fail_added": false, "metric_deltas": { "fake_rubric_score": 0.0, @@ -841,8 +877,8 @@ "baseline_score": 1.0, "candidate_passed": true, "candidate_score": 1.0, - "critical": false, - "eval_id": "val_stable_faq", + "critical": true, + "eval_id": "val_refund_critical", "hard_fail_added": false, "metric_deltas": { "fake_rubric_score": 0.0, @@ -854,12 +890,12 @@ "transition": "UNCHANGED" }, { - "baseline_passed": false, - "baseline_score": 0.0, - "candidate_passed": false, - "candidate_score": 0.0, + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_passed": true, + "candidate_score": 1.0, "critical": false, - "eval_id": "val_json_generalization", + "eval_id": "val_stable_faq", "hard_fail_added": false, "metric_deltas": { "fake_rubric_score": 0.0, @@ -892,6 +928,24 @@ "reason": "independent validation evaluation is incomplete", "rule": "evaluation_complete" }, + { + "actual": { + "actual": [ + "val_json_generalization", + "val_refund_critical", + "val_stable_faq" + ], + "expected": [ + "val_json_generalization", + "val_refund_critical", + "val_stable_faq" + ] + }, + "expected": "one unique delta per validation case", + "passed": true, + "reason": "validation case deltas are incomplete, duplicated, or do not match evaluated validation cases", + "rule": "validation_case_deltas_complete" + }, { "actual": 0.06666666666666676, "expected": "number", @@ -935,8 +989,8 @@ "rule": "validation_regressions" }, { - "actual": true, - "expected": false, + "actual": 1, + "expected": 0, "passed": false, "reason": "critical validation cases must not regress", "rule": "no_critical_regression" @@ -973,7 +1027,7 @@ "cases": [ { "aggregate_score": 1.0, - "eval_id": "train_tool_argument", + "eval_id": "train_json_format", "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ @@ -1057,7 +1111,7 @@ }, { "aggregate_score": 1.0, - "eval_id": "train_json_format", + "eval_id": "train_tool_argument", "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ @@ -1110,6 +1164,55 @@ "validation": { "aggregate_score": 0.7333333333333334, "cases": [ + { + "aggregate_score": 1.0, + "eval_id": "val_json_generalization", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:99d82b8a08b7eeeda5b5f41d08a3afc1c9f4f22879605c257b78458c4ef9e561" + }, { "aggregate_score": 0.2, "eval_id": "val_refund_critical", @@ -1203,60 +1306,31 @@ "tool_calls": [], "tool_responses": [], "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" - }, - { - "aggregate_score": 1.0, - "eval_id": "val_json_generalization", - "execution_errors": [], - "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", - "expected_tool_calls": [ - { - "arguments": { - "order_id": "B200" - }, - "name": "lookup_order" - } - ], - "failure_attribution": null, - "failure_reasons": [], - "failure_types": [], - "final_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", - "hard_failed": false, - "metric_passed": { - "fake_rubric_score": true, - "final_response_avg_score": true - }, - "metric_reasons": { - "fake_rubric_score": [ - "all fake rubric checks passed" - ] - }, - "metric_scores": { - "fake_rubric_score": 1.0, - "final_response_avg_score": 1.0 - }, - "metric_thresholds": { - "fake_rubric_score": 0.75, - "final_response_avg_score": 1.0 - }, - "passed": true, - "run_count": 1, - "split": "validation", - "tool_calls": [ - { - "arguments": { - "order_id": "B200" - }, - "name": "lookup_order" - } - ], - "tool_responses": [], - "trace_digest": "sha256:99d82b8a08b7eeeda5b5f41d08a3afc1c9f4f22879605c257b78458c4ef9e561" } ], "pass_rate": 0.6666666666666666 }, "validation_case_deltas": [ + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_passed": true, + "candidate_score": 1.0, + "critical": false, + "eval_id": "val_json_generalization", + "hard_fail_added": false, + "metric_deltas": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "new_failure_types": [], + "resolved_failure_types": [ + "final_response_mismatch", + "format_violation" + ], + "score_delta": 1.0, + "transition": "NEW_PASS" + }, { "baseline_passed": true, "baseline_score": 1.0, @@ -1294,26 +1368,6 @@ "resolved_failure_types": [], "score_delta": 0.0, "transition": "UNCHANGED" - }, - { - "baseline_passed": false, - "baseline_score": 0.0, - "candidate_passed": true, - "candidate_score": 1.0, - "critical": false, - "eval_id": "val_json_generalization", - "hard_fail_added": false, - "metric_deltas": { - "fake_rubric_score": 1.0, - "final_response_avg_score": 1.0 - }, - "new_failure_types": [], - "resolved_failure_types": [ - "final_response_mismatch", - "format_violation" - ], - "score_delta": 1.0, - "transition": "NEW_PASS" } ] } diff --git a/examples/optimization/eval_optimize_loop/sample_output/audit/gate_decisions.json b/examples/optimization/eval_optimize_loop/sample_output/audit/gate_decisions.json index 5b314869..da324379 100644 --- a/examples/optimization/eval_optimize_loop/sample_output/audit/gate_decisions.json +++ b/examples/optimization/eval_optimize_loop/sample_output/audit/gate_decisions.json @@ -16,6 +16,24 @@ "reason": "independent validation evaluation is incomplete", "rule": "evaluation_complete" }, + { + "actual": { + "actual": [ + "val_json_generalization", + "val_refund_critical", + "val_stable_faq" + ], + "expected": [ + "val_json_generalization", + "val_refund_critical", + "val_stable_faq" + ] + }, + "expected": "one unique delta per validation case", + "passed": true, + "reason": "validation case deltas are incomplete, duplicated, or do not match evaluated validation cases", + "rule": "validation_case_deltas_complete" + }, { "actual": 0.33333333333333337, "expected": "number", @@ -59,8 +77,8 @@ "rule": "validation_regressions" }, { - "actual": false, - "expected": false, + "actual": 0, + "expected": 0, "passed": true, "reason": "critical validation cases must not regress", "rule": "no_critical_regression" @@ -102,6 +120,24 @@ "reason": "independent validation evaluation is incomplete", "rule": "evaluation_complete" }, + { + "actual": { + "actual": [ + "val_json_generalization", + "val_refund_critical", + "val_stable_faq" + ], + "expected": [ + "val_json_generalization", + "val_refund_critical", + "val_stable_faq" + ] + }, + "expected": "one unique delta per validation case", + "passed": true, + "reason": "validation case deltas are incomplete, duplicated, or do not match evaluated validation cases", + "rule": "validation_case_deltas_complete" + }, { "actual": 0.0, "expected": "number", @@ -145,8 +181,8 @@ "rule": "validation_regressions" }, { - "actual": false, - "expected": false, + "actual": 0, + "expected": 0, "passed": true, "reason": "critical validation cases must not regress", "rule": "no_critical_regression" @@ -189,6 +225,24 @@ "reason": "independent validation evaluation is incomplete", "rule": "evaluation_complete" }, + { + "actual": { + "actual": [ + "val_json_generalization", + "val_refund_critical", + "val_stable_faq" + ], + "expected": [ + "val_json_generalization", + "val_refund_critical", + "val_stable_faq" + ] + }, + "expected": "one unique delta per validation case", + "passed": true, + "reason": "validation case deltas are incomplete, duplicated, or do not match evaluated validation cases", + "rule": "validation_case_deltas_complete" + }, { "actual": 0.06666666666666676, "expected": "number", @@ -232,8 +286,8 @@ "rule": "validation_regressions" }, { - "actual": true, - "expected": false, + "actual": 1, + "expected": 0, "passed": false, "reason": "critical validation cases must not regress", "rule": "no_critical_regression" diff --git a/examples/optimization/eval_optimize_loop/sample_output/audit/normalized_reports.json b/examples/optimization/eval_optimize_loop/sample_output/audit/normalized_reports.json index aa02dbb2..94642302 100644 --- a/examples/optimization/eval_optimize_loop/sample_output/audit/normalized_reports.json +++ b/examples/optimization/eval_optimize_loop/sample_output/audit/normalized_reports.json @@ -3,8 +3,8 @@ "aggregate_score": 0.20000000000000004, "cases": [ { - "aggregate_score": 0.30000000000000004, - "eval_id": "train_tool_argument", + "aggregate_score": 0.0, + "eval_id": "train_json_format", "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ @@ -17,24 +17,28 @@ ], "failure_attribution": null, "failure_reasons": [ - "metric final_response_avg_score did not meet threshold 1.0" + "metric final_response_avg_score did not meet threshold 1.0", + "metric fake_rubric_score did not meet threshold 0.75", + "invalid JSON response", + "actual response: invalid JSON response" ], "failure_types": [ - "final_response_mismatch" + "final_response_mismatch", + "format_violation" ], - "final_response": "{\"answer\": \"正在查询订单。\", \"arguments\": {}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "final_response": "订单 A100 正在查询", "hard_failed": false, "metric_passed": { - "fake_rubric_score": true, + "fake_rubric_score": false, "final_response_avg_score": false }, "metric_reasons": { "fake_rubric_score": [ - "required tool arguments do not match expected response" + "invalid JSON response" ] }, "metric_scores": { - "fake_rubric_score": 0.75, + "fake_rubric_score": 0.0, "final_response_avg_score": 0.0 }, "metric_thresholds": { @@ -44,14 +48,9 @@ "passed": false, "run_count": 1, "split": "train", - "tool_calls": [ - { - "arguments": {}, - "name": "lookup_order" - } - ], + "tool_calls": [], "tool_responses": [], - "trace_digest": "sha256:f086f4b22284734f1462eee58a4d204a31361c26d67449e5b23d292aaf2642bf" + "trace_digest": "sha256:65fdc3d48fd18eb37b2aa5138ee27435257ecbd0ef582c523a42673d87b04b77" }, { "aggregate_score": 0.30000000000000004, @@ -93,8 +92,8 @@ "trace_digest": "sha256:3f059c7504becc8cb98f605431f3c3714a0e70b4e62931895b888bba4ffcf9ed" }, { - "aggregate_score": 0.0, - "eval_id": "train_json_format", + "aggregate_score": 0.30000000000000004, + "eval_id": "train_tool_argument", "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ @@ -106,6 +105,63 @@ } ], "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "failure_types": [ + "final_response_mismatch" + ], + "final_response": "{\"answer\": \"正在查询订单。\", \"arguments\": {}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "required tool arguments do not match expected response" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": {}, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:f086f4b22284734f1462eee58a4d204a31361c26d67449e5b23d292aaf2642bf" + } + ], + "pass_rate": 0.0 + }, + "baseline_validation": { + "aggregate_score": 0.6666666666666666, + "cases": [ + { + "aggregate_score": 0.0, + "eval_id": "val_json_generalization", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, "failure_reasons": [ "metric final_response_avg_score did not meet threshold 1.0", "metric fake_rubric_score did not meet threshold 0.75", @@ -116,7 +172,7 @@ "final_response_mismatch", "format_violation" ], - "final_response": "订单 A100 正在查询", + "final_response": "订单 B200 正在查询", "hard_failed": false, "metric_passed": { "fake_rubric_score": false, @@ -137,17 +193,11 @@ }, "passed": false, "run_count": 1, - "split": "train", + "split": "validation", "tool_calls": [], "tool_responses": [], - "trace_digest": "sha256:65fdc3d48fd18eb37b2aa5138ee27435257ecbd0ef582c523a42673d87b04b77" - } - ], - "pass_rate": 0.0 - }, - "baseline_validation": { - "aggregate_score": 0.6666666666666666, - "cases": [ + "trace_digest": "sha256:8684ddee9351120c1bf48affbd280c5b73835b2c1d58d04682622b577c37a455" + }, { "aggregate_score": 1.0, "eval_id": "val_refund_critical", @@ -235,56 +285,6 @@ "tool_calls": [], "tool_responses": [], "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" - }, - { - "aggregate_score": 0.0, - "eval_id": "val_json_generalization", - "execution_errors": [], - "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", - "expected_tool_calls": [ - { - "arguments": { - "order_id": "B200" - }, - "name": "lookup_order" - } - ], - "failure_attribution": null, - "failure_reasons": [ - "metric final_response_avg_score did not meet threshold 1.0", - "metric fake_rubric_score did not meet threshold 0.75", - "invalid JSON response", - "actual response: invalid JSON response" - ], - "failure_types": [ - "final_response_mismatch", - "format_violation" - ], - "final_response": "订单 B200 正在查询", - "hard_failed": false, - "metric_passed": { - "fake_rubric_score": false, - "final_response_avg_score": false - }, - "metric_reasons": { - "fake_rubric_score": [ - "invalid JSON response" - ] - }, - "metric_scores": { - "fake_rubric_score": 0.0, - "final_response_avg_score": 0.0 - }, - "metric_thresholds": { - "fake_rubric_score": 0.75, - "final_response_avg_score": 1.0 - }, - "passed": false, - "run_count": 1, - "split": "validation", - "tool_calls": [], - "tool_responses": [], - "trace_digest": "sha256:8684ddee9351120c1bf48affbd280c5b73835b2c1d58d04682622b577c37a455" } ], "pass_rate": 0.6666666666666666 @@ -297,7 +297,7 @@ "cases": [ { "aggregate_score": 1.0, - "eval_id": "train_tool_argument", + "eval_id": "train_json_format", "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ @@ -385,7 +385,7 @@ }, { "aggregate_score": 1.0, - "eval_id": "train_json_format", + "eval_id": "train_tool_argument", "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ @@ -440,23 +440,21 @@ "cases": [ { "aggregate_score": 1.0, - "eval_id": "val_refund_critical", + "eval_id": "val_json_generalization", "execution_errors": [], - "expected_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ { "arguments": { - "amount": "12", - "currency": "USD", - "order_id": "R900" + "order_id": "B200" }, - "name": "refund_order" + "name": "lookup_order" } ], "failure_attribution": null, "failure_reasons": [], "failure_types": [], - "final_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "final_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "hard_failed": false, "metric_passed": { "fake_rubric_score": true, @@ -481,26 +479,33 @@ "tool_calls": [ { "arguments": { - "amount": "12", - "currency": "USD", - "order_id": "R900" + "order_id": "B200" }, - "name": "refund_order" + "name": "lookup_order" } ], "tool_responses": [], - "trace_digest": "sha256:19bb632d52c6f72bc177991367b8e146fd76b0fb3a68bc8ee0622be9fa4df8a8" + "trace_digest": "sha256:99d82b8a08b7eeeda5b5f41d08a3afc1c9f4f22879605c257b78458c4ef9e561" }, { "aggregate_score": 1.0, - "eval_id": "val_stable_faq", + "eval_id": "val_refund_critical", "execution_errors": [], - "expected_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", - "expected_tool_calls": [], + "expected_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "amount": "12", + "currency": "USD", + "order_id": "R900" + }, + "name": "refund_order" + } + ], "failure_attribution": null, "failure_reasons": [], "failure_types": [], - "final_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "final_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", "hard_failed": false, "metric_passed": { "fake_rubric_score": true, @@ -522,27 +527,29 @@ "passed": true, "run_count": 1, "split": "validation", - "tool_calls": [], - "tool_responses": [], - "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" - }, - { - "aggregate_score": 1.0, - "eval_id": "val_json_generalization", - "execution_errors": [], - "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", - "expected_tool_calls": [ + "tool_calls": [ { "arguments": { - "order_id": "B200" + "amount": "12", + "currency": "USD", + "order_id": "R900" }, - "name": "lookup_order" + "name": "refund_order" } ], + "tool_responses": [], + "trace_digest": "sha256:19bb632d52c6f72bc177991367b8e146fd76b0fb3a68bc8ee0622be9fa4df8a8" + }, + { + "aggregate_score": 1.0, + "eval_id": "val_stable_faq", + "execution_errors": [], + "expected_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "expected_tool_calls": [], "failure_attribution": null, "failure_reasons": [], "failure_types": [], - "final_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "final_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", "hard_failed": false, "metric_passed": { "fake_rubric_score": true, @@ -564,16 +571,9 @@ "passed": true, "run_count": 1, "split": "validation", - "tool_calls": [ - { - "arguments": { - "order_id": "B200" - }, - "name": "lookup_order" - } - ], + "tool_calls": [], "tool_responses": [], - "trace_digest": "sha256:99d82b8a08b7eeeda5b5f41d08a3afc1c9f4f22879605c257b78458c4ef9e561" + "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" } ], "pass_rate": 1.0 @@ -585,8 +585,8 @@ "aggregate_score": 0.20000000000000004, "cases": [ { - "aggregate_score": 0.30000000000000004, - "eval_id": "train_tool_argument", + "aggregate_score": 0.0, + "eval_id": "train_json_format", "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ @@ -599,24 +599,28 @@ ], "failure_attribution": null, "failure_reasons": [ - "metric final_response_avg_score did not meet threshold 1.0" + "metric final_response_avg_score did not meet threshold 1.0", + "metric fake_rubric_score did not meet threshold 0.75", + "invalid JSON response", + "actual response: invalid JSON response" ], "failure_types": [ - "final_response_mismatch" + "final_response_mismatch", + "format_violation" ], - "final_response": "{\"answer\": \"正在查询订单。\", \"arguments\": {}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "final_response": "订单 A100 正在查询", "hard_failed": false, "metric_passed": { - "fake_rubric_score": true, + "fake_rubric_score": false, "final_response_avg_score": false }, "metric_reasons": { "fake_rubric_score": [ - "required tool arguments do not match expected response" + "invalid JSON response" ] }, "metric_scores": { - "fake_rubric_score": 0.75, + "fake_rubric_score": 0.0, "final_response_avg_score": 0.0 }, "metric_thresholds": { @@ -626,14 +630,9 @@ "passed": false, "run_count": 1, "split": "train", - "tool_calls": [ - { - "arguments": {}, - "name": "lookup_order" - } - ], + "tool_calls": [], "tool_responses": [], - "trace_digest": "sha256:f086f4b22284734f1462eee58a4d204a31361c26d67449e5b23d292aaf2642bf" + "trace_digest": "sha256:65fdc3d48fd18eb37b2aa5138ee27435257ecbd0ef582c523a42673d87b04b77" }, { "aggregate_score": 0.30000000000000004, @@ -675,8 +674,8 @@ "trace_digest": "sha256:3f059c7504becc8cb98f605431f3c3714a0e70b4e62931895b888bba4ffcf9ed" }, { - "aggregate_score": 0.0, - "eval_id": "train_json_format", + "aggregate_score": 0.30000000000000004, + "eval_id": "train_tool_argument", "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ @@ -688,6 +687,63 @@ } ], "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "failure_types": [ + "final_response_mismatch" + ], + "final_response": "{\"answer\": \"正在查询订单。\", \"arguments\": {}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "required tool arguments do not match expected response" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": {}, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:f086f4b22284734f1462eee58a4d204a31361c26d67449e5b23d292aaf2642bf" + } + ], + "pass_rate": 0.0 + }, + "validation": { + "aggregate_score": 0.6666666666666666, + "cases": [ + { + "aggregate_score": 0.0, + "eval_id": "val_json_generalization", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, "failure_reasons": [ "metric final_response_avg_score did not meet threshold 1.0", "metric fake_rubric_score did not meet threshold 0.75", @@ -698,7 +754,7 @@ "final_response_mismatch", "format_violation" ], - "final_response": "订单 A100 正在查询", + "final_response": "订单 B200 正在查询", "hard_failed": false, "metric_passed": { "fake_rubric_score": false, @@ -719,17 +775,11 @@ }, "passed": false, "run_count": 1, - "split": "train", + "split": "validation", "tool_calls": [], "tool_responses": [], - "trace_digest": "sha256:65fdc3d48fd18eb37b2aa5138ee27435257ecbd0ef582c523a42673d87b04b77" - } - ], - "pass_rate": 0.0 - }, - "validation": { - "aggregate_score": 0.6666666666666666, - "cases": [ + "trace_digest": "sha256:8684ddee9351120c1bf48affbd280c5b73835b2c1d58d04682622b577c37a455" + }, { "aggregate_score": 1.0, "eval_id": "val_refund_critical", @@ -817,56 +867,6 @@ "tool_calls": [], "tool_responses": [], "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" - }, - { - "aggregate_score": 0.0, - "eval_id": "val_json_generalization", - "execution_errors": [], - "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", - "expected_tool_calls": [ - { - "arguments": { - "order_id": "B200" - }, - "name": "lookup_order" - } - ], - "failure_attribution": null, - "failure_reasons": [ - "metric final_response_avg_score did not meet threshold 1.0", - "metric fake_rubric_score did not meet threshold 0.75", - "invalid JSON response", - "actual response: invalid JSON response" - ], - "failure_types": [ - "final_response_mismatch", - "format_violation" - ], - "final_response": "订单 B200 正在查询", - "hard_failed": false, - "metric_passed": { - "fake_rubric_score": false, - "final_response_avg_score": false - }, - "metric_reasons": { - "fake_rubric_score": [ - "invalid JSON response" - ] - }, - "metric_scores": { - "fake_rubric_score": 0.0, - "final_response_avg_score": 0.0 - }, - "metric_thresholds": { - "fake_rubric_score": 0.75, - "final_response_avg_score": 1.0 - }, - "passed": false, - "run_count": 1, - "split": "validation", - "tool_calls": [], - "tool_responses": [], - "trace_digest": "sha256:8684ddee9351120c1bf48affbd280c5b73835b2c1d58d04682622b577c37a455" } ], "pass_rate": 0.6666666666666666 @@ -879,7 +879,7 @@ "cases": [ { "aggregate_score": 1.0, - "eval_id": "train_tool_argument", + "eval_id": "train_json_format", "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ @@ -963,7 +963,7 @@ }, { "aggregate_score": 1.0, - "eval_id": "train_json_format", + "eval_id": "train_tool_argument", "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ @@ -1016,6 +1016,55 @@ "validation": { "aggregate_score": 0.7333333333333334, "cases": [ + { + "aggregate_score": 1.0, + "eval_id": "val_json_generalization", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:99d82b8a08b7eeeda5b5f41d08a3afc1c9f4f22879605c257b78458c4ef9e561" + }, { "aggregate_score": 0.2, "eval_id": "val_refund_critical", @@ -1109,55 +1158,6 @@ "tool_calls": [], "tool_responses": [], "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" - }, - { - "aggregate_score": 1.0, - "eval_id": "val_json_generalization", - "execution_errors": [], - "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", - "expected_tool_calls": [ - { - "arguments": { - "order_id": "B200" - }, - "name": "lookup_order" - } - ], - "failure_attribution": null, - "failure_reasons": [], - "failure_types": [], - "final_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", - "hard_failed": false, - "metric_passed": { - "fake_rubric_score": true, - "final_response_avg_score": true - }, - "metric_reasons": { - "fake_rubric_score": [ - "all fake rubric checks passed" - ] - }, - "metric_scores": { - "fake_rubric_score": 1.0, - "final_response_avg_score": 1.0 - }, - "metric_thresholds": { - "fake_rubric_score": 0.75, - "final_response_avg_score": 1.0 - }, - "passed": true, - "run_count": 1, - "split": "validation", - "tool_calls": [ - { - "arguments": { - "order_id": "B200" - }, - "name": "lookup_order" - } - ], - "tool_responses": [], - "trace_digest": "sha256:99d82b8a08b7eeeda5b5f41d08a3afc1c9f4f22879605c257b78458c4ef9e561" } ], "pass_rate": 0.6666666666666666 diff --git a/examples/optimization/eval_optimize_loop/sample_output/audit/raw_reports.json b/examples/optimization/eval_optimize_loop/sample_output/audit/raw_reports.json index 0105f68c..b3afbb6f 100644 --- a/examples/optimization/eval_optimize_loop/sample_output/audit/raw_reports.json +++ b/examples/optimization/eval_optimize_loop/sample_output/audit/raw_reports.json @@ -3,8 +3,8 @@ "aggregate_score": 0.20000000000000004, "cases": [ { - "aggregate_score": 0.30000000000000004, - "eval_id": "train_tool_argument", + "aggregate_score": 0.0, + "eval_id": "train_json_format", "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ @@ -17,24 +17,28 @@ ], "failure_attribution": null, "failure_reasons": [ - "metric final_response_avg_score did not meet threshold 1.0" + "metric final_response_avg_score did not meet threshold 1.0", + "metric fake_rubric_score did not meet threshold 0.75", + "invalid JSON response", + "actual response: invalid JSON response" ], "failure_types": [ - "final_response_mismatch" + "final_response_mismatch", + "format_violation" ], - "final_response": "{\"answer\": \"正在查询订单。\", \"arguments\": {}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "final_response": "订单 A100 正在查询", "hard_failed": false, "metric_passed": { - "fake_rubric_score": true, + "fake_rubric_score": false, "final_response_avg_score": false }, "metric_reasons": { "fake_rubric_score": [ - "required tool arguments do not match expected response" + "invalid JSON response" ] }, "metric_scores": { - "fake_rubric_score": 0.75, + "fake_rubric_score": 0.0, "final_response_avg_score": 0.0 }, "metric_thresholds": { @@ -44,14 +48,9 @@ "passed": false, "run_count": 1, "split": "train", - "tool_calls": [ - { - "arguments": {}, - "name": "lookup_order" - } - ], + "tool_calls": [], "tool_responses": [], - "trace_digest": "sha256:f086f4b22284734f1462eee58a4d204a31361c26d67449e5b23d292aaf2642bf" + "trace_digest": "sha256:65fdc3d48fd18eb37b2aa5138ee27435257ecbd0ef582c523a42673d87b04b77" }, { "aggregate_score": 0.30000000000000004, @@ -93,8 +92,8 @@ "trace_digest": "sha256:3f059c7504becc8cb98f605431f3c3714a0e70b4e62931895b888bba4ffcf9ed" }, { - "aggregate_score": 0.0, - "eval_id": "train_json_format", + "aggregate_score": 0.30000000000000004, + "eval_id": "train_tool_argument", "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ @@ -106,6 +105,63 @@ } ], "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "failure_types": [ + "final_response_mismatch" + ], + "final_response": "{\"answer\": \"正在查询订单。\", \"arguments\": {}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "required tool arguments do not match expected response" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": {}, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:f086f4b22284734f1462eee58a4d204a31361c26d67449e5b23d292aaf2642bf" + } + ], + "pass_rate": 0.0 + }, + "baseline_validation": { + "aggregate_score": 0.6666666666666666, + "cases": [ + { + "aggregate_score": 0.0, + "eval_id": "val_json_generalization", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, "failure_reasons": [ "metric final_response_avg_score did not meet threshold 1.0", "metric fake_rubric_score did not meet threshold 0.75", @@ -116,7 +172,7 @@ "final_response_mismatch", "format_violation" ], - "final_response": "订单 A100 正在查询", + "final_response": "订单 B200 正在查询", "hard_failed": false, "metric_passed": { "fake_rubric_score": false, @@ -137,17 +193,11 @@ }, "passed": false, "run_count": 1, - "split": "train", + "split": "validation", "tool_calls": [], "tool_responses": [], - "trace_digest": "sha256:65fdc3d48fd18eb37b2aa5138ee27435257ecbd0ef582c523a42673d87b04b77" - } - ], - "pass_rate": 0.0 - }, - "baseline_validation": { - "aggregate_score": 0.6666666666666666, - "cases": [ + "trace_digest": "sha256:8684ddee9351120c1bf48affbd280c5b73835b2c1d58d04682622b577c37a455" + }, { "aggregate_score": 1.0, "eval_id": "val_refund_critical", @@ -235,56 +285,6 @@ "tool_calls": [], "tool_responses": [], "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" - }, - { - "aggregate_score": 0.0, - "eval_id": "val_json_generalization", - "execution_errors": [], - "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", - "expected_tool_calls": [ - { - "arguments": { - "order_id": "B200" - }, - "name": "lookup_order" - } - ], - "failure_attribution": null, - "failure_reasons": [ - "metric final_response_avg_score did not meet threshold 1.0", - "metric fake_rubric_score did not meet threshold 0.75", - "invalid JSON response", - "actual response: invalid JSON response" - ], - "failure_types": [ - "final_response_mismatch", - "format_violation" - ], - "final_response": "订单 B200 正在查询", - "hard_failed": false, - "metric_passed": { - "fake_rubric_score": false, - "final_response_avg_score": false - }, - "metric_reasons": { - "fake_rubric_score": [ - "invalid JSON response" - ] - }, - "metric_scores": { - "fake_rubric_score": 0.0, - "final_response_avg_score": 0.0 - }, - "metric_thresholds": { - "fake_rubric_score": 0.75, - "final_response_avg_score": 1.0 - }, - "passed": false, - "run_count": 1, - "split": "validation", - "tool_calls": [], - "tool_responses": [], - "trace_digest": "sha256:8684ddee9351120c1bf48affbd280c5b73835b2c1d58d04682622b577c37a455" } ], "pass_rate": 0.6666666666666666 diff --git a/examples/optimization/eval_optimize_loop/sample_output/environment.snapshot.json b/examples/optimization/eval_optimize_loop/sample_output/environment.snapshot.json index 8f671e94..40238f1f 100644 --- a/examples/optimization/eval_optimize_loop/sample_output/environment.snapshot.json +++ b/examples/optimization/eval_optimize_loop/sample_output/environment.snapshot.json @@ -1 +1 @@ -{"dependencies":{"pydantic":"2.11.3","pytest":"8.4.2"},"mode":"fake","platform":"Windows-11-10.0.26200-SP0","python_version":"3.12.5","sdk_commit":"9cf5ac409b8bc7091d08250d826d6cf30ca1229e","sdk_version":"1.1.11","seed":42} +{"dependencies":{"pydantic":"2.11.3","pytest":"8.4.2"},"mode":"fake","platform":"Windows-11-10.0.26200-SP0","python_version":"3.12.5","sdk_commit":"d84e51b384e41de07e4d2f042e81b4711c101706","sdk_version":"1.1.11","seed":42} diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json index 6c208381..13444f27 100644 --- a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json +++ b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json @@ -13,8 +13,8 @@ "aggregate_score": 0.20000000000000004, "cases": [ { - "aggregate_score": 0.30000000000000004, - "eval_id": "train_tool_argument", + "aggregate_score": 0.0, + "eval_id": "train_json_format", "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ @@ -27,24 +27,28 @@ ], "failure_attribution": null, "failure_reasons": [ - "metric final_response_avg_score did not meet threshold 1.0" + "metric final_response_avg_score did not meet threshold 1.0", + "metric fake_rubric_score did not meet threshold 0.75", + "invalid JSON response", + "actual response: invalid JSON response" ], "failure_types": [ - "final_response_mismatch" + "final_response_mismatch", + "format_violation" ], - "final_response": "{\"answer\": \"正在查询订单。\", \"arguments\": {}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "final_response": "订单 A100 正在查询", "hard_failed": false, "metric_passed": { - "fake_rubric_score": true, + "fake_rubric_score": false, "final_response_avg_score": false }, "metric_reasons": { "fake_rubric_score": [ - "required tool arguments do not match expected response" + "invalid JSON response" ] }, "metric_scores": { - "fake_rubric_score": 0.75, + "fake_rubric_score": 0.0, "final_response_avg_score": 0.0 }, "metric_thresholds": { @@ -54,14 +58,9 @@ "passed": false, "run_count": 1, "split": "train", - "tool_calls": [ - { - "arguments": {}, - "name": "lookup_order" - } - ], + "tool_calls": [], "tool_responses": [], - "trace_digest": "sha256:f086f4b22284734f1462eee58a4d204a31361c26d67449e5b23d292aaf2642bf" + "trace_digest": "sha256:65fdc3d48fd18eb37b2aa5138ee27435257ecbd0ef582c523a42673d87b04b77" }, { "aggregate_score": 0.30000000000000004, @@ -103,8 +102,8 @@ "trace_digest": "sha256:3f059c7504becc8cb98f605431f3c3714a0e70b4e62931895b888bba4ffcf9ed" }, { - "aggregate_score": 0.0, - "eval_id": "train_json_format", + "aggregate_score": 0.30000000000000004, + "eval_id": "train_tool_argument", "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ @@ -116,6 +115,63 @@ } ], "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "failure_types": [ + "final_response_mismatch" + ], + "final_response": "{\"answer\": \"正在查询订单。\", \"arguments\": {}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "required tool arguments do not match expected response" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": {}, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:f086f4b22284734f1462eee58a4d204a31361c26d67449e5b23d292aaf2642bf" + } + ], + "pass_rate": 0.0 + }, + "baseline_validation": { + "aggregate_score": 0.6666666666666666, + "cases": [ + { + "aggregate_score": 0.0, + "eval_id": "val_json_generalization", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, "failure_reasons": [ "metric final_response_avg_score did not meet threshold 1.0", "metric fake_rubric_score did not meet threshold 0.75", @@ -126,7 +182,7 @@ "final_response_mismatch", "format_violation" ], - "final_response": "订单 A100 正在查询", + "final_response": "订单 B200 正在查询", "hard_failed": false, "metric_passed": { "fake_rubric_score": false, @@ -147,17 +203,11 @@ }, "passed": false, "run_count": 1, - "split": "train", + "split": "validation", "tool_calls": [], "tool_responses": [], - "trace_digest": "sha256:65fdc3d48fd18eb37b2aa5138ee27435257ecbd0ef582c523a42673d87b04b77" - } - ], - "pass_rate": 0.0 - }, - "baseline_validation": { - "aggregate_score": 0.6666666666666666, - "cases": [ + "trace_digest": "sha256:8684ddee9351120c1bf48affbd280c5b73835b2c1d58d04682622b577c37a455" + }, { "aggregate_score": 1.0, "eval_id": "val_refund_critical", @@ -245,56 +295,6 @@ "tool_calls": [], "tool_responses": [], "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" - }, - { - "aggregate_score": 0.0, - "eval_id": "val_json_generalization", - "execution_errors": [], - "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", - "expected_tool_calls": [ - { - "arguments": { - "order_id": "B200" - }, - "name": "lookup_order" - } - ], - "failure_attribution": null, - "failure_reasons": [ - "metric final_response_avg_score did not meet threshold 1.0", - "metric fake_rubric_score did not meet threshold 0.75", - "invalid JSON response", - "actual response: invalid JSON response" - ], - "failure_types": [ - "final_response_mismatch", - "format_violation" - ], - "final_response": "订单 B200 正在查询", - "hard_failed": false, - "metric_passed": { - "fake_rubric_score": false, - "final_response_avg_score": false - }, - "metric_reasons": { - "fake_rubric_score": [ - "invalid JSON response" - ] - }, - "metric_scores": { - "fake_rubric_score": 0.0, - "final_response_avg_score": 0.0 - }, - "metric_thresholds": { - "fake_rubric_score": 0.75, - "final_response_avg_score": 1.0 - }, - "passed": false, - "run_count": 1, - "split": "validation", - "tool_calls": [], - "tool_responses": [], - "trace_digest": "sha256:8684ddee9351120c1bf48affbd280c5b73835b2c1d58d04682622b577c37a455" } ], "pass_rate": 0.6666666666666666 @@ -318,6 +318,24 @@ "reason": "independent validation evaluation is incomplete", "rule": "evaluation_complete" }, + { + "actual": { + "actual": [ + "val_json_generalization", + "val_refund_critical", + "val_stable_faq" + ], + "expected": [ + "val_json_generalization", + "val_refund_critical", + "val_stable_faq" + ] + }, + "expected": "one unique delta per validation case", + "passed": true, + "reason": "validation case deltas are incomplete, duplicated, or do not match evaluated validation cases", + "rule": "validation_case_deltas_complete" + }, { "actual": 0.33333333333333337, "expected": "number", @@ -361,8 +379,8 @@ "rule": "validation_regressions" }, { - "actual": false, - "expected": false, + "actual": 0, + "expected": 0, "passed": true, "reason": "critical validation cases must not regress", "rule": "no_critical_regression" @@ -397,7 +415,7 @@ "cases": [ { "aggregate_score": 1.0, - "eval_id": "train_tool_argument", + "eval_id": "train_json_format", "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ @@ -485,7 +503,7 @@ }, { "aggregate_score": 1.0, - "eval_id": "train_json_format", + "eval_id": "train_tool_argument", "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ @@ -540,23 +558,21 @@ "cases": [ { "aggregate_score": 1.0, - "eval_id": "val_refund_critical", + "eval_id": "val_json_generalization", "execution_errors": [], - "expected_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ { "arguments": { - "amount": "12", - "currency": "USD", - "order_id": "R900" + "order_id": "B200" }, - "name": "refund_order" + "name": "lookup_order" } ], "failure_attribution": null, "failure_reasons": [], "failure_types": [], - "final_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "final_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "hard_failed": false, "metric_passed": { "fake_rubric_score": true, @@ -581,26 +597,33 @@ "tool_calls": [ { "arguments": { - "amount": "12", - "currency": "USD", - "order_id": "R900" + "order_id": "B200" }, - "name": "refund_order" + "name": "lookup_order" } ], "tool_responses": [], - "trace_digest": "sha256:19bb632d52c6f72bc177991367b8e146fd76b0fb3a68bc8ee0622be9fa4df8a8" + "trace_digest": "sha256:99d82b8a08b7eeeda5b5f41d08a3afc1c9f4f22879605c257b78458c4ef9e561" }, { "aggregate_score": 1.0, - "eval_id": "val_stable_faq", + "eval_id": "val_refund_critical", "execution_errors": [], - "expected_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", - "expected_tool_calls": [], + "expected_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "amount": "12", + "currency": "USD", + "order_id": "R900" + }, + "name": "refund_order" + } + ], "failure_attribution": null, "failure_reasons": [], "failure_types": [], - "final_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "final_response": "{\"answer\": \"正在退款订单 R900。\", \"arguments\": {\"amount\": \"12\", \"currency\": \"USD\", \"order_id\": \"R900\"}, \"route\": \"refund\", \"tool\": \"refund_order\"}", "hard_failed": false, "metric_passed": { "fake_rubric_score": true, @@ -622,27 +645,29 @@ "passed": true, "run_count": 1, "split": "validation", - "tool_calls": [], - "tool_responses": [], - "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" - }, - { - "aggregate_score": 1.0, - "eval_id": "val_json_generalization", - "execution_errors": [], - "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", - "expected_tool_calls": [ + "tool_calls": [ { "arguments": { - "order_id": "B200" + "amount": "12", + "currency": "USD", + "order_id": "R900" }, - "name": "lookup_order" + "name": "refund_order" } ], + "tool_responses": [], + "trace_digest": "sha256:19bb632d52c6f72bc177991367b8e146fd76b0fb3a68bc8ee0622be9fa4df8a8" + }, + { + "aggregate_score": 1.0, + "eval_id": "val_stable_faq", + "execution_errors": [], + "expected_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", + "expected_tool_calls": [], "failure_attribution": null, "failure_reasons": [], "failure_types": [], - "final_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "final_response": "{\"answer\": \"在订单详情页可查看订单状态。\", \"arguments\": {}, \"route\": \"faq\", \"tool\": \"none\"}", "hard_failed": false, "metric_passed": { "fake_rubric_score": true, @@ -664,45 +689,41 @@ "passed": true, "run_count": 1, "split": "validation", - "tool_calls": [ - { - "arguments": { - "order_id": "B200" - }, - "name": "lookup_order" - } - ], + "tool_calls": [], "tool_responses": [], - "trace_digest": "sha256:99d82b8a08b7eeeda5b5f41d08a3afc1c9f4f22879605c257b78458c4ef9e561" + "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" } ], "pass_rate": 1.0 }, "validation_case_deltas": [ { - "baseline_passed": true, - "baseline_score": 1.0, + "baseline_passed": false, + "baseline_score": 0.0, "candidate_passed": true, "candidate_score": 1.0, - "critical": true, - "eval_id": "val_refund_critical", + "critical": false, + "eval_id": "val_json_generalization", "hard_fail_added": false, "metric_deltas": { - "fake_rubric_score": 0.0, - "final_response_avg_score": 0.0 + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 }, "new_failure_types": [], - "resolved_failure_types": [], - "score_delta": 0.0, - "transition": "UNCHANGED" + "resolved_failure_types": [ + "final_response_mismatch", + "format_violation" + ], + "score_delta": 1.0, + "transition": "NEW_PASS" }, { "baseline_passed": true, "baseline_score": 1.0, "candidate_passed": true, "candidate_score": 1.0, - "critical": false, - "eval_id": "val_stable_faq", + "critical": true, + "eval_id": "val_refund_critical", "hard_fail_added": false, "metric_deltas": { "fake_rubric_score": 0.0, @@ -714,24 +735,21 @@ "transition": "UNCHANGED" }, { - "baseline_passed": false, - "baseline_score": 0.0, + "baseline_passed": true, + "baseline_score": 1.0, "candidate_passed": true, "candidate_score": 1.0, "critical": false, - "eval_id": "val_json_generalization", + "eval_id": "val_stable_faq", "hard_fail_added": false, "metric_deltas": { - "fake_rubric_score": 1.0, - "final_response_avg_score": 1.0 + "fake_rubric_score": 0.0, + "final_response_avg_score": 0.0 }, "new_failure_types": [], - "resolved_failure_types": [ - "final_response_mismatch", - "format_violation" - ], - "score_delta": 1.0, - "transition": "NEW_PASS" + "resolved_failure_types": [], + "score_delta": 0.0, + "transition": "UNCHANGED" } ] }, @@ -754,6 +772,24 @@ "reason": "independent validation evaluation is incomplete", "rule": "evaluation_complete" }, + { + "actual": { + "actual": [ + "val_json_generalization", + "val_refund_critical", + "val_stable_faq" + ], + "expected": [ + "val_json_generalization", + "val_refund_critical", + "val_stable_faq" + ] + }, + "expected": "one unique delta per validation case", + "passed": true, + "reason": "validation case deltas are incomplete, duplicated, or do not match evaluated validation cases", + "rule": "validation_case_deltas_complete" + }, { "actual": 0.0, "expected": "number", @@ -797,8 +833,8 @@ "rule": "validation_regressions" }, { - "actual": false, - "expected": false, + "actual": 0, + "expected": 0, "passed": true, "reason": "critical validation cases must not regress", "rule": "no_critical_regression" @@ -833,8 +869,8 @@ "aggregate_score": 0.20000000000000004, "cases": [ { - "aggregate_score": 0.30000000000000004, - "eval_id": "train_tool_argument", + "aggregate_score": 0.0, + "eval_id": "train_json_format", "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ @@ -847,24 +883,28 @@ ], "failure_attribution": null, "failure_reasons": [ - "metric final_response_avg_score did not meet threshold 1.0" + "metric final_response_avg_score did not meet threshold 1.0", + "metric fake_rubric_score did not meet threshold 0.75", + "invalid JSON response", + "actual response: invalid JSON response" ], "failure_types": [ - "final_response_mismatch" + "final_response_mismatch", + "format_violation" ], - "final_response": "{\"answer\": \"正在查询订单。\", \"arguments\": {}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "final_response": "订单 A100 正在查询", "hard_failed": false, "metric_passed": { - "fake_rubric_score": true, + "fake_rubric_score": false, "final_response_avg_score": false }, "metric_reasons": { "fake_rubric_score": [ - "required tool arguments do not match expected response" + "invalid JSON response" ] }, "metric_scores": { - "fake_rubric_score": 0.75, + "fake_rubric_score": 0.0, "final_response_avg_score": 0.0 }, "metric_thresholds": { @@ -874,14 +914,9 @@ "passed": false, "run_count": 1, "split": "train", - "tool_calls": [ - { - "arguments": {}, - "name": "lookup_order" - } - ], + "tool_calls": [], "tool_responses": [], - "trace_digest": "sha256:f086f4b22284734f1462eee58a4d204a31361c26d67449e5b23d292aaf2642bf" + "trace_digest": "sha256:65fdc3d48fd18eb37b2aa5138ee27435257ecbd0ef582c523a42673d87b04b77" }, { "aggregate_score": 0.30000000000000004, @@ -923,8 +958,8 @@ "trace_digest": "sha256:3f059c7504becc8cb98f605431f3c3714a0e70b4e62931895b888bba4ffcf9ed" }, { - "aggregate_score": 0.0, - "eval_id": "train_json_format", + "aggregate_score": 0.30000000000000004, + "eval_id": "train_tool_argument", "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ @@ -936,6 +971,63 @@ } ], "failure_attribution": null, + "failure_reasons": [ + "metric final_response_avg_score did not meet threshold 1.0" + ], + "failure_types": [ + "final_response_mismatch" + ], + "final_response": "{\"answer\": \"正在查询订单。\", \"arguments\": {}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": false + }, + "metric_reasons": { + "fake_rubric_score": [ + "required tool arguments do not match expected response" + ] + }, + "metric_scores": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 0.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": false, + "run_count": 1, + "split": "train", + "tool_calls": [ + { + "arguments": {}, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:f086f4b22284734f1462eee58a4d204a31361c26d67449e5b23d292aaf2642bf" + } + ], + "pass_rate": 0.0 + }, + "validation": { + "aggregate_score": 0.6666666666666666, + "cases": [ + { + "aggregate_score": 0.0, + "eval_id": "val_json_generalization", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, "failure_reasons": [ "metric final_response_avg_score did not meet threshold 1.0", "metric fake_rubric_score did not meet threshold 0.75", @@ -946,7 +1038,7 @@ "final_response_mismatch", "format_violation" ], - "final_response": "订单 A100 正在查询", + "final_response": "订单 B200 正在查询", "hard_failed": false, "metric_passed": { "fake_rubric_score": false, @@ -967,17 +1059,11 @@ }, "passed": false, "run_count": 1, - "split": "train", + "split": "validation", "tool_calls": [], "tool_responses": [], - "trace_digest": "sha256:65fdc3d48fd18eb37b2aa5138ee27435257ecbd0ef582c523a42673d87b04b77" - } - ], - "pass_rate": 0.0 - }, - "validation": { - "aggregate_score": 0.6666666666666666, - "cases": [ + "trace_digest": "sha256:8684ddee9351120c1bf48affbd280c5b73835b2c1d58d04682622b577c37a455" + }, { "aggregate_score": 1.0, "eval_id": "val_refund_critical", @@ -1065,68 +1151,18 @@ "tool_calls": [], "tool_responses": [], "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" - }, - { - "aggregate_score": 0.0, - "eval_id": "val_json_generalization", - "execution_errors": [], - "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", - "expected_tool_calls": [ - { - "arguments": { - "order_id": "B200" - }, - "name": "lookup_order" - } - ], - "failure_attribution": null, - "failure_reasons": [ - "metric final_response_avg_score did not meet threshold 1.0", - "metric fake_rubric_score did not meet threshold 0.75", - "invalid JSON response", - "actual response: invalid JSON response" - ], - "failure_types": [ - "final_response_mismatch", - "format_violation" - ], - "final_response": "订单 B200 正在查询", - "hard_failed": false, - "metric_passed": { - "fake_rubric_score": false, - "final_response_avg_score": false - }, - "metric_reasons": { - "fake_rubric_score": [ - "invalid JSON response" - ] - }, - "metric_scores": { - "fake_rubric_score": 0.0, - "final_response_avg_score": 0.0 - }, - "metric_thresholds": { - "fake_rubric_score": 0.75, - "final_response_avg_score": 1.0 - }, - "passed": false, - "run_count": 1, - "split": "validation", - "tool_calls": [], - "tool_responses": [], - "trace_digest": "sha256:8684ddee9351120c1bf48affbd280c5b73835b2c1d58d04682622b577c37a455" } ], "pass_rate": 0.6666666666666666 }, "validation_case_deltas": [ { - "baseline_passed": true, - "baseline_score": 1.0, - "candidate_passed": true, - "candidate_score": 1.0, - "critical": true, - "eval_id": "val_refund_critical", + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_passed": false, + "candidate_score": 0.0, + "critical": false, + "eval_id": "val_json_generalization", "hard_fail_added": false, "metric_deltas": { "fake_rubric_score": 0.0, @@ -1142,8 +1178,8 @@ "baseline_score": 1.0, "candidate_passed": true, "candidate_score": 1.0, - "critical": false, - "eval_id": "val_stable_faq", + "critical": true, + "eval_id": "val_refund_critical", "hard_fail_added": false, "metric_deltas": { "fake_rubric_score": 0.0, @@ -1155,12 +1191,12 @@ "transition": "UNCHANGED" }, { - "baseline_passed": false, - "baseline_score": 0.0, - "candidate_passed": false, - "candidate_score": 0.0, + "baseline_passed": true, + "baseline_score": 1.0, + "candidate_passed": true, + "candidate_score": 1.0, "critical": false, - "eval_id": "val_json_generalization", + "eval_id": "val_stable_faq", "hard_fail_added": false, "metric_deltas": { "fake_rubric_score": 0.0, @@ -1193,6 +1229,24 @@ "reason": "independent validation evaluation is incomplete", "rule": "evaluation_complete" }, + { + "actual": { + "actual": [ + "val_json_generalization", + "val_refund_critical", + "val_stable_faq" + ], + "expected": [ + "val_json_generalization", + "val_refund_critical", + "val_stable_faq" + ] + }, + "expected": "one unique delta per validation case", + "passed": true, + "reason": "validation case deltas are incomplete, duplicated, or do not match evaluated validation cases", + "rule": "validation_case_deltas_complete" + }, { "actual": 0.06666666666666676, "expected": "number", @@ -1236,8 +1290,8 @@ "rule": "validation_regressions" }, { - "actual": true, - "expected": false, + "actual": 1, + "expected": 0, "passed": false, "reason": "critical validation cases must not regress", "rule": "no_critical_regression" @@ -1274,7 +1328,7 @@ "cases": [ { "aggregate_score": 1.0, - "eval_id": "train_tool_argument", + "eval_id": "train_json_format", "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ @@ -1358,7 +1412,7 @@ }, { "aggregate_score": 1.0, - "eval_id": "train_json_format", + "eval_id": "train_tool_argument", "execution_errors": [], "expected_response": "{\"answer\": \"正在查询订单 A100。\", \"arguments\": {\"order_id\": \"A100\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", "expected_tool_calls": [ @@ -1411,6 +1465,55 @@ "validation": { "aggregate_score": 0.7333333333333334, "cases": [ + { + "aggregate_score": 1.0, + "eval_id": "val_json_generalization", + "execution_errors": [], + "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "expected_tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "failure_attribution": null, + "failure_reasons": [], + "failure_types": [], + "final_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", + "hard_failed": false, + "metric_passed": { + "fake_rubric_score": true, + "final_response_avg_score": true + }, + "metric_reasons": { + "fake_rubric_score": [ + "all fake rubric checks passed" + ] + }, + "metric_scores": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "metric_thresholds": { + "fake_rubric_score": 0.75, + "final_response_avg_score": 1.0 + }, + "passed": true, + "run_count": 1, + "split": "validation", + "tool_calls": [ + { + "arguments": { + "order_id": "B200" + }, + "name": "lookup_order" + } + ], + "tool_responses": [], + "trace_digest": "sha256:99d82b8a08b7eeeda5b5f41d08a3afc1c9f4f22879605c257b78458c4ef9e561" + }, { "aggregate_score": 0.2, "eval_id": "val_refund_critical", @@ -1504,60 +1607,31 @@ "tool_calls": [], "tool_responses": [], "trace_digest": "sha256:0d706eb4e659fd05e056460987cfedf7195c4ffeb68bec08072c97d6e784311d" - }, - { - "aggregate_score": 1.0, - "eval_id": "val_json_generalization", - "execution_errors": [], - "expected_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", - "expected_tool_calls": [ - { - "arguments": { - "order_id": "B200" - }, - "name": "lookup_order" - } - ], - "failure_attribution": null, - "failure_reasons": [], - "failure_types": [], - "final_response": "{\"answer\": \"正在查询订单 B200。\", \"arguments\": {\"order_id\": \"B200\"}, \"route\": \"order_lookup\", \"tool\": \"lookup_order\"}", - "hard_failed": false, - "metric_passed": { - "fake_rubric_score": true, - "final_response_avg_score": true - }, - "metric_reasons": { - "fake_rubric_score": [ - "all fake rubric checks passed" - ] - }, - "metric_scores": { - "fake_rubric_score": 1.0, - "final_response_avg_score": 1.0 - }, - "metric_thresholds": { - "fake_rubric_score": 0.75, - "final_response_avg_score": 1.0 - }, - "passed": true, - "run_count": 1, - "split": "validation", - "tool_calls": [ - { - "arguments": { - "order_id": "B200" - }, - "name": "lookup_order" - } - ], - "tool_responses": [], - "trace_digest": "sha256:99d82b8a08b7eeeda5b5f41d08a3afc1c9f4f22879605c257b78458c4ef9e561" } ], "pass_rate": 0.6666666666666666 }, "validation_case_deltas": [ + { + "baseline_passed": false, + "baseline_score": 0.0, + "candidate_passed": true, + "candidate_score": 1.0, + "critical": false, + "eval_id": "val_json_generalization", + "hard_fail_added": false, + "metric_deltas": { + "fake_rubric_score": 1.0, + "final_response_avg_score": 1.0 + }, + "new_failure_types": [], + "resolved_failure_types": [ + "final_response_mismatch", + "format_violation" + ], + "score_delta": 1.0, + "transition": "NEW_PASS" + }, { "baseline_passed": true, "baseline_score": 1.0, @@ -1595,26 +1669,6 @@ "resolved_failure_types": [], "score_delta": 0.0, "transition": "UNCHANGED" - }, - { - "baseline_passed": false, - "baseline_score": 0.0, - "candidate_passed": true, - "candidate_score": 1.0, - "critical": false, - "eval_id": "val_json_generalization", - "hard_fail_added": false, - "metric_deltas": { - "fake_rubric_score": 1.0, - "final_response_avg_score": 1.0 - }, - "new_failure_types": [], - "resolved_failure_types": [ - "final_response_mismatch", - "format_violation" - ], - "score_delta": 1.0, - "transition": "NEW_PASS" } ] } diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md index 0ba8f17c..684a32a7 100644 --- a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md +++ b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md @@ -38,22 +38,23 @@ Reasons: candidate passed all independent gate rules | Case | Transition | Score delta | Critical | | --- | --- | ---: | --- | +| `val_json_generalization` | NEW_PASS | +1.000 | no | | `val_refund_critical` | UNCHANGED | +0.000 | yes | | `val_stable_faq` | UNCHANGED | +0.000 | no | -| `val_json_generalization` | NEW_PASS | +1.000 | no | #### Gate rules | Rule | Passed | Actual | Expected | | --- | --- | ---: | ---: | | `evaluation_complete` | yes | `complete` | `complete` | +| `validation_case_deltas_complete` | yes | `{'expected': ['val_json_generalization', 'val_refund_critical', 'val_stable_faq'], 'actual': ['val_json_generalization', 'val_refund_critical', 'val_stable_faq']}` | `one unique delta per validation case` | | `validation_score_delta_available` | yes | `0.33333333333333337` | `number` | | `validation_pass_rate_delta_available` | yes | `0.33333333333333337` | `number` | | `validation_score_improved` | yes | `0.33333333333333337` | `0.05` | | `validation_pass_rate_not_worse` | yes | `0.33333333333333337` | `0.0` | | `new_hard_fails` | yes | `0` | `0` | | `validation_regressions` | yes | `0` | `0` | -| `no_critical_regression` | yes | `False` | `False` | +| `no_critical_regression` | yes | `0` | `0` | | `no_overfit` | yes | `False` | `False` | | `tie_policy` | yes | `False` | `False` | @@ -68,22 +69,23 @@ Reasons: validation aggregate score must improve; tie policy rejects a non-impro | Case | Transition | Score delta | Critical | | --- | --- | ---: | --- | +| `val_json_generalization` | UNCHANGED | +0.000 | no | | `val_refund_critical` | UNCHANGED | +0.000 | yes | | `val_stable_faq` | UNCHANGED | +0.000 | no | -| `val_json_generalization` | UNCHANGED | +0.000 | no | #### Gate rules | Rule | Passed | Actual | Expected | | --- | --- | ---: | ---: | | `evaluation_complete` | yes | `complete` | `complete` | +| `validation_case_deltas_complete` | yes | `{'expected': ['val_json_generalization', 'val_refund_critical', 'val_stable_faq'], 'actual': ['val_json_generalization', 'val_refund_critical', 'val_stable_faq']}` | `one unique delta per validation case` | | `validation_score_delta_available` | yes | `0.0` | `number` | | `validation_pass_rate_delta_available` | yes | `0.0` | `number` | | `validation_score_improved` | no | `0.0` | `0.05` | | `validation_pass_rate_not_worse` | yes | `0.0` | `0.0` | | `new_hard_fails` | yes | `0` | `0` | | `validation_regressions` | yes | `0` | `0` | -| `no_critical_regression` | yes | `False` | `False` | +| `no_critical_regression` | yes | `0` | `0` | | `no_overfit` | yes | `False` | `False` | | `tie_policy` | no | `True` | `False` | @@ -98,22 +100,23 @@ Reasons: new hard failures are not allowed; validation regressions exceed the li | Case | Transition | Score delta | Critical | | --- | --- | ---: | --- | +| `val_json_generalization` | NEW_PASS | +1.000 | no | | `val_refund_critical` | REGRESSION | -0.800 | yes | | `val_stable_faq` | UNCHANGED | +0.000 | no | -| `val_json_generalization` | NEW_PASS | +1.000 | no | #### Gate rules | Rule | Passed | Actual | Expected | | --- | --- | ---: | ---: | | `evaluation_complete` | yes | `complete` | `complete` | +| `validation_case_deltas_complete` | yes | `{'expected': ['val_json_generalization', 'val_refund_critical', 'val_stable_faq'], 'actual': ['val_json_generalization', 'val_refund_critical', 'val_stable_faq']}` | `one unique delta per validation case` | | `validation_score_delta_available` | yes | `0.06666666666666676` | `number` | | `validation_pass_rate_delta_available` | yes | `0.0` | `number` | | `validation_score_improved` | yes | `0.06666666666666676` | `0.05` | | `validation_pass_rate_not_worse` | yes | `0.0` | `0.0` | | `new_hard_fails` | no | `1` | `0` | | `validation_regressions` | no | `1` | `0` | -| `no_critical_regression` | no | `True` | `False` | +| `no_critical_regression` | no | `1` | `0` | | `no_overfit` | yes | `False` | `False` | | `tie_policy` | yes | `False` | `False` | diff --git a/tests/examples/optimization/eval_optimize_loop/test_attribution_and_trace.py b/tests/examples/optimization/eval_optimize_loop/test_attribution_and_trace.py index cf08a171..32f40c32 100644 --- a/tests/examples/optimization/eval_optimize_loop/test_attribution_and_trace.py +++ b/tests/examples/optimization/eval_optimize_loop/test_attribution_and_trace.py @@ -255,11 +255,33 @@ async def test_trace_mode_evaluates_recorded_conversations_without_a_key( assert [tool.name for tool in intermediate_wins.expected_tool_calls] == ["lookup_order"] assert (tmp_path / "optimization_report.json").is_file() assert (tmp_path / "optimization_report.md").is_file() + assert (tmp_path / report.audit_references.raw_reports_path).is_file() assert (tmp_path / "trace_raw_results.json").is_file() assert (tmp_path / "trace_normalized_cases.json").is_file() assert json.loads((tmp_path / "trace_raw_results.json").read_text(encoding="utf-8"))["raw_evaluator_ran"] is True +@pytest.mark.asyncio +async def test_trace_raw_projection_is_stable_and_redacts_nested_secrets(tmp_path: Path) -> None: + from examples.optimization.eval_optimize_loop.run_pipeline import run_trace_pipeline + from examples.optimization.eval_optimize_loop.pipeline.reporter import write_secret_free_json + + first_dir, second_dir = tmp_path / "first", tmp_path / "second" + first = await run_trace_pipeline(output_dir=first_dir) + second = await run_trace_pipeline(output_dir=second_dir) + first_raw = (first_dir / first.audit_references.raw_reports_path).read_text(encoding="utf-8") + second_raw = (second_dir / second.audit_references.raw_reports_path).read_text(encoding="utf-8") + assert first_raw == second_raw + assert "session_id" not in first_raw + + sensitive_path = tmp_path / "sensitive.json" + write_secret_free_json(sensitive_path, {"nested": {"api_key": "do-not-leak", "refresh_token": "also-secret"}}) + serialized = sensitive_path.read_text(encoding="utf-8") + assert "do-not-leak" not in serialized + assert "also-secret" not in serialized + assert "***REDACTED***" in serialized + + def test_trace_cli_writes_report_paths(tmp_path: Path) -> None: repo_root = Path(__file__).resolve().parents[4] result = subprocess.run( diff --git a/tests/examples/optimization/eval_optimize_loop/test_report_and_gate_matrix.py b/tests/examples/optimization/eval_optimize_loop/test_report_and_gate_matrix.py index 95744fe2..c751e977 100644 --- a/tests/examples/optimization/eval_optimize_loop/test_report_and_gate_matrix.py +++ b/tests/examples/optimization/eval_optimize_loop/test_report_and_gate_matrix.py @@ -10,6 +10,8 @@ CandidateReport, CaseDelta, CaseSnapshot, + GateDecision, + GateRuleResult, GateSettings, OptimizationReport, SplitReport, @@ -72,6 +74,30 @@ def test_gate_records_unknown_cost_warning_and_rejects_configured_tie() -> None: assert next(rule for rule in tie.rules if rule.rule == "tie_policy").passed is False +@pytest.mark.parametrize( + "case_deltas", + [ + [], + [_delta(), _delta()], + [ + _delta(), + _delta().model_copy(update={"eval_id": "unexpected-validation-case"}), + ], + ], +) +def test_gate_rejects_missing_duplicate_or_mismatched_validation_case_deltas(case_deltas: list[CaseDelta]) -> None: + decision = _decision(case_deltas=case_deltas) + assert decision.accepted is False + rule = next(rule for rule in decision.rules if rule.rule == "validation_case_deltas_complete") + assert rule.passed is False + + +def test_gate_accepts_one_unique_delta_for_each_validation_case() -> None: + decision = _decision(case_deltas=[_delta()]) + rule = next(rule for rule in decision.rules if rule.rule == "validation_case_deltas_complete") + assert rule.passed is True + + def test_winner_selection_is_stable_and_requires_independent_evaluation() -> None: train = SplitReport.from_cases([_case(eval_id="train-1")]) validation = SplitReport.from_cases([_case()]) @@ -81,6 +107,22 @@ def test_winner_selection_is_stable_and_requires_independent_evaluation() -> Non assert select_winner([accepted, also_accepted, unverified]) == "a" +def test_winner_prefers_fewer_critical_regressions_when_policy_allows_them() -> None: + train = SplitReport.from_cases([_case(eval_id="train-1")]) + validation = SplitReport.from_cases([_case()]) + + def candidate(candidate_id: str, critical_regressions: int) -> CandidateReport: + return CandidateReport( + candidate_id=candidate_id, accepted=True, train=train, validation=validation, independently_evaluated=True, + gate=GateDecision( + accepted=True, risk_level="low", reasons=[], + rules=[GateRuleResult(rule="no_critical_regression", passed=True, actual=critical_regressions, expected=0, reason="allowed by policy")], + ), + ) + + assert select_winner([candidate("two-critical", 2), candidate("one-critical", 1)]) == "one-critical" + + @pytest.mark.asyncio async def test_fake_report_round_trip_has_secret_free_audit_evidence(tmp_path: Path) -> None: report = await run_fake_pipeline(output_dir=tmp_path) From e48ca2aacee9e1b84798d2c57f819f868f9d2b96 Mon Sep 17 00:00:00 2001 From: shiquan_zhan Date: Sat, 11 Jul 2026 15:03:45 +0800 Subject: [PATCH 14/15] chore: remove internal task report --- .superpowers/sdd/task-3-report.md | 115 ------------------------------ 1 file changed, 115 deletions(-) delete mode 100644 .superpowers/sdd/task-3-report.md diff --git a/.superpowers/sdd/task-3-report.md b/.superpowers/sdd/task-3-report.md deleted file mode 100644 index 72389810..00000000 --- a/.superpowers/sdd/task-3-report.md +++ /dev/null @@ -1,115 +0,0 @@ -# Task 3 Report: Rule-first attribution and offline trace mode - -## Scope - -- Added deterministic `attribute_case()` classification in the example pipeline. -- Added an offline four-case trace EvalSet and trace-only CLI mode. -- Extended only example contracts and reporting to persist attribution and trace artifacts. -- No SDK public API was edited. Trace mode uses no credentials, agent inference, optimizer, network judge, or model object. - -## TDD evidence - -### RED - -Command: - -```text -python -m pytest tests/examples/optimization/eval_optimize_loop/test_attribution_and_trace.py -q -``` - -Result before implementation: `15 failed` (exit code 1). The failures were the intended missing-feature signals: `pipeline.attribution` did not exist, `run_trace_pipeline` was not exported, and the CLI rejected `--mode trace`. A subsequent added boundary test also failed as expected before the implementation treated every recorded non-timeout execution error as `EXECUTION_ERROR`. - -### GREEN - -The focused Task 3 suite passed after implementation: - -```text -16 passed -``` - -It covers all rule precedence categories, judge fallback/invalid output, structural-rule precedence over judge output, trace evaluation without `TRPC_AGENT_API_KEY`, generated report/artifact files, and the trace CLI. - -## Final verification - -```text -python -m pytest tests/examples/optimization/eval_optimize_loop/test_attribution_and_trace.py -q -# 16 passed - -python -m pytest tests/examples/optimization/eval_optimize_loop -q -# 49 passed - -python examples/optimization/eval_optimize_loop/run_pipeline.py --mode trace --output-dir -# exit 0; wrote optimization_report.json, optimization_report.md, -# trace_raw_results.json, and trace_normalized_cases.json -``` - -`git diff --check` completed without whitespace errors. - -## Notes - -- The invalid-format trace uses a reference with `tool: none`, so no higher-priority tool-selection mismatch exists; it therefore correctly reaches `FORMAT_VIOLATION` under the required precedence. -- Test and CLI runs emit pre-existing third-party dependency warnings from `requests` and `langgraph`; they do not affect exit status. - -## P1 reviewer follow-up - -### Changes - -- Tool-call snapshots now come from each evaluator invocation's `intermediate_data.tool_uses`, rather than the fake final-response JSON. -- `ToolCallSnapshot.arguments` preserves the raw value, including invalid non-dict shapes such as `[]`, so structural comparison can classify `[]` versus `{}` as `TOOL_ARGUMENT_ERROR`. -- `CaseSnapshot.tool_responses` now preserves `intermediate_data.tool_responses` values. Explicit `error`, `failed`, failed `status`, or empty responses yield rule-first `TOOL_EXECUTION_ERROR` before final-response metrics. -- The trace EvalSet now includes a final-JSON-match / intermediate-tool-mismatch case and an explicit tool-response-error case. It evaluates trajectories locally with `tool_trajectory_avg_score`. - -### P1 RED/GREEN evidence - -Before the follow-up implementation, the three new normalizer tests failed because final fake JSON overwrote intermediate calls, `[]` was coerced to `{}`, and `CaseSnapshot` lacked `tool_responses`. The trace E2E expectation also failed before the tool-response-error fixture existed. - -After implementation: - -```text -python -m pytest tests/examples/optimization/eval_optimize_loop/test_attribution_and_trace.py -q -# 19 passed - -python -m pytest tests/examples/optimization/eval_optimize_loop -q -# 52 passed - -python examples/optimization/eval_optimize_loop/run_pipeline.py --mode trace --output-dir -# exit 0; normalized output preserves intermediate lookup_refund versus lookup_order, -# and tool_responses.error as raw structural evidence -``` - -## P2 reviewer follow-up - -`normalize_eval_results()` now uses intermediate tool calls only when a recorded call exists. For black-box fake evaluation, where `RemoteEvalService` deliberately leaves actual `intermediate_data` empty, it falls back to the parsed fake JSON calls. The fake regression confirms the baseline actual `lookup_order` with `{}` arguments and reference `lookup_order` with `{"order_id": "A100"}` arguments both remain visible. - -Verification after the P2 change: - -```text -python -m pytest tests/examples/optimization/eval_optimize_loop/test_fake_loop.py -q -k keeps_parsed_json_tool_calls -# 1 passed - -python -m pytest tests/examples/optimization/eval_optimize_loop/test_attribution_and_trace.py -q -# 19 passed - -python -m pytest tests/examples/optimization/eval_optimize_loop -q -# 53 passed - -python examples/optimization/eval_optimize_loop/run_pipeline.py --mode trace --output-dir -# exit 0 -``` - -## Final P1 reviewer follow-up - -Fallback now depends on whether an invocation has `intermediate_data is None`, rather than whether extracting its `tool_uses` produced an empty list. An explicit `IntermediateData(tool_uses=[])` is therefore authoritative and remains empty even if the final fake JSON claims a tool. The matching regression verifies the missing-tool structural attribution against a reference trace that contains `lookup_order`. - -Verification after this final adjustment: - -```text -python -m pytest tests/examples/optimization/eval_optimize_loop/test_attribution_and_trace.py -q -# 20 passed - -python -m pytest tests/examples/optimization/eval_optimize_loop -q -# 54 passed - -python examples/optimization/eval_optimize_loop/run_pipeline.py --mode trace --output-dir -# exit 0 -``` From 5459223f0979c119d2d3d7cc41aee26237c81b88 Mon Sep 17 00:00:00 2001 From: shiquan_zhan Date: Sat, 11 Jul 2026 16:06:51 +0800 Subject: [PATCH 15/15] docs(eval): add Chinese design explanation --- examples/optimization/eval_optimize_loop/DESIGN.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/optimization/eval_optimize_loop/DESIGN.md b/examples/optimization/eval_optimize_loop/DESIGN.md index 0a52b875..7ee068c1 100644 --- a/examples/optimization/eval_optimize_loop/DESIGN.md +++ b/examples/optimization/eval_optimize_loop/DESIGN.md @@ -1,9 +1,9 @@ -# Design notes +# 方案设计说明 -The loop has four boundaries. Evaluation normalizes SDK results into stable case snapshots. Attribution reads actual intermediate trace data first and uses expected conversation data only as a reference. Candidate generation is either the checked-in fixture backend or a thin live adapter around the public `AgentOptimizer` API. Acceptance is owned by the Gate, never by an optimizer round status or summary. +本示例在公开的 `AgentEvaluator` 与 `AgentOptimizer` 之上增加独立编排层，形成“基线评测、失败归因、候选生成、独立回归、Gate 决策、审计落盘”的完整闭环，不修改 SDK 公共 API。失败归因采用规则优先策略：先根据执行异常、实际与期望工具选择、参数差异、工具响应、JSON 格式和判定原因分类；trace 模式优先使用真实中间轨迹，期望会话只作为参照。只有规则不能解释时才保留可扩展的 judge 入口；每个结论都带有类型、证据和置信度。 -`PromptSandbox` isolates every candidate: source prompt files stay untouched during evaluation. The live adapter builds a temporary target, passes an SDK-only runtime configuration with environment placeholders intact, uses `update_source=False` and `verbose=0`, and archives optimizer artifacts. It extracts complete prompt maps only, de-duplicates their canonical digest, then independently runs full train and validation evaluation for each proposal. Optional write-back remains off by default and uses a baseline/candidate digest check after Gate success. +接受策略由 Gate 独立掌握，优化器的 round 状态或摘要不能直接决定通过。候选必须具有完整提示词映射，并在 `PromptSandbox` 中重新跑完整训练集和验证集。Gate 会拒绝评测不完整、新增 hard fail、关键回归、超出允许范围的回归、指标低于下限、验证集无增益、成本或耗时超预算以及违反平局策略的候选；未知成本会记录为警告。赢家按新增 hard fail、关键回归数、验证集通过率和得分、成本、耗时与稳定候选 ID 排序。 -The Gate rejects incomplete evidence, unavailable validation deltas, new hard failures, critical or excessive regressions, metric-floor misses, train-up/validation-down overfit, configured generalization-gap, cost, duration, and tie-policy violations. Unknown cost is recorded as a warning. It ranks only independently evaluated and accepted candidates by hard failures, critical regressions, validation pass rate, validation score, cost, duration, and candidate id, making selection stable. +防过拟合依靠训练集与验证集的独立全量回归，而不是读取优化器轮次指标。若训练提升而验证下降，或泛化差距超过配置阈值，Gate 会直接拒绝。优化期间固定 `update_source=False`，候选只作用于临时目标提示词目录；默认不回写，显式允许回写时还要校验基线和候选摘要。 -Every path writes a Pydantic-readable `OptimizationReport` plus secret-free input, environment, raw, normalized, candidate, and Gate artifacts. Fake and trace runs are deterministic and suitable for regression tests. Existing third-party `LangChainPendingDeprecationWarning` and local `requests` dependency-compatibility warnings may appear during test execution; they are external warnings rather than pipeline decisions. +每次运行都会生成可由 Pydantic 读取的 `OptimizationReport`，并保存脱敏后的输入、环境、原始、归一化、候选与 Gate 审计产物。fake 和 trace 路径不需要 API Key、网络或 LLM judge，且输出稳定可复现；live 模式缺少必要环境变量会在启动优化器前安全退出。