From dc474ad2b6607700479fd140216d080de9b5ea19 Mon Sep 17 00:00:00 2001 From: Leoluis0705 Date: Sun, 12 Jul 2026 23:10:27 +0800 Subject: [PATCH 1/3] feat(examples): add counterfactual trace optimization loop --- .../optimization/eval_optimize_loop/DESIGN.md | 16 + .../optimization/eval_optimize_loop/README.md | 33 + .../eval_optimize_loop/__init__.py | 1 + .../eval_optimize_loop/fake/__init__.py | 1 + .../eval_optimize_loop/fake/model.py | 64 ++ .../eval_optimize_loop/fake/optimizer.py | 56 + .../optimization/eval_optimize_loop/gate.json | 1 + .../eval_optimize_loop/optimizer.json | 1 + .../eval_optimize_loop/pipeline/__init__.py | 1 + .../pipeline/counterfactual.py | 70 ++ .../eval_optimize_loop/pipeline/diagnosis.py | 141 +++ .../eval_optimize_loop/pipeline/gate.py | 59 + .../pipeline/input_audit.py | 22 + .../pipeline/interventions.py | 121 ++ .../eval_optimize_loop/pipeline/models.py | 41 + .../eval_optimize_loop/pipeline/optimizer.py | 94 ++ .../eval_optimize_loop/pipeline/pipeline.py | 385 +++++++ .../eval_optimize_loop/pipeline/probe.py | 300 +++++ .../eval_optimize_loop/prompts/router.md | 1 + .../eval_optimize_loop/prompts/skill.md | 1 + .../eval_optimize_loop/prompts/system.md | 1 + .../counterfactual_probe.json | 616 ++++++++++ .../prototype_output/counterfactual_probe.md | 33 + .../run_counterfactual_probe.py | 35 + .../eval_optimize_loop/run_pipeline.py | 30 + .../sample_output/optimization_report.json | 1007 +++++++++++++++++ .../sample_output/optimization_report.md | 35 + .../optimizer_failure_digest.json | 187 +++ .../eval_optimize_loop/train.evalset.json | 7 + .../eval_optimize_loop/val.evalset.json | 7 + .../test_counterfactual_trace_probe.py | 150 +++ tests/evaluation/test_eval_optimize_loop.py | 480 ++++++++ 32 files changed, 3997 insertions(+) create mode 100644 examples/optimization/eval_optimize_loop/DESIGN.md create mode 100644 examples/optimization/eval_optimize_loop/README.md create mode 100644 examples/optimization/eval_optimize_loop/__init__.py create mode 100644 examples/optimization/eval_optimize_loop/fake/__init__.py create mode 100644 examples/optimization/eval_optimize_loop/fake/model.py create mode 100644 examples/optimization/eval_optimize_loop/fake/optimizer.py create mode 100644 examples/optimization/eval_optimize_loop/gate.json create mode 100644 examples/optimization/eval_optimize_loop/optimizer.json create mode 100644 examples/optimization/eval_optimize_loop/pipeline/__init__.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/counterfactual.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/diagnosis.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/gate.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/input_audit.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/interventions.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/models.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/optimizer.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/pipeline.py create mode 100644 examples/optimization/eval_optimize_loop/pipeline/probe.py create mode 100644 examples/optimization/eval_optimize_loop/prompts/router.md create mode 100644 examples/optimization/eval_optimize_loop/prompts/skill.md create mode 100644 examples/optimization/eval_optimize_loop/prompts/system.md create mode 100644 examples/optimization/eval_optimize_loop/prototype_output/counterfactual_probe.json create mode 100644 examples/optimization/eval_optimize_loop/prototype_output/counterfactual_probe.md create mode 100644 examples/optimization/eval_optimize_loop/run_counterfactual_probe.py create mode 100644 examples/optimization/eval_optimize_loop/run_pipeline.py create mode 100644 examples/optimization/eval_optimize_loop/sample_output/optimization_report.json create mode 100644 examples/optimization/eval_optimize_loop/sample_output/optimization_report.md create mode 100644 examples/optimization/eval_optimize_loop/sample_output/optimizer_failure_digest.json create mode 100644 examples/optimization/eval_optimize_loop/train.evalset.json create mode 100644 examples/optimization/eval_optimize_loop/val.evalset.json create mode 100644 tests/evaluation/test_counterfactual_trace_probe.py create mode 100644 tests/evaluation/test_eval_optimize_loop.py diff --git a/examples/optimization/eval_optimize_loop/DESIGN.md b/examples/optimization/eval_optimize_loop/DESIGN.md new file mode 100644 index 00000000..8cdf5e01 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/DESIGN.md @@ -0,0 +1,16 @@ +# 方案设计 + +本方案将评测优化闭环的核心从“根据失败文本猜测原因”改为“通过同一评测器验证最小行为反事实”。系统先检查训练集、验证集、提示词和预算契约，并将样例标记为 trusted、suspect 或 invalid。只有 trusted 且具有修复证据的失败能够驱动优化。 + +对于失败轨迹，系统深拷贝实际对话，分别替换最终回复、工具名称、工具参数或受限组合，保持预期对话不变，再调用 `AgentEvaluator.get_executer()` 评分。单一干预修复失败时形成强归因；只有组合有效时判定为复合失败。数据、评测器和基础设施异常不进入优化摘要。若局部替换与原工具响应不一致，证据会降低置信度。 + +归因结果映射到 router、skill 和 system 三类 `TargetPrompt`。真实优化入口固定使用 `AgentOptimizer.optimize(update_source=False)`，候选必须在完整验证集重新评测，新增退化再次执行反事实诊断。Gate 采用 all-must-pass，检查验证增益、可信子集、hard fail、关键样例、严重度、成本、耗时和证据。只有 gate 接受且显式传入 `--apply` 时才调用 `TargetPrompt.write_all()`，并记录写前写后哈希。报告保存轨迹证据、候选差异、拒绝原因、随机种子及复现命令。 + +## 技术附录 + +- 反事实归因：结论来自真实 metric delta，不依赖 case ID、failure reason 或人工标签。 +- Prompt actionability：只有 agent behavior failure 能够选择优化表面。 +- Gate：关键检查必须全部通过，证据不足时以 `NEEDS_REVIEW` 拒绝。 +- 防过拟合：训练失败驱动优化，完整 validation 决策，新退化独立诊断。 +- 审计：记录输入和提示词哈希、seed、耗时、成本、候选及每项 gate 证据。 +- 限制：局部轨迹替换可能形成现实中不可执行的状态，因此显式记录一致性并降低置信度。 diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md new file mode 100644 index 00000000..8a72be74 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/README.md @@ -0,0 +1,33 @@ +# Trust-Aware Counterfactual Trace Diagnosis Loop + +This example closes evaluation and prompt optimization with evidence from trace interventions. Unlike loops that classify a failure from reason text or sample metadata, it deep-copies the actual trace, changes one execution surface, and sends the counterfactual `EvalCase` through the same public `AgentEvaluator` metrics. Evaluation-data, evaluator, and infrastructure failures are excluded from prompt optimization. + +## Quick start + +```bash +python examples/optimization/eval_optimize_loop/run_counterfactual_probe.py +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode trace +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake --candidate-profile accepted +python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake --candidate-profile ineffective +``` + +Both fake and trace modes need no API key. Trace mode replays `actual_conversation`; `conversation` remains the expected trace. The fake optimizer is prompt-sensitive and introduces an intentionally broad billing rule without reading case IDs. + +## Real optimizer + +Fake and trace modes are fully runnable and verified end to end. `pipeline.optimizer.run_real_optimizer()` wires the public `AgentOptimizer.optimize(..., update_source=False)` API to selected `TargetPrompt` files and is verified with a mock/spy, including actionable-case filtering and unified optimization fields. It has not been run against a real model in this example. Production use requires a business `call_agent`, model credentials, and a trace-capture adapter for candidate regression diagnosis. + +## Write-back + +`--apply` is necessary but not sufficient. Files are written only after every gate check passes. The implementation records baseline hashes, calls `TargetPrompt.write_all()` once, verifies changed hashes, and restores the baseline on failure. The bundled candidate is rejected, so running with `--apply` leaves all source prompts unchanged. + +## Outputs + +- `prototype_output/counterfactual_probe.{json,md}`: feasibility evidence. +- `sample_output/optimizer_failure_digest.json`: actionable and excluded failures. +- `sample_output/optimization_report.{json,md}`: baseline, reliability, attribution, candidate deltas, regression diagnosis, gate, cost, duration, hashes, and reproduction command. + +## Known limits + +Counterfactual traces can create states that a real agent could not produce, especially when a tool name changes but its response does not. The loop therefore restricts combinations, validates trace shape, exposes invalid interventions, and rejects insufficient evidence. Exact metrics are deterministic; LLM judges require repeated sampling before a case can be trusted. Windows absolute evalset paths are converted to relative paths because the current SDK interprets a drive-letter colon as a case selector. diff --git a/examples/optimization/eval_optimize_loop/__init__.py b/examples/optimization/eval_optimize_loop/__init__.py new file mode 100644 index 00000000..bdd5a8be --- /dev/null +++ b/examples/optimization/eval_optimize_loop/__init__.py @@ -0,0 +1 @@ +"""Trust-aware counterfactual trace optimization example.""" diff --git a/examples/optimization/eval_optimize_loop/fake/__init__.py b/examples/optimization/eval_optimize_loop/fake/__init__.py new file mode 100644 index 00000000..653b96eb --- /dev/null +++ b/examples/optimization/eval_optimize_loop/fake/__init__.py @@ -0,0 +1 @@ +"""Deterministic semantic fake components.""" diff --git a/examples/optimization/eval_optimize_loop/fake/model.py b/examples/optimization/eval_optimize_loop/fake/model.py new file mode 100644 index 00000000..f322bc38 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/fake/model.py @@ -0,0 +1,64 @@ +"""Prompt-sensitive fake behavior generator; never reads eval IDs.""" + +from __future__ import annotations + +import re + +from trpc_agent_sdk.evaluation import EvalCase, Invocation + + +def user_text(case: EvalCase) -> str: + return " ".join(part.text or "" for part in case.conversation[0].user_content.parts).lower() + + +def generate_trace(case: EvalCase, prompts: dict[str, str]) -> EvalCase: + """Generate behavior from user semantics and prompt rules.""" + changed = case.model_copy(deep=True) + text = user_text(case) + router = prompts.get("router_prompt", "") + skill = prompts.get("skill_prompt", "") + system = prompts.get("system_prompt", "") + tool_name = None + tool_args = None + if "shipping" in text or "shipment" in text: + tracking = re.search(r"t-\d+", text).group(0).upper() + tool_name, tool_args = "track_shipment", {"tracking_id": tracking} + response = f'{{"tracking_id":"{tracking}","status":"in_transit"}}' + elif "billing" in text or "invoice" in text: + invoice = re.search(r"i-\d+", text).group(0).upper() + tool_name = "create_refund" if "BILLING_TO_REFUND=ON" in router else "get_invoice" + tool_args = {"invoice_id": invoice} + response = f'{{"invoice_id":"{invoice}","status":"open"}}' + else: + order_match = re.search(r"o-\d+", text) + if order_match is None: + response = '{"status":"unsupported"}' + raw = { + "user_content": case.conversation[0].user_content.model_dump(mode="json"), + "final_response": {"role": "model", "parts": [{"text": response}]}, + } + changed.actual_conversation = [Invocation.model_validate(raw)] + return changed + order = order_match.group(0).upper() + if "summarize" in text: + response = ( + f'{{"status":"pending","order_id":"{order}"}}' + if "JSON_STATUS=ALWAYS_REQUIRED" in system + else f'{{"order_id":"{order}"}}' + ) + else: + reason = "duplicate" if "duplicate" in text else "damaged" + strict = "REFUND_ROUTE=STRICT" in router + tool_name = "create_refund" if strict or reason == "duplicate" else "get_invoice" + tool_args = {"order_id": order} + if "REFUND_REASON=REQUIRED" in skill or reason == "damaged": + tool_args["reason"] = reason + response = f'{{"status":"submitted","order_id":"{order}"}}' + raw = { + "user_content": case.conversation[0].user_content.model_dump(mode="json"), + "final_response": {"role": "model", "parts": [{"text": response}]}, + } + if tool_name: + raw["intermediate_data"] = {"tool_uses": [{"name": tool_name, "args": tool_args}]} + changed.actual_conversation = [Invocation.model_validate(raw)] + return changed diff --git a/examples/optimization/eval_optimize_loop/fake/optimizer.py b/examples/optimization/eval_optimize_loop/fake/optimizer.py new file mode 100644 index 00000000..2ed8c6fc --- /dev/null +++ b/examples/optimization/eval_optimize_loop/fake/optimizer.py @@ -0,0 +1,56 @@ +"""Fake optimizer with the same candidate contract as the real adapter.""" + +from __future__ import annotations + + +async def optimize(prompts: dict[str, str], targets: list[str], profile: str = "overfit") -> dict: + candidate = dict(prompts) + if profile == "ineffective": + return { + "total_rounds": 1, + "rounds": [ + { + "round": 1, + "profile": profile, + "targets": [], + "candidate_prompts": candidate, + "cost": 0.0, + "tokens": 0, + "duration_seconds": 0.0, + } + ], + "best_prompts": candidate, + "cost": 0.0, + "tokens": 0, + "seed": 42, + "candidate_profile": profile, + } + if profile not in ("accepted", "overfit"): + raise ValueError(f"unknown candidate profile: {profile}") + if "router_prompt" in targets: + candidate["router_prompt"] += "\nREFUND_ROUTE=STRICT\n" + if profile == "overfit": + candidate["router_prompt"] += "BILLING_TO_REFUND=ON\n" + if "skill_prompt" in targets: + candidate["skill_prompt"] += "\nREFUND_REASON=REQUIRED\n" + if "system_prompt" in targets: + candidate["system_prompt"] += "\nJSON_STATUS=ALWAYS_REQUIRED\n" + return { + "total_rounds": 1, + "rounds": [ + { + "round": 1, + "profile": profile, + "targets": list(targets), + "candidate_prompts": candidate, + "cost": 0.0, + "tokens": 0, + "duration_seconds": 0.0, + } + ], + "best_prompts": candidate, + "cost": 0.0, + "tokens": 0, + "seed": 42, + "candidate_profile": profile, + } diff --git a/examples/optimization/eval_optimize_loop/gate.json b/examples/optimization/eval_optimize_loop/gate.json new file mode 100644 index 00000000..4d5dd55f --- /dev/null +++ b/examples/optimization/eval_optimize_loop/gate.json @@ -0,0 +1 @@ +{"min_validation_delta":0.01,"max_cost":1.0,"max_latency_seconds":180,"max_counterfactual_evaluations_per_case":7,"protected_cases":["val_billing"]} diff --git a/examples/optimization/eval_optimize_loop/optimizer.json b/examples/optimization/eval_optimize_loop/optimizer.json new file mode 100644 index 00000000..df070b32 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/optimizer.json @@ -0,0 +1 @@ +{"evaluate":{"metrics":[{"metric_name":"tool_trajectory_avg_score","threshold":1.0},{"metric_name":"final_response_avg_score","threshold":1.0}],"num_runs":1},"optimize":{"algorithm":{"name":"gepa_reflective","seed":42,"reflection_lm":{"model_name":"${TRPC_AGENT_MODEL_NAME}","base_url":"${TRPC_AGENT_BASE_URL}","api_key":"${TRPC_AGENT_API_KEY}"},"max_metric_calls":20}}} diff --git a/examples/optimization/eval_optimize_loop/pipeline/__init__.py b/examples/optimization/eval_optimize_loop/pipeline/__init__.py new file mode 100644 index 00000000..de80f79c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/__init__.py @@ -0,0 +1 @@ +"""Pipeline components for the counterfactual trace optimization loop.""" diff --git a/examples/optimization/eval_optimize_loop/pipeline/counterfactual.py b/examples/optimization/eval_optimize_loop/pipeline/counterfactual.py new file mode 100644 index 00000000..db324265 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/counterfactual.py @@ -0,0 +1,70 @@ +"""Budgeted counterfactual evaluation using the official evaluator.""" + +from __future__ import annotations + +from pathlib import Path + +from trpc_agent_sdk.evaluation import EvalCase + +from .diagnosis import attribute_from_evidence +from .interventions import InterventionKind, build_counterfactual +from .models import CounterfactualEvidence, FailureAttribution +from .probe import evaluate_trace_cases + + +async def diagnose_trace_case(case: EvalCase, workspace: Path, max_evaluations: int = 7) -> FailureAttribution: + baseline = (await evaluate_trace_cases([case], workspace))[case.eval_id] + evidence = [] + singles = [ + InterventionKind.REPLACE_FINAL_RESPONSE, + InterventionKind.REPLACE_TOOL_NAME, + InterventionKind.REPLACE_TOOL_ARGUMENTS, + InterventionKind.NORMALIZE_FORMAT, + ] + combinations = [ + InterventionKind.REPLACE_TOOL_NAME_AND_ARGUMENTS, + InterventionKind.REPLACE_TOOL_NAME_AND_FINAL_RESPONSE, + InterventionKind.REPLACE_TOOL_ARGUMENTS_AND_FINAL_RESPONSE, + ] + for kind in singles + combinations: + if len(evidence) >= max_evaluations: + break + built = build_counterfactual(case, kind) + if not built.valid or built.eval_case is None: + evidence.append( + CounterfactualEvidence( + kind.value, + False, + built.status, + False, + [], + [], + baseline, + {}, + structurally_valid=built.structurally_valid, + semantically_coherent=built.semantically_coherent, + coherence_warnings=list(built.coherence_warnings), + ) + ) + continue + after = (await evaluate_trace_cases([built.eval_case], workspace))[built.eval_case.eval_id] + repaired = sorted(k for k, v in baseline.items() if v < 1 and after.get(k, v) >= 1) + unchanged = sorted(k for k, v in baseline.items() if after.get(k) == v) + evidence.append( + CounterfactualEvidence( + kind.value, + True, + built.status, + all(v >= 1 for v in after.values()), + repaired, + unchanged, + baseline, + after, + structurally_valid=built.structurally_valid, + semantically_coherent=built.semantically_coherent, + coherence_warnings=list(built.coherence_warnings), + ) + ) + if evidence[-1].changed_fail_to_pass and kind in singles: + break + return attribute_from_evidence(case.eval_id, evidence, evaluations_used=len(evidence), budget=max_evaluations) diff --git a/examples/optimization/eval_optimize_loop/pipeline/diagnosis.py b/examples/optimization/eval_optimize_loop/pipeline/diagnosis.py new file mode 100644 index 00000000..476f8514 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/diagnosis.py @@ -0,0 +1,141 @@ +"""Evidence-first failure attribution and prompt-surface selection.""" + +from __future__ import annotations + +from collections import Counter + +from .models import CounterfactualEvidence, FailureAttribution + +TARGETS = { + "tool_selection_error": ["router_prompt"], + "tool_parameter_error": ["skill_prompt"], + "tool_sequence_error": ["skill_prompt"], + "knowledge_recall_insufficient": ["skill_prompt"], + "final_response_mismatch": ["system_prompt"], + "format_violation": ["system_prompt"], + "llm_rubric_not_met": ["system_prompt"], + "compound_failure": ["router_prompt", "skill_prompt"], +} + + +class InfrastructureFailure(RuntimeError): + """Explicit non-prompt failure raised by model/tool/backend adapters.""" + + def __init__(self, category: str, message: str): + super().__init__(message) + self.category = category + + +def attribute_from_evidence( + case_id: str, + evidence: list[CounterfactualEvidence], + *, + failure_reason: str = "", + evaluations_used: int | None = None, + budget: int = 7, +) -> FailureAttribution: + """Infer category solely from intervention effects; reason is intentionally unused.""" + del failure_reason + by_name = {item.intervention: item for item in evidence} + single_map = { + "replace_final_response": "final_response_mismatch", + "replace_tool_name": "tool_selection_error", + "replace_tool_arguments": "tool_parameter_error", + "normalize_format": "format_violation", + } + repaired_singles = [ + (name, category) + for name, category in single_map.items() + if name in by_name and by_name[name].changed_fail_to_pass + ] + used = len(evidence) if evaluations_used is None else evaluations_used + extra: list = [] + if repaired_singles: + primary = repaired_singles[0][1] + secondary = sorted({category for _, category in repaired_singles[1:]}) + coherent = all(by_name[name].semantically_coherent for name, _ in repaired_singles) + confidence = (0.95 if len(repaired_singles) == 1 else 0.75) if coherent else 0.65 + else: + repaired_combinations = [ + item + for item in evidence + if "+" in item.intervention or item.intervention == "replace_tool_name_and_arguments" + if item.changed_fail_to_pass + ] + if repaired_combinations: + coherent = all(item.semantically_coherent for item in repaired_combinations) + primary, secondary, confidence = "compound_failure", [], 0.9 if coherent else 0.6 + else: + metric_repairs = [item for item in evidence if item.repaired_metrics] + if metric_repairs: + first = metric_repairs[0] + primary = single_map.get(first.intervention, "insufficient_evidence") + secondary, confidence = [], 0.6 + else: + statuses = {item.status for item in evidence} + failed_metrics = { + name for item in evidence for name, score in item.before_metrics.items() if score < 1.0 + } + if "tool_call_count_mismatch" in statuses: + primary, confidence = "tool_sequence_error", 0.85 + elif "llm_rubric_knowledge_recall" in failed_metrics: + primary, confidence = "knowledge_recall_insufficient", 0.8 + elif "llm_rubric_response" in failed_metrics: + primary, confidence = "llm_rubric_not_met", 0.8 + elif any("format" in name for name in failed_metrics): + primary, confidence = "format_violation", 0.8 + else: + primary, confidence = "insufficient_evidence", 0.0 + secondary = [] + if used >= budget and primary == "insufficient_evidence": + extra.append("counterfactual_budget_exhausted") + actionable = primary in TARGETS + return FailureAttribution( + case_id=case_id, + failure_domain="agent_behavior_failure" if actionable else "evaluation_data_failure", + primary_category=primary, + secondary_categories=secondary, + prompt_actionable=actionable, + confidence=confidence, + evidence=[*evidence, *extra], + recommended_target_prompts=TARGETS.get(primary, []), + evaluations_used=used, + ) + + +def classify_non_agent_failure( + case_id: str, + *, + reliability: str = "trusted", + issues: list[str] | None = None, + error: Exception | None = None, +) -> FailureAttribution: + issues = issues or [] + if isinstance(error, InfrastructureFailure): + return FailureAttribution(case_id, "infrastructure_failure", error.category, evidence=[str(error)]) + if isinstance(error, (TimeoutError, asyncio.TimeoutError)): + return FailureAttribution(case_id, "infrastructure_failure", "model_timeout") + if error is not None: + return FailureAttribution(case_id, "evaluator_failure", "metric_execution_error", evidence=[str(error)]) + category = "missing_reference" if "missing_reference" in issues else "invalid_reference" + if reliability == "suspect": + category = "insufficient_evidence" + return FailureAttribution(case_id, "evaluation_data_failure", category, evidence=issues) + + +def select_target_prompts(attributions: list[FailureAttribution]) -> list[str]: + return sorted( + {target for item in attributions if item.prompt_actionable for target in item.recommended_target_prompts} + ) + + +def build_failure_digest(attributions: list[FailureAttribution]) -> dict: + """Separate prompt-actionable evidence from quarantined failure domains.""" + return { + "actionable_failures": [item.to_dict() for item in attributions if item.prompt_actionable], + "excluded_failures": [item.to_dict() for item in attributions if not item.prompt_actionable], + "category_statistics": dict(Counter(item.primary_category for item in attributions)), + } + + +import asyncio # placed last to keep failure taxonomy prominent diff --git a/examples/optimization/eval_optimize_loop/pipeline/gate.py b/examples/optimization/eval_optimize_loop/pipeline/gate.py new file mode 100644 index 00000000..e2a082b6 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/gate.py @@ -0,0 +1,59 @@ +"""All-must-pass acceptance gate with explicit evidence.""" + +from __future__ import annotations + + +def evaluate_gate(observed: dict, config: dict) -> dict: + delta = observed["candidate_validation"] - observed["baseline_validation"] + train_delta = observed["candidate_train"] - observed["baseline_train"] + min_delta = config["min_validation_delta"] + specs = [ + ("validation_delta", delta >= min_delta, delta, min_delta, "VALIDATION_DELTA"), + ( + "trusted_validation_non_regression", + observed["trusted_candidate"] >= observed["trusted_baseline"], + observed["trusted_candidate"] - observed["trusted_baseline"], + 0.0, + "TRUSTED_REGRESSION", + ), + ("no_new_hard_fail", not observed["new_hard_fails"], observed["new_hard_fails"], [], "NEW_HARD_FAIL"), + ( + "protected_cases", + not observed["protected_regressions"], + observed["protected_regressions"], + [], + "PROTECTED_REGRESSION", + ), + ( + "no_train_only_improvement", + not (train_delta > 0 and delta <= 0), + {"train_delta": train_delta, "validation_delta": delta}, + "validation_delta > 0 when train_delta > 0", + "TRAIN_ONLY_IMPROVEMENT", + ), + ( + "no_severity_escalation", + not observed["severity_escalations"], + observed["severity_escalations"], + [], + "SEVERITY_ESCALATION", + ), + ("cost_budget", observed["cost"] <= config["max_cost"], observed["cost"], config["max_cost"], "COST_BUDGET"), + ( + "latency_budget", + observed["duration_seconds"] <= config["max_latency_seconds"], + observed["duration_seconds"], + config["max_latency_seconds"], + "LATENCY_BUDGET", + ), + ("evidence_sufficient", observed["evidence_sufficient"], observed["evidence_sufficient"], True, "NEEDS_REVIEW"), + ] + checks = [ + {"name": n, "passed": p, "observed": o, "threshold": t, "reason": "passed" if p else code} + for n, p, o, t, code in specs + ] + return { + "accepted": all(item["passed"] for item in checks), + "checks": checks, + "reason_codes": [code for (_, passed, _, _, code) in specs if not passed], + } diff --git a/examples/optimization/eval_optimize_loop/pipeline/input_audit.py b/examples/optimization/eval_optimize_loop/pipeline/input_audit.py new file mode 100644 index 00000000..3fe21798 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/input_audit.py @@ -0,0 +1,22 @@ +"""Input contract and per-case reliability checks.""" + +from __future__ import annotations + + +def audit_eval_cases(cases: list[dict], overlapping_inputs: set[str]) -> list[dict]: + audited = [] + for case in cases: + issues = list(case.get("issues", [])) + if not case.get("has_actual", True): + issues.append("missing_actual_trace") + if not case.get("has_reference", True): + issues.append("missing_reference") + normalized = case.get("normalized_input", "") + status = case.get("status", "trusted") + if issues: + status = "invalid" + elif normalized and normalized in overlapping_inputs: + status = "suspect" + issues.append("train_validation_leakage") + audited.append({"case_id": case["case_id"], "status": status, "issues": sorted(set(issues))}) + return audited diff --git a/examples/optimization/eval_optimize_loop/pipeline/interventions.py b/examples/optimization/eval_optimize_loop/pipeline/interventions.py new file mode 100644 index 00000000..226c6e3e --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/interventions.py @@ -0,0 +1,121 @@ +"""Deep-copy counterfactual edits for trace-mode evaluation cases.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Optional + +from trpc_agent_sdk.evaluation import EvalCase, get_all_tool_calls, get_all_tool_responses + + +class InterventionKind(str, Enum): + """Supported minimal counterfactual interventions.""" + + REPLACE_FINAL_RESPONSE = "replace_final_response" + REPLACE_TOOL_NAME = "replace_tool_name" + REPLACE_TOOL_ARGUMENTS = "replace_tool_arguments" + REPLACE_TOOL_NAME_AND_ARGUMENTS = "replace_tool_name_and_arguments" + REPLACE_TOOL_NAME_AND_FINAL_RESPONSE = "replace_tool_name+final_response" + REPLACE_TOOL_ARGUMENTS_AND_FINAL_RESPONSE = "replace_tool_arguments+final_response" + NORMALIZE_FORMAT = "normalize_format" + + +@dataclass(frozen=True) +class InterventionResult: + """Construction result with an explicit validity status.""" + + intervention: InterventionKind + valid: bool + status: str + eval_case: Optional[EvalCase] + structurally_valid: bool = True + semantically_coherent: bool = True + coherence_warnings: tuple[str, ...] = () + + +def _invalid(kind: InterventionKind, status: str) -> InterventionResult: + return InterventionResult( + kind, + False, + status, + None, + structurally_valid=False, + semantically_coherent=False, + coherence_warnings=(status,), + ) + + +def build_counterfactual(case: EvalCase, kind: InterventionKind) -> InterventionResult: + """Return a legal deep-copied trace case with only the requested fields edited.""" + if case.eval_mode != "trace" or not case.actual_conversation: + return _invalid(kind, "missing_actual_trace") + if not case.conversation: + return _invalid(kind, "missing_expected_trace") + if len(case.actual_conversation) != len(case.conversation): + return _invalid(kind, "invocation_count_mismatch") + + changed = case.model_copy(deep=True) + changed.eval_id = f"{case.eval_id}__{kind.value}" + + replace_final = kind in ( + InterventionKind.REPLACE_FINAL_RESPONSE, + InterventionKind.REPLACE_TOOL_NAME_AND_FINAL_RESPONSE, + InterventionKind.REPLACE_TOOL_ARGUMENTS_AND_FINAL_RESPONSE, + ) + if replace_final: + for actual, expected in zip(changed.actual_conversation, changed.conversation): + if expected.final_response is None: + return _invalid(kind, "missing_expected_final_response") + actual.final_response = expected.final_response.model_copy(deep=True) + if kind == InterventionKind.REPLACE_FINAL_RESPONSE: + return InterventionResult(kind, True, "constructed", changed) + + if kind == InterventionKind.NORMALIZE_FORMAT: + return _invalid(kind, "format_not_normalizable") + + coherence_warnings: list[str] = [] + for actual, expected in zip(changed.actual_conversation, changed.conversation): + actual_tools = get_all_tool_calls(actual.intermediate_data) + expected_tools = get_all_tool_calls(expected.intermediate_data) + actual_responses = get_all_tool_responses(actual.intermediate_data) + if not expected_tools: + return _invalid(kind, "missing_expected_tool") + if len(actual_tools) != len(expected_tools): + return _invalid(kind, "tool_call_count_mismatch") + for actual_tool, expected_tool in zip(actual_tools, expected_tools): + if kind in ( + InterventionKind.REPLACE_TOOL_ARGUMENTS, + InterventionKind.REPLACE_TOOL_NAME_AND_ARGUMENTS, + InterventionKind.REPLACE_TOOL_ARGUMENTS_AND_FINAL_RESPONSE, + ): + if not isinstance(actual_tool.args, dict) or not isinstance(expected_tool.args, dict): + return _invalid(kind, "tool_arguments_not_dict") + actual_tool.args = dict(expected_tool.args) + if actual_responses: + coherence_warnings.append("tool_response_may_depend_on_original_arguments") + if kind in ( + InterventionKind.REPLACE_TOOL_NAME, + InterventionKind.REPLACE_TOOL_NAME_AND_ARGUMENTS, + InterventionKind.REPLACE_TOOL_NAME_AND_FINAL_RESPONSE, + ): + actual_tool.name = expected_tool.name + if actual_responses and kind in ( + InterventionKind.REPLACE_TOOL_NAME, + InterventionKind.REPLACE_TOOL_NAME_AND_ARGUMENTS, + InterventionKind.REPLACE_TOOL_NAME_AND_FINAL_RESPONSE, + ): + expected_names = {tool.name for tool in expected_tools} + if any(response.name not in expected_names for response in actual_responses): + coherence_warnings.append("tool_response_matches_original_call") + + warnings = tuple(sorted(set(coherence_warnings))) + return InterventionResult( + kind, + True, + "constructed", + changed, + structurally_valid=True, + semantically_coherent=not warnings, + coherence_warnings=warnings, + ) diff --git a/examples/optimization/eval_optimize_loop/pipeline/models.py b/examples/optimization/eval_optimize_loop/pipeline/models.py new file mode 100644 index 00000000..c8dd44c6 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/models.py @@ -0,0 +1,41 @@ +"""Typed records shared by the trust-aware pipeline.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field + + +@dataclass +class CounterfactualEvidence: + intervention: str + valid: bool + status: str + changed_fail_to_pass: bool + repaired_metrics: list[str] + unchanged_metrics: list[str] + before_metrics: dict[str, float] + after_metrics: dict[str, float] + structurally_valid: bool = True + semantically_coherent: bool = True + coherence_warnings: list[str] = field(default_factory=list) + + def to_dict(self) -> dict: + return asdict(self) + + +@dataclass +class FailureAttribution: + case_id: str + failure_domain: str + primary_category: str + secondary_categories: list[str] = field(default_factory=list) + prompt_actionable: bool = False + confidence: float = 0.0 + evidence: list = field(default_factory=list) + recommended_target_prompts: list[str] = field(default_factory=list) + evaluations_used: int = 0 + + def to_dict(self) -> dict: + value = asdict(self) + value["evidence"] = [e.to_dict() if hasattr(e, "to_dict") else e for e in self.evidence] + return value diff --git a/examples/optimization/eval_optimize_loop/pipeline/optimizer.py b/examples/optimization/eval_optimize_loop/pipeline/optimizer.py new file mode 100644 index 00000000..8a02c327 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/optimizer.py @@ -0,0 +1,94 @@ +"""Optimizer adapters and gated atomic source write-back.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +from trpc_agent_sdk.evaluation import AgentOptimizer, EvalSet, TargetPrompt + + +def _hash(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +async def apply_if_accepted(gate: dict, apply: bool, prompts: dict[str, str], paths: dict[str, Path]) -> dict: + before = {name: _hash(path) for name, path in paths.items()} + result = {"requested": apply, "applied": False, "baseline_hashes": before, "final_hashes": before} + if not gate.get("accepted") or not apply: + return result + target = TargetPrompt() + for name, path in paths.items(): + target.add_path(name, str(path)) + baseline = await target.read_all() + try: + await target.write_all(prompts) + after = {name: _hash(path) for name, path in paths.items()} + if after == before: + raise RuntimeError("write-back did not change prompt hashes") + result.update({"applied": True, "final_hashes": after}) + return result + except BaseException: + await target.write_all(baseline) + raise + + +async def run_real_optimizer( + *, + config_path: Path, + call_agent, + prompt_paths: dict[str, Path], + train_path: Path, + actionable_case_ids: set[str], + validation_path: Path, + output_dir: Path, +) -> dict: + """Invoke the public optimizer without permitting source mutation.""" + output_dir.mkdir(parents=True, exist_ok=True) + train = EvalSet.model_validate_json(train_path.read_text(encoding="utf-8")) + filtered = train.model_copy( + update={"eval_cases": [case for case in train.eval_cases if case.eval_id in actionable_case_ids]}, + deep=True, + ) + if not filtered.eval_cases: + raise ValueError("real optimizer requires at least one actionable trusted training case") + filtered_path = output_dir / "actionable_train.evalset.json" + filtered_path.write_text(filtered.model_dump_json(by_alias=True, indent=2), encoding="utf-8") + target = TargetPrompt() + for name, path in prompt_paths.items(): + target.add_path(name, str(path)) + baseline_prompts = await target.read_all() + try: + result = await AgentOptimizer.optimize( + config_path=str(config_path), + call_agent=call_agent, + target_prompt=target, + train_dataset_path=str(filtered_path), + validation_dataset_path=str(validation_path), + output_dir=str(output_dir), + update_source=False, + ) + finally: + if await target.read_all() != baseline_prompts: + await target.write_all(baseline_prompts) + rounds = [record.model_dump(mode="json") if hasattr(record, "model_dump") else record for record in result.rounds] + config = json.loads(config_path.read_text(encoding="utf-8")) + return { + "algorithm": result.algorithm, + "total_rounds": result.total_rounds, + "rounds": rounds, + "baseline_pass_rate": result.baseline_pass_rate, + "best_pass_rate": result.best_pass_rate, + "pass_rate_improvement": result.pass_rate_improvement, + "baseline_metric_breakdown": result.baseline_metric_breakdown, + "best_metric_breakdown": result.best_metric_breakdown, + "metric_thresholds": result.metric_thresholds, + "baseline_prompts": result.baseline_prompts, + "best_prompts": result.best_prompts, + "cost": result.total_llm_cost, + "tokens": result.total_token_usage, + "duration_seconds": result.duration_seconds, + "seed": config["optimize"]["algorithm"].get("seed", 42), + "status": result.status, + } diff --git a/examples/optimization/eval_optimize_loop/pipeline/pipeline.py b/examples/optimization/eval_optimize_loop/pipeline/pipeline.py new file mode 100644 index 00000000..1f043f22 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/pipeline.py @@ -0,0 +1,385 @@ +"""End-to-end trust-aware counterfactual evaluation/optimization pipeline.""" + +from __future__ import annotations + +import hashlib +import json +import time +from pathlib import Path + +from trpc_agent_sdk.evaluation import EvalSet, get_all_tool_calls + +from ..fake.model import generate_trace +from ..fake.optimizer import optimize as fake_optimize +from .counterfactual import diagnose_trace_case +from .diagnosis import build_failure_digest, classify_non_agent_failure, select_target_prompts +from .gate import evaluate_gate +from .input_audit import audit_eval_cases +from .optimizer import apply_if_accepted +from .probe import evaluate_trace_cases + + +def _load(path: Path) -> EvalSet: + return EvalSet.model_validate_json(path.read_text(encoding="utf-8")) + + +def _text(case) -> str: + return " ".join(p.text or "" for p in case.conversation[0].user_content.parts).strip().lower() + + +def _pass(metrics: dict[str, float]) -> bool: + return bool(metrics) and all(v >= 1 for v in metrics.values()) + + +def _score(results: dict[str, dict[str, float]], total_cases: int | None = None) -> float: + denominator = len(results) if total_cases is None else total_cases + return sum(_pass(v) for v in results.values()) / denominator if denominator else 0.0 + + +def _metric_breakdown(results: dict[str, dict[str, float]], total_cases: int) -> dict[str, float]: + names = sorted({name for metrics in results.values() for name in metrics}) + return ( + {name: sum(metrics.get(name, 0.0) for metrics in results.values()) / total_cases for name in names} + if total_cases + else {} + ) + + +def _sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _content_text(content) -> str: + if not content: + return "" + return "".join(part.text or "" for part in content.parts) + + +async def _evaluate_with_triage(cases, workspace: Path): + """Evaluate each case so evaluator/infrastructure failures remain attributable.""" + metrics_by_id = {} + failures = {} + for case in cases: + try: + metrics_by_id.update(await evaluate_trace_cases([case], workspace)) + except Exception as error: # classification is intentionally centralized + failures[case.eval_id] = classify_non_agent_failure(case.eval_id, error=error) + return metrics_by_id, failures + + +def _case_results(cases, metrics_by_id: dict[str, dict[str, float]], failures=None) -> list[dict]: + failures = failures or {} + results = [] + for case in cases: + expected = case.conversation[0] + actual = case.actual_conversation[0] + metrics = metrics_by_id.get(case.eval_id, {}) + failure = failures.get(case.eval_id) + failed_metrics = sorted(name for name, score in metrics.items() if score < 1.0) + results.append( + { + "case_id": case.eval_id, + "passed": _pass(metrics) and failure is None, + "metrics": metrics, + "execution_failure": failure.to_dict() if failure else None, + "failure_reason": ( + failure.primary_category + if failure + else f"failed_metrics:{','.join(failed_metrics)}" if failed_metrics else None + ), + "trace_summary": { + "user_input": _content_text(expected.user_content), + "expected_tools": [tool.name for tool in get_all_tool_calls(expected.intermediate_data)], + "actual_tools": [tool.name for tool in get_all_tool_calls(actual.intermediate_data)], + "expected_final_response": _content_text(expected.final_response), + "actual_final_response": _content_text(actual.final_response), + }, + } + ) + return results + + +def _markdown(report: dict) -> str: + attrs = report["failure_attribution"]["items"] + deltas = report["candidate_validation"]["case_deltas"] + return "\n".join( + [ + "# Trust-Aware Counterfactual Optimization Report", + "", + f"Gate: **{'ACCEPTED' if report['gate']['accepted'] else 'REJECTED'}**", + "", + "## Baseline failures", + "", + *[f"- `{x['case_id']}`: {x['primary_category']} (actionable={x['prompt_actionable']})" for x in attrs], + "", + "## Candidate changes", + "", + *[f"- `{x['case_id']}`: {x['change']}" for x in deltas], + "", + "## Counterfactual regression diagnosis", + "", + *[ + f"- `{x['case_id']}`: {x['regression_category']} via {x['counterfactual_evidence']['intervention']}" + for x in report["regression_diagnosis"]["items"] + ], + "", + "## Decision", + "", + f"- Reasons: {', '.join(report['gate']['reason_codes']) or 'all checks passed'}", + f"- Source write-back: {report['audit']['write_back']['applied']}", + "", + "## Optimization", + "", + f"- Target prompts: {', '.join(report['target_selection']['selected']) or 'none'}", + f"- Candidate profile: {report['optimization'].get('candidate_profile', 'real')}", + "", + "## Known limitations", + "", + *[f"- {item}" for item in report["known_limitations"]], + "", + ] + ) + + +async def run_pipeline( + base_dir: Path, mode: str, output_dir: Path, apply: bool = False, candidate_profile: str = "overfit" +) -> dict: + if mode not in {"fake", "trace"}: + raise ValueError("mode must be fake or trace; real optimization uses run_real_optimizer()") + started = time.perf_counter() + train_path, val_path = base_dir / "train.evalset.json", base_dir / "val.evalset.json" + gate_config = json.loads((base_dir / "gate.json").read_text(encoding="utf-8")) + prompt_paths = { + name: base_dir / "prompts" / filename + for name, filename in { + "router_prompt": "router.md", + "skill_prompt": "skill.md", + "system_prompt": "system.md", + }.items() + } + prompts = {name: path.read_text(encoding="utf-8") for name, path in prompt_paths.items()} + train, val = _load(train_path), _load(val_path) + train_inputs, val_inputs = {_text(c) for c in train.eval_cases}, {_text(c) for c in val.eval_cases} + overlap = train_inputs & val_inputs + raw_cases = [ + { + "case_id": c.eval_id, + "normalized_input": _text(c), + "has_actual": bool(c.actual_conversation), + "has_reference": bool(c.conversation), + } + for c in train.eval_cases + val.eval_cases + ] + reliability = audit_eval_cases(raw_cases, overlap) + input_errors = [] + if train_path.resolve() == val_path.resolve(): + input_errors.append("train_and_validation_paths_equal") + ids = [c.eval_id for c in train.eval_cases + val.eval_cases] + if len(ids) != len(set(ids)): + input_errors.append("duplicate_eval_id") + if train_inputs == val_inputs: + input_errors.append("train_validation_inputs_identical") + + work = output_dir / ".trace_work" + baseline_train_cases = ( + [generate_trace(c, prompts) for c in train.eval_cases] if mode == "fake" else train.eval_cases + ) + baseline_val_cases = [generate_trace(c, prompts) for c in val.eval_cases] if mode == "fake" else val.eval_cases + baseline_train, baseline_train_errors = await _evaluate_with_triage(baseline_train_cases, work) + baseline_val, baseline_val_errors = await _evaluate_with_triage(baseline_val_cases, work) + trusted = {x["case_id"] for x in reliability if x["status"] == "trusted"} + attributions = [] + baseline_train_by_id = {c.eval_id: c for c in baseline_train_cases} + for case in train.eval_cases: + if case.eval_id in baseline_train_errors: + attributions.append(baseline_train_errors[case.eval_id]) + elif not _pass(baseline_train[case.eval_id]): + state = next(x for x in reliability if x["case_id"] == case.eval_id) + if state["status"] == "invalid": + attributions.append( + classify_non_agent_failure(case.eval_id, reliability="invalid", issues=state["issues"]) + ) + else: + attributions.append( + await diagnose_trace_case( + baseline_train_by_id[case.eval_id], work, gate_config["max_counterfactual_evaluations_per_case"] + ) + ) + targets = select_target_prompts([x for x in attributions if x.case_id in trusted]) + optimization = await fake_optimize(prompts, targets, profile=candidate_profile) + candidates_train = [generate_trace(c, optimization["best_prompts"]) for c in train.eval_cases] + candidates_val = [generate_trace(c, optimization["best_prompts"]) for c in val.eval_cases] + candidate_train, candidate_train_errors = await _evaluate_with_triage(candidates_train, work) + candidate_val, candidate_val_errors = await _evaluate_with_triage(candidates_val, work) + deltas, new_hard, protected = [], [], [] + for case in val.eval_cases: + before, after = baseline_val.get(case.eval_id, {}), candidate_val.get(case.eval_id, {}) + bp, ap = _pass(before), _pass(after) + candidate_error = candidate_val_errors.get(case.eval_id) + change = ( + "new_pass" + if not bp and ap + else ( + "new_fail" + if bp and not ap + else ( + "score_improved" + if sum(after.values()) > sum(before.values()) + else "score_regressed" if sum(after.values()) < sum(before.values()) else "unchanged" + ) + ) + ) + deltas.append( + { + "case_id": case.eval_id, + "change": change, + "before": before, + "after": after, + "execution_failure": candidate_error.to_dict() if candidate_error else None, + } + ) + if change == "new_fail": + new_hard.append(case.eval_id) + if case.eval_id in gate_config["protected_cases"]: + protected.append(case.eval_id) + regressions = [] + candidate_by_id = {c.eval_id: c for c in candidates_val} + for delta in deltas: + if delta["change"] in ("new_fail", "score_regressed"): + diagnosis = candidate_val_errors.get(delta["case_id"]) + proof = None + if diagnosis is None: + diagnosis = await diagnose_trace_case( + candidate_by_id[delta["case_id"]], work, gate_config["max_counterfactual_evaluations_per_case"] + ) + proof = next( + (e for e in diagnosis.evidence if hasattr(e, "changed_fail_to_pass") and e.changed_fail_to_pass), + None, + ) + regressions.append( + { + "case_id": delta["case_id"], + "change": delta["change"], + "regression_category": diagnosis.primary_category, + "candidate_related_surface": ( + diagnosis.recommended_target_prompts[0] if diagnosis.recommended_target_prompts else None + ), + "counterfactual_evidence": { + "intervention": proof.intervention if proof else None, + "result": "pass" if proof else "unresolved", + }, + } + ) + duration = time.perf_counter() - started + trusted_val_count = sum(case.eval_id in trusted for case in val.eval_cases) + severity_escalations = [ + {"case_id": item["case_id"], "before": "none", "after": item["regression_category"]} + for item in regressions + if item["change"] == "new_fail" + ] + gate = evaluate_gate( + { + "baseline_train": _score(baseline_train, len(train.eval_cases)), + "candidate_train": _score(candidate_train, len(train.eval_cases)), + "baseline_validation": _score(baseline_val, len(val.eval_cases)), + "candidate_validation": _score(candidate_val, len(val.eval_cases)), + "trusted_baseline": _score({k: v for k, v in baseline_val.items() if k in trusted}, trusted_val_count), + "trusted_candidate": _score({k: v for k, v in candidate_val.items() if k in trusted}, trusted_val_count), + "new_hard_fails": new_hard, + "protected_regressions": protected, + "severity_escalations": severity_escalations, + "cost": optimization["cost"], + "duration_seconds": duration, + "evidence_sufficient": ( + not candidate_train_errors + and not candidate_val_errors + and all(x["counterfactual_evidence"]["result"] == "pass" for x in regressions) + ), + }, + gate_config, + ) + for round_record in optimization["rounds"]: + round_record.update( + { + "validation_score": _score(candidate_val, len(val.eval_cases)), + "metric_breakdown": _metric_breakdown(candidate_val, len(val.eval_cases)), + "accepted": gate["accepted"], + "acceptance_reason": "all gate checks passed" if gate["accepted"] else ",".join(gate["reason_codes"]), + } + ) + write_back = await apply_if_accepted(gate, apply, optimization["best_prompts"], prompt_paths) + digest = build_failure_digest(attributions) + candidate_train_score = _score(candidate_train, len(train.eval_cases)) + candidate_validation_score = _score(candidate_val, len(val.eval_cases)) + baseline_train_score = _score(baseline_train, len(train.eval_cases)) + baseline_validation_score = _score(baseline_val, len(val.eval_cases)) + report = { + "schema_version": "1.0", + "input_audit": {"valid": not input_errors, "errors": input_errors}, + "baseline": { + "train": { + "score": baseline_train_score, + "cases": baseline_train, + "case_results": _case_results(baseline_train_cases, baseline_train, baseline_train_errors), + }, + "validation": { + "score": baseline_validation_score, + "cases": baseline_val, + "case_results": _case_results(baseline_val_cases, baseline_val, baseline_val_errors), + }, + }, + "evalset_reliability": {"cases": reliability}, + "failure_attribution": {"items": [x.to_dict() for x in attributions]}, + "prompt_actionability": digest, + "target_selection": {"selected": targets}, + "optimization": optimization, + "candidate": { + "train": { + "score": candidate_train_score, + "cases": candidate_train, + "case_results": _case_results(candidates_train, candidate_train, candidate_train_errors), + }, + "validation": { + "score": candidate_validation_score, + "cases": candidate_val, + "case_results": _case_results(candidates_val, candidate_val, candidate_val_errors), + }, + }, + "delta": { + "train_score": candidate_train_score - baseline_train_score, + "validation_score": candidate_validation_score - baseline_validation_score, + "case_deltas": deltas, + }, + "candidate_validation": { + "train_score": candidate_train_score, + "validation_score": candidate_validation_score, + "case_deltas": deltas, + }, + "regression_diagnosis": {"items": regressions}, + "gate": gate, + "audit": { + "seed": 42, + "mode": mode, + "duration_seconds": duration, + "cost": {"total": optimization["cost"]}, + "input_hashes": {"train": _sha(train_path), "validation": _sha(val_path)}, + "prompt_hashes": {k: _sha(v) for k, v in prompt_paths.items()}, + "write_back": write_back, + "reproduction_command": ( + "python examples/optimization/eval_optimize_loop/run_pipeline.py " + f"--mode {mode} --candidate-profile {candidate_profile}" + ), + }, + "known_limitations": [ + "A local trace edit can be structurally valid but semantically incoherent with an original tool response.", + "LLM-judge variance auditing requires repeated real-judge samples and is not " + "exercised by this deterministic example.", + "Real optimizer wiring is mock-verified; production execution requires " + "credentials, call_agent, and trace capture.", + ], + } + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "optimizer_failure_digest.json").write_text(json.dumps(digest, indent=2), encoding="utf-8") + (output_dir / "optimization_report.json").write_text(json.dumps(report, indent=2), encoding="utf-8") + (output_dir / "optimization_report.md").write_text(_markdown(report), encoding="utf-8") + return report diff --git a/examples/optimization/eval_optimize_loop/pipeline/probe.py b/examples/optimization/eval_optimize_loop/pipeline/probe.py new file mode 100644 index 00000000..1a68247d --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline/probe.py @@ -0,0 +1,300 @@ +"""Minimal official-evaluator probe for counterfactual trace diagnosis.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Iterable +from uuid import uuid4 + +from trpc_agent_sdk.evaluation import AgentEvaluator, EvalCase, EvalSet + +from .diagnosis import attribute_from_evidence +from .interventions import InterventionKind, build_counterfactual +from .models import CounterfactualEvidence + + +def _content(text: str, role: str) -> dict: + return {"role": role, "parts": [{"text": text}]} + + +def _invocation( + invocation_id: str, + user: str, + response: str, + tool_name: str | None = None, + tool_args: dict | None = None, +) -> dict: + invocation = { + "invocation_id": invocation_id, + "user_content": _content(user, "user"), + "final_response": _content(response, "model"), + } + if tool_name is not None: + invocation["intermediate_data"] = { + "tool_uses": [{"id": f"{invocation_id}-tool", "name": tool_name, "args": tool_args or {}}] + } + return invocation + + +def build_probe_cases() -> list[EvalCase]: + """Build three semantic traces without attribution labels or case metadata.""" + raw_cases = [ + { + "eval_id": "probe_final_response", + "eval_mode": "trace", + "conversation": [_invocation("expected-a", "What is the refund status?", "Refund is pending.")], + "actual_conversation": [_invocation("actual-a", "What is the refund status?", "Refund was denied.")], + }, + { + "eval_id": "probe_tool_name", + "eval_mode": "trace", + "conversation": [ + _invocation( + "expected-b", + "Find refund R-102.", + "Refund R-102 is pending.", + "get_refund", + {"refund_id": "R-102"}, + ) + ], + "actual_conversation": [ + _invocation( + "actual-b", + "Find refund R-102.", + "Refund R-102 is pending.", + "get_invoice", + {"refund_id": "R-102"}, + ) + ], + }, + { + "eval_id": "probe_compound_tool", + "eval_mode": "trace", + "conversation": [ + _invocation( + "expected-c", + "Refund order O-77 because it is damaged.", + "Refund for O-77 was submitted.", + "create_refund", + {"order_id": "O-77", "reason": "damaged"}, + ) + ], + "actual_conversation": [ + _invocation( + "actual-c", + "Refund order O-77 because it is damaged.", + "Refund for O-77 was submitted.", + "get_invoice", + {"invoice_id": "O-77"}, + ) + ], + }, + ] + return [EvalCase.model_validate(case) for case in raw_cases] + + +async def evaluate_trace_cases(cases: Iterable[EvalCase], workspace: Path) -> dict[str, dict[str, float]]: + """Evaluate trace cases with the public executor and return per-case metric scores.""" + workspace.mkdir(parents=True, exist_ok=True) + run_id = uuid4().hex + evalset_path = workspace / f"counterfactual-{run_id}.evalset.json" + metrics_path = workspace / f"counterfactual-{run_id}.metrics.json" + + eval_set = EvalSet(eval_set_id=f"counterfactual_{run_id}", eval_cases=list(cases)) + evalset_path.write_text(eval_set.model_dump_json(by_alias=True, indent=2), encoding="utf-8") + metrics_path.write_text( + json.dumps( + { + "metrics": [ + {"metric_name": "tool_trajectory_avg_score", "threshold": 1.0}, + {"metric_name": "final_response_avg_score", "threshold": 1.0}, + ] + }, + indent=2, + ), + encoding="utf-8", + ) + + executor = AgentEvaluator.get_executer( + os.path.relpath(evalset_path, Path.cwd()), + eval_metrics_file_path_or_dir=os.path.relpath(metrics_path, Path.cwd()), + print_detailed_results=False, + print_summary_report=False, + ) + try: + await executor.evaluate() + except Exception: + # AgentEvaluator raises when one or more cases fail, after preserving + # the EvaluateResult. Counterfactual diagnosis needs those failed + # per-case metrics; genuine execution failures have no result. + if executor.get_result() is None: + raise + result = executor.get_result() + + scores: dict[str, dict[str, float]] = {} + for aggregate in result.results_by_eval_set_id.values(): + for eval_id, runs in aggregate.eval_results_by_eval_id.items(): + if not runs: + continue + scores[eval_id] = { + metric.metric_name: float(metric.score or 0.0) for metric in runs[0].overall_eval_metric_results + } + evalset_path.unlink(missing_ok=True) + metrics_path.unlink(missing_ok=True) + return scores + + +def _passed(metrics: dict[str, float]) -> bool: + return bool(metrics) and all(score >= 1.0 for score in metrics.values()) + + +def _render_probe_markdown(report: dict) -> str: + lines = [ + "# Counterfactual Trace Feasibility Probe", + "", + f"Supported: **{str(report['feasibility']['supported']).lower()}**", + "", + "The probe uses trace-mode `EvalCase` objects and the public " + "`AgentEvaluator.get_executer()` API for baseline and counterfactual scoring.", + "", + "| Case | Intervention | Legal | Fail to pass | Repaired metrics | Unchanged metrics |", + "|---|---|---:|---:|---|---|", + ] + for case in report["cases"]: + for item in case["interventions"]: + lines.append( + "| {case} | {intervention} | {legal} | {fixed} | {repaired} | {unchanged} |".format( + case=case["case_id"], + intervention=item["intervention"], + legal=str(item["construction"]["valid"]).lower(), + fixed=str(item["changed_fail_to_pass"]).lower(), + repaired=", ".join(item["repaired_metrics"]) or "-", + unchanged=", ".join(item["unchanged_metrics"]) or "-", + ) + ) + lines.extend( + [ + "", + "## Conclusion", + "", + report["feasibility"]["reason"], + "", + ] + ) + return "\n".join(lines) + + +async def run_counterfactual_probe(output_dir: Path, cases: list[EvalCase] | None = None) -> dict: + """Run and persist the A/B/C feasibility experiment.""" + output_dir.mkdir(parents=True, exist_ok=True) + cases = cases or build_probe_cases() + originals = {case.eval_id: case.model_dump(mode="json", by_alias=True) for case in cases} + baseline = await evaluate_trace_cases(cases, output_dir) + + built: dict[str, list] = {} + valid_cases = [] + for case in cases: + built[case.eval_id] = [] + for kind in InterventionKind: + result = build_counterfactual(case, kind) + built[case.eval_id].append(result) + if result.valid and result.eval_case is not None: + valid_cases.append(result.eval_case) + changed = await evaluate_trace_cases(valid_cases, output_dir) + + case_reports = [] + for case in cases: + before = baseline[case.eval_id] + interventions = [] + for result in built[case.eval_id]: + after = changed.get(result.eval_case.eval_id, {}) if result.eval_case else {} + repaired = sorted(name for name, score in before.items() if score < 1.0 and after.get(name, score) >= 1.0) + unchanged = sorted(name for name, score in before.items() if after.get(name) == score) + interventions.append( + { + "intervention": result.intervention.value, + "construction": { + "valid": result.valid, + "status": result.status, + "structurally_valid": result.structurally_valid, + "semantically_coherent": result.semantically_coherent, + "coherence_warnings": list(result.coherence_warnings), + }, + "structurally_valid": result.structurally_valid, + "semantically_coherent": result.semantically_coherent, + "coherence_warnings": list(result.coherence_warnings), + "before_metrics": before, + "after_metrics": after, + "changed_fail_to_pass": (result.valid and not _passed(before) and _passed(after)), + "repaired_metrics": repaired, + "unchanged_metrics": unchanged, + } + ) + evidence = [ + CounterfactualEvidence( + intervention=item["intervention"], + valid=item["construction"]["valid"], + status=item["construction"]["status"], + changed_fail_to_pass=item["changed_fail_to_pass"], + repaired_metrics=item["repaired_metrics"], + unchanged_metrics=item["unchanged_metrics"], + before_metrics=item["before_metrics"], + after_metrics=item["after_metrics"], + structurally_valid=item["structurally_valid"], + semantically_coherent=item["semantically_coherent"], + coherence_warnings=item["coherence_warnings"], + ) + for item in interventions + ] + diagnosis = attribute_from_evidence(case.eval_id, evidence) + case_reports.append( + { + "case_id": case.eval_id, + "baseline_metrics": before, + "baseline_passed": _passed(before), + "interventions": interventions, + "diagnosis": { + "primary_category": diagnosis.primary_category, + "secondary_categories": diagnosis.secondary_categories, + "compound_failure": diagnosis.primary_category == "compound_failure", + "confidence": diagnosis.confidence, + }, + } + ) + + intervention_sets = [{item["intervention"]: item for item in case["interventions"]} for case in case_reports] + supported = ( + any(items["replace_final_response"]["changed_fail_to_pass"] for items in intervention_sets) + and any(items["replace_tool_name"]["changed_fail_to_pass"] for items in intervention_sets) + and any( + not items["replace_tool_name"]["changed_fail_to_pass"] + and not items["replace_tool_arguments"]["changed_fail_to_pass"] + and items["replace_tool_name_and_arguments"]["changed_fail_to_pass"] + for items in intervention_sets + ) + ) + source_unchanged = all(case.model_dump(mode="json", by_alias=True) == originals[case.eval_id] for case in cases) + report = { + "schema_version": "1.0", + "probe": "trust-aware-counterfactual-trace-diagnosis", + "official_api": "AgentEvaluator.get_executer", + "feasibility": { + "supported": supported, + "reason": ( + "Official trace metrics distinguish final-response, tool-name, and " + "tool-argument interventions; the compound case passes only after the " + "combined tool-name-and-arguments intervention." + if supported + else "Official trace metrics did not produce the required explanatory deltas." + ), + }, + "source_trace_unchanged": source_unchanged, + "cases": case_reports, + } + (output_dir / "counterfactual_probe.json").write_text( + json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8" + ) + (output_dir / "counterfactual_probe.md").write_text(_render_probe_markdown(report), encoding="utf-8") + return report diff --git a/examples/optimization/eval_optimize_loop/prompts/router.md b/examples/optimization/eval_optimize_loop/prompts/router.md new file mode 100644 index 00000000..d4c41833 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/prompts/router.md @@ -0,0 +1 @@ +Route refund requests to `create_refund`, invoice questions to `get_invoice`, and shipping questions to `track_shipment`. diff --git a/examples/optimization/eval_optimize_loop/prompts/skill.md b/examples/optimization/eval_optimize_loop/prompts/skill.md new file mode 100644 index 00000000..20b6eeca --- /dev/null +++ b/examples/optimization/eval_optimize_loop/prompts/skill.md @@ -0,0 +1 @@ +Extract identifiers explicitly supplied by the user. Do not invent missing fields. diff --git a/examples/optimization/eval_optimize_loop/prompts/system.md b/examples/optimization/eval_optimize_loop/prompts/system.md new file mode 100644 index 00000000..252b8426 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/prompts/system.md @@ -0,0 +1 @@ +Return a compact JSON object describing the operation result. diff --git a/examples/optimization/eval_optimize_loop/prototype_output/counterfactual_probe.json b/examples/optimization/eval_optimize_loop/prototype_output/counterfactual_probe.json new file mode 100644 index 00000000..629f1c16 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/prototype_output/counterfactual_probe.json @@ -0,0 +1,616 @@ +{ + "schema_version": "1.0", + "probe": "trust-aware-counterfactual-trace-diagnosis", + "official_api": "AgentEvaluator.get_executer", + "feasibility": { + "supported": true, + "reason": "Official trace metrics distinguish final-response, tool-name, and tool-argument interventions; the compound case passes only after the combined tool-name-and-arguments intervention." + }, + "source_trace_unchanged": true, + "cases": [ + { + "case_id": "probe_final_response", + "baseline_metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 0.0 + }, + "baseline_passed": false, + "interventions": [ + { + "intervention": "replace_final_response", + "construction": { + "valid": true, + "status": "constructed", + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [], + "before_metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 0.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "changed_fail_to_pass": true, + "repaired_metrics": [ + "final_response_avg_score" + ], + "unchanged_metrics": [ + "tool_trajectory_avg_score" + ] + }, + { + "intervention": "replace_tool_name", + "construction": { + "valid": false, + "status": "missing_expected_tool", + "structurally_valid": false, + "semantically_coherent": false, + "coherence_warnings": [ + "missing_expected_tool" + ] + }, + "structurally_valid": false, + "semantically_coherent": false, + "coherence_warnings": [ + "missing_expected_tool" + ], + "before_metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 0.0 + }, + "after_metrics": {}, + "changed_fail_to_pass": false, + "repaired_metrics": [], + "unchanged_metrics": [] + }, + { + "intervention": "replace_tool_arguments", + "construction": { + "valid": false, + "status": "missing_expected_tool", + "structurally_valid": false, + "semantically_coherent": false, + "coherence_warnings": [ + "missing_expected_tool" + ] + }, + "structurally_valid": false, + "semantically_coherent": false, + "coherence_warnings": [ + "missing_expected_tool" + ], + "before_metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 0.0 + }, + "after_metrics": {}, + "changed_fail_to_pass": false, + "repaired_metrics": [], + "unchanged_metrics": [] + }, + { + "intervention": "replace_tool_name_and_arguments", + "construction": { + "valid": false, + "status": "missing_expected_tool", + "structurally_valid": false, + "semantically_coherent": false, + "coherence_warnings": [ + "missing_expected_tool" + ] + }, + "structurally_valid": false, + "semantically_coherent": false, + "coherence_warnings": [ + "missing_expected_tool" + ], + "before_metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 0.0 + }, + "after_metrics": {}, + "changed_fail_to_pass": false, + "repaired_metrics": [], + "unchanged_metrics": [] + }, + { + "intervention": "replace_tool_name+final_response", + "construction": { + "valid": false, + "status": "missing_expected_tool", + "structurally_valid": false, + "semantically_coherent": false, + "coherence_warnings": [ + "missing_expected_tool" + ] + }, + "structurally_valid": false, + "semantically_coherent": false, + "coherence_warnings": [ + "missing_expected_tool" + ], + "before_metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 0.0 + }, + "after_metrics": {}, + "changed_fail_to_pass": false, + "repaired_metrics": [], + "unchanged_metrics": [] + }, + { + "intervention": "replace_tool_arguments+final_response", + "construction": { + "valid": false, + "status": "missing_expected_tool", + "structurally_valid": false, + "semantically_coherent": false, + "coherence_warnings": [ + "missing_expected_tool" + ] + }, + "structurally_valid": false, + "semantically_coherent": false, + "coherence_warnings": [ + "missing_expected_tool" + ], + "before_metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 0.0 + }, + "after_metrics": {}, + "changed_fail_to_pass": false, + "repaired_metrics": [], + "unchanged_metrics": [] + }, + { + "intervention": "normalize_format", + "construction": { + "valid": false, + "status": "format_not_normalizable", + "structurally_valid": false, + "semantically_coherent": false, + "coherence_warnings": [ + "format_not_normalizable" + ] + }, + "structurally_valid": false, + "semantically_coherent": false, + "coherence_warnings": [ + "format_not_normalizable" + ], + "before_metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 0.0 + }, + "after_metrics": {}, + "changed_fail_to_pass": false, + "repaired_metrics": [], + "unchanged_metrics": [] + } + ], + "diagnosis": { + "primary_category": "final_response_mismatch", + "secondary_categories": [], + "compound_failure": false, + "confidence": 0.95 + } + }, + { + "case_id": "probe_tool_name", + "baseline_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "baseline_passed": false, + "interventions": [ + { + "intervention": "replace_final_response", + "construction": { + "valid": true, + "status": "constructed", + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "changed_fail_to_pass": false, + "repaired_metrics": [], + "unchanged_metrics": [ + "final_response_avg_score", + "tool_trajectory_avg_score" + ] + }, + { + "intervention": "replace_tool_name", + "construction": { + "valid": true, + "status": "constructed", + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "changed_fail_to_pass": true, + "repaired_metrics": [ + "tool_trajectory_avg_score" + ], + "unchanged_metrics": [ + "final_response_avg_score" + ] + }, + { + "intervention": "replace_tool_arguments", + "construction": { + "valid": true, + "status": "constructed", + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "changed_fail_to_pass": false, + "repaired_metrics": [], + "unchanged_metrics": [ + "final_response_avg_score", + "tool_trajectory_avg_score" + ] + }, + { + "intervention": "replace_tool_name_and_arguments", + "construction": { + "valid": true, + "status": "constructed", + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "changed_fail_to_pass": true, + "repaired_metrics": [ + "tool_trajectory_avg_score" + ], + "unchanged_metrics": [ + "final_response_avg_score" + ] + }, + { + "intervention": "replace_tool_name+final_response", + "construction": { + "valid": true, + "status": "constructed", + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "changed_fail_to_pass": true, + "repaired_metrics": [ + "tool_trajectory_avg_score" + ], + "unchanged_metrics": [ + "final_response_avg_score" + ] + }, + { + "intervention": "replace_tool_arguments+final_response", + "construction": { + "valid": true, + "status": "constructed", + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "changed_fail_to_pass": false, + "repaired_metrics": [], + "unchanged_metrics": [ + "final_response_avg_score", + "tool_trajectory_avg_score" + ] + }, + { + "intervention": "normalize_format", + "construction": { + "valid": false, + "status": "format_not_normalizable", + "structurally_valid": false, + "semantically_coherent": false, + "coherence_warnings": [ + "format_not_normalizable" + ] + }, + "structurally_valid": false, + "semantically_coherent": false, + "coherence_warnings": [ + "format_not_normalizable" + ], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": {}, + "changed_fail_to_pass": false, + "repaired_metrics": [], + "unchanged_metrics": [] + } + ], + "diagnosis": { + "primary_category": "tool_selection_error", + "secondary_categories": [], + "compound_failure": false, + "confidence": 0.95 + } + }, + { + "case_id": "probe_compound_tool", + "baseline_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "baseline_passed": false, + "interventions": [ + { + "intervention": "replace_final_response", + "construction": { + "valid": true, + "status": "constructed", + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "changed_fail_to_pass": false, + "repaired_metrics": [], + "unchanged_metrics": [ + "final_response_avg_score", + "tool_trajectory_avg_score" + ] + }, + { + "intervention": "replace_tool_name", + "construction": { + "valid": true, + "status": "constructed", + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "changed_fail_to_pass": false, + "repaired_metrics": [], + "unchanged_metrics": [ + "final_response_avg_score", + "tool_trajectory_avg_score" + ] + }, + { + "intervention": "replace_tool_arguments", + "construction": { + "valid": true, + "status": "constructed", + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "changed_fail_to_pass": false, + "repaired_metrics": [], + "unchanged_metrics": [ + "final_response_avg_score", + "tool_trajectory_avg_score" + ] + }, + { + "intervention": "replace_tool_name_and_arguments", + "construction": { + "valid": true, + "status": "constructed", + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "changed_fail_to_pass": true, + "repaired_metrics": [ + "tool_trajectory_avg_score" + ], + "unchanged_metrics": [ + "final_response_avg_score" + ] + }, + { + "intervention": "replace_tool_name+final_response", + "construction": { + "valid": true, + "status": "constructed", + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "changed_fail_to_pass": false, + "repaired_metrics": [], + "unchanged_metrics": [ + "final_response_avg_score", + "tool_trajectory_avg_score" + ] + }, + { + "intervention": "replace_tool_arguments+final_response", + "construction": { + "valid": true, + "status": "constructed", + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "changed_fail_to_pass": false, + "repaired_metrics": [], + "unchanged_metrics": [ + "final_response_avg_score", + "tool_trajectory_avg_score" + ] + }, + { + "intervention": "normalize_format", + "construction": { + "valid": false, + "status": "format_not_normalizable", + "structurally_valid": false, + "semantically_coherent": false, + "coherence_warnings": [ + "format_not_normalizable" + ] + }, + "structurally_valid": false, + "semantically_coherent": false, + "coherence_warnings": [ + "format_not_normalizable" + ], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": {}, + "changed_fail_to_pass": false, + "repaired_metrics": [], + "unchanged_metrics": [] + } + ], + "diagnosis": { + "primary_category": "compound_failure", + "secondary_categories": [], + "compound_failure": true, + "confidence": 0.9 + } + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/prototype_output/counterfactual_probe.md b/examples/optimization/eval_optimize_loop/prototype_output/counterfactual_probe.md new file mode 100644 index 00000000..bbf7eb35 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/prototype_output/counterfactual_probe.md @@ -0,0 +1,33 @@ +# Counterfactual Trace Feasibility Probe + +Supported: **true** + +The probe uses trace-mode `EvalCase` objects and the public `AgentEvaluator.get_executer()` API for baseline and counterfactual scoring. + +| Case | Intervention | Legal | Fail to pass | Repaired metrics | Unchanged metrics | +|---|---|---:|---:|---|---| +| probe_final_response | replace_final_response | true | true | final_response_avg_score | tool_trajectory_avg_score | +| probe_final_response | replace_tool_name | false | false | - | - | +| probe_final_response | replace_tool_arguments | false | false | - | - | +| probe_final_response | replace_tool_name_and_arguments | false | false | - | - | +| probe_final_response | replace_tool_name+final_response | false | false | - | - | +| probe_final_response | replace_tool_arguments+final_response | false | false | - | - | +| probe_final_response | normalize_format | false | false | - | - | +| probe_tool_name | replace_final_response | true | false | - | final_response_avg_score, tool_trajectory_avg_score | +| probe_tool_name | replace_tool_name | true | true | tool_trajectory_avg_score | final_response_avg_score | +| probe_tool_name | replace_tool_arguments | true | false | - | final_response_avg_score, tool_trajectory_avg_score | +| probe_tool_name | replace_tool_name_and_arguments | true | true | tool_trajectory_avg_score | final_response_avg_score | +| probe_tool_name | replace_tool_name+final_response | true | true | tool_trajectory_avg_score | final_response_avg_score | +| probe_tool_name | replace_tool_arguments+final_response | true | false | - | final_response_avg_score, tool_trajectory_avg_score | +| probe_tool_name | normalize_format | false | false | - | - | +| probe_compound_tool | replace_final_response | true | false | - | final_response_avg_score, tool_trajectory_avg_score | +| probe_compound_tool | replace_tool_name | true | false | - | final_response_avg_score, tool_trajectory_avg_score | +| probe_compound_tool | replace_tool_arguments | true | false | - | final_response_avg_score, tool_trajectory_avg_score | +| probe_compound_tool | replace_tool_name_and_arguments | true | true | tool_trajectory_avg_score | final_response_avg_score | +| probe_compound_tool | replace_tool_name+final_response | true | false | - | final_response_avg_score, tool_trajectory_avg_score | +| probe_compound_tool | replace_tool_arguments+final_response | true | false | - | final_response_avg_score, tool_trajectory_avg_score | +| probe_compound_tool | normalize_format | false | false | - | - | + +## Conclusion + +Official trace metrics distinguish final-response, tool-name, and tool-argument interventions; the compound case passes only after the combined tool-name-and-arguments intervention. diff --git a/examples/optimization/eval_optimize_loop/run_counterfactual_probe.py b/examples/optimization/eval_optimize_loop/run_counterfactual_probe.py new file mode 100644 index 00000000..550cb53b --- /dev/null +++ b/examples/optimization/eval_optimize_loop/run_counterfactual_probe.py @@ -0,0 +1,35 @@ +"""Run the minimal counterfactual trace feasibility probe.""" + +from __future__ import annotations + +import argparse +import asyncio +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +REPO_ROOT = HERE.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from examples.optimization.eval_optimize_loop.pipeline.probe import ( # noqa: E402 + run_counterfactual_probe, +) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--output-dir", + type=Path, + default=HERE / "prototype_output", + help="Directory for counterfactual_probe.json and counterfactual_probe.md", + ) + args = parser.parse_args() + report = asyncio.run(run_counterfactual_probe(args.output_dir.resolve())) + if not report["feasibility"]["supported"]: + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py new file mode 100644 index 00000000..71eafd16 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -0,0 +1,30 @@ +"""Run the trust-aware counterfactual evaluation/optimization loop.""" + +from __future__ import annotations + +import argparse +import asyncio +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) +from examples.optimization.eval_optimize_loop.pipeline.pipeline import run_pipeline # noqa: E402 + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--mode", choices=["fake", "trace"], default="fake") + p.add_argument("--candidate-profile", choices=["accepted", "overfit", "ineffective"], default="overfit") + p.add_argument("--output-dir", type=Path, default=None) + p.add_argument("--apply", action="store_true") + a = p.parse_args() + asyncio.run( + run_pipeline(HERE, a.mode, (a.output_dir or HERE / "sample_output").resolve(), a.apply, a.candidate_profile) + ) + + +if __name__ == "__main__": + main() diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json new file mode 100644 index 00000000..fba3689e --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json @@ -0,0 +1,1007 @@ +{ + "schema_version": "1.0", + "input_audit": { + "valid": true, + "errors": [] + }, + "baseline": { + "train": { + "score": 0.0, + "cases": { + "train_route": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "train_args": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "train_json": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 0.0 + } + }, + "case_results": [ + { + "case_id": "train_route", + "passed": false, + "metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "execution_failure": null, + "failure_reason": "failed_metrics:tool_trajectory_avg_score", + "trace_summary": { + "user_input": "Refund order O-1 because damaged", + "expected_tools": [ + "create_refund" + ], + "actual_tools": [ + "get_invoice" + ], + "expected_final_response": "{\"status\":\"submitted\",\"order_id\":\"O-1\"}", + "actual_final_response": "{\"status\":\"submitted\",\"order_id\":\"O-1\"}" + } + }, + { + "case_id": "train_args", + "passed": false, + "metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "execution_failure": null, + "failure_reason": "failed_metrics:tool_trajectory_avg_score", + "trace_summary": { + "user_input": "Refund order O-2 because duplicate", + "expected_tools": [ + "create_refund" + ], + "actual_tools": [ + "create_refund" + ], + "expected_final_response": "{\"status\":\"submitted\",\"order_id\":\"O-2\"}", + "actual_final_response": "{\"status\":\"submitted\",\"order_id\":\"O-2\"}" + } + }, + { + "case_id": "train_json", + "passed": false, + "metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 0.0 + }, + "execution_failure": null, + "failure_reason": "failed_metrics:final_response_avg_score", + "trace_summary": { + "user_input": "Summarize refund O-3", + "expected_tools": [], + "actual_tools": [], + "expected_final_response": "{\"status\":\"pending\",\"order_id\":\"O-3\"}", + "actual_final_response": "{\"order_id\":\"O-3\"}" + } + } + ] + }, + "validation": { + "score": 0.6666666666666666, + "cases": { + "val_refund": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "val_shipping": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "val_billing": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + } + }, + "case_results": [ + { + "case_id": "val_refund", + "passed": false, + "metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "execution_failure": null, + "failure_reason": "failed_metrics:tool_trajectory_avg_score", + "trace_summary": { + "user_input": "Refund order O-9 because damaged", + "expected_tools": [ + "create_refund" + ], + "actual_tools": [ + "get_invoice" + ], + "expected_final_response": "{\"status\":\"submitted\",\"order_id\":\"O-9\"}", + "actual_final_response": "{\"status\":\"submitted\",\"order_id\":\"O-9\"}" + } + }, + { + "case_id": "val_shipping", + "passed": true, + "metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "execution_failure": null, + "failure_reason": null, + "trace_summary": { + "user_input": "Track shipment T-1", + "expected_tools": [ + "track_shipment" + ], + "actual_tools": [ + "track_shipment" + ], + "expected_final_response": "{\"tracking_id\":\"T-1\",\"status\":\"in_transit\"}", + "actual_final_response": "{\"tracking_id\":\"T-1\",\"status\":\"in_transit\"}" + } + }, + { + "case_id": "val_billing", + "passed": true, + "metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "execution_failure": null, + "failure_reason": null, + "trace_summary": { + "user_input": "Check billing invoice I-9", + "expected_tools": [ + "get_invoice" + ], + "actual_tools": [ + "get_invoice" + ], + "expected_final_response": "{\"invoice_id\":\"I-9\",\"status\":\"open\"}", + "actual_final_response": "{\"invoice_id\":\"I-9\",\"status\":\"open\"}" + } + } + ] + } + }, + "evalset_reliability": { + "cases": [ + { + "case_id": "train_route", + "status": "trusted", + "issues": [] + }, + { + "case_id": "train_args", + "status": "trusted", + "issues": [] + }, + { + "case_id": "train_json", + "status": "trusted", + "issues": [] + }, + { + "case_id": "val_refund", + "status": "trusted", + "issues": [] + }, + { + "case_id": "val_shipping", + "status": "trusted", + "issues": [] + }, + { + "case_id": "val_billing", + "status": "trusted", + "issues": [] + } + ] + }, + "failure_attribution": { + "items": [ + { + "case_id": "train_route", + "failure_domain": "agent_behavior_failure", + "primary_category": "tool_selection_error", + "secondary_categories": [], + "prompt_actionable": true, + "confidence": 0.95, + "evidence": [ + { + "intervention": "replace_final_response", + "valid": true, + "status": "constructed", + "changed_fail_to_pass": false, + "repaired_metrics": [], + "unchanged_metrics": [ + "final_response_avg_score", + "tool_trajectory_avg_score" + ], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + }, + { + "intervention": "replace_tool_name", + "valid": true, + "status": "constructed", + "changed_fail_to_pass": true, + "repaired_metrics": [ + "tool_trajectory_avg_score" + ], + "unchanged_metrics": [ + "final_response_avg_score" + ], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + } + ], + "recommended_target_prompts": [ + "router_prompt" + ], + "evaluations_used": 2 + }, + { + "case_id": "train_args", + "failure_domain": "agent_behavior_failure", + "primary_category": "tool_parameter_error", + "secondary_categories": [], + "prompt_actionable": true, + "confidence": 0.95, + "evidence": [ + { + "intervention": "replace_final_response", + "valid": true, + "status": "constructed", + "changed_fail_to_pass": false, + "repaired_metrics": [], + "unchanged_metrics": [ + "final_response_avg_score", + "tool_trajectory_avg_score" + ], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + }, + { + "intervention": "replace_tool_name", + "valid": true, + "status": "constructed", + "changed_fail_to_pass": false, + "repaired_metrics": [], + "unchanged_metrics": [ + "final_response_avg_score", + "tool_trajectory_avg_score" + ], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + }, + { + "intervention": "replace_tool_arguments", + "valid": true, + "status": "constructed", + "changed_fail_to_pass": true, + "repaired_metrics": [ + "tool_trajectory_avg_score" + ], + "unchanged_metrics": [ + "final_response_avg_score" + ], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + } + ], + "recommended_target_prompts": [ + "skill_prompt" + ], + "evaluations_used": 3 + }, + { + "case_id": "train_json", + "failure_domain": "agent_behavior_failure", + "primary_category": "final_response_mismatch", + "secondary_categories": [], + "prompt_actionable": true, + "confidence": 0.95, + "evidence": [ + { + "intervention": "replace_final_response", + "valid": true, + "status": "constructed", + "changed_fail_to_pass": true, + "repaired_metrics": [ + "final_response_avg_score" + ], + "unchanged_metrics": [ + "tool_trajectory_avg_score" + ], + "before_metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 0.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + } + ], + "recommended_target_prompts": [ + "system_prompt" + ], + "evaluations_used": 1 + } + ] + }, + "prompt_actionability": { + "actionable_failures": [ + { + "case_id": "train_route", + "failure_domain": "agent_behavior_failure", + "primary_category": "tool_selection_error", + "secondary_categories": [], + "prompt_actionable": true, + "confidence": 0.95, + "evidence": [ + { + "intervention": "replace_final_response", + "valid": true, + "status": "constructed", + "changed_fail_to_pass": false, + "repaired_metrics": [], + "unchanged_metrics": [ + "final_response_avg_score", + "tool_trajectory_avg_score" + ], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + }, + { + "intervention": "replace_tool_name", + "valid": true, + "status": "constructed", + "changed_fail_to_pass": true, + "repaired_metrics": [ + "tool_trajectory_avg_score" + ], + "unchanged_metrics": [ + "final_response_avg_score" + ], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + } + ], + "recommended_target_prompts": [ + "router_prompt" + ], + "evaluations_used": 2 + }, + { + "case_id": "train_args", + "failure_domain": "agent_behavior_failure", + "primary_category": "tool_parameter_error", + "secondary_categories": [], + "prompt_actionable": true, + "confidence": 0.95, + "evidence": [ + { + "intervention": "replace_final_response", + "valid": true, + "status": "constructed", + "changed_fail_to_pass": false, + "repaired_metrics": [], + "unchanged_metrics": [ + "final_response_avg_score", + "tool_trajectory_avg_score" + ], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + }, + { + "intervention": "replace_tool_name", + "valid": true, + "status": "constructed", + "changed_fail_to_pass": false, + "repaired_metrics": [], + "unchanged_metrics": [ + "final_response_avg_score", + "tool_trajectory_avg_score" + ], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + }, + { + "intervention": "replace_tool_arguments", + "valid": true, + "status": "constructed", + "changed_fail_to_pass": true, + "repaired_metrics": [ + "tool_trajectory_avg_score" + ], + "unchanged_metrics": [ + "final_response_avg_score" + ], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + } + ], + "recommended_target_prompts": [ + "skill_prompt" + ], + "evaluations_used": 3 + }, + { + "case_id": "train_json", + "failure_domain": "agent_behavior_failure", + "primary_category": "final_response_mismatch", + "secondary_categories": [], + "prompt_actionable": true, + "confidence": 0.95, + "evidence": [ + { + "intervention": "replace_final_response", + "valid": true, + "status": "constructed", + "changed_fail_to_pass": true, + "repaired_metrics": [ + "final_response_avg_score" + ], + "unchanged_metrics": [ + "tool_trajectory_avg_score" + ], + "before_metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 0.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + } + ], + "recommended_target_prompts": [ + "system_prompt" + ], + "evaluations_used": 1 + } + ], + "excluded_failures": [], + "category_statistics": { + "tool_selection_error": 1, + "tool_parameter_error": 1, + "final_response_mismatch": 1 + } + }, + "target_selection": { + "selected": [ + "router_prompt", + "skill_prompt", + "system_prompt" + ] + }, + "optimization": { + "total_rounds": 1, + "rounds": [ + { + "round": 1, + "profile": "overfit", + "targets": [ + "router_prompt", + "skill_prompt", + "system_prompt" + ], + "candidate_prompts": { + "router_prompt": "Route refund requests to `create_refund`, invoice questions to `get_invoice`, and shipping questions to `track_shipment`.\n\nREFUND_ROUTE=STRICT\nBILLING_TO_REFUND=ON\n", + "skill_prompt": "Extract identifiers explicitly supplied by the user. Do not invent missing fields.\n\nREFUND_REASON=REQUIRED\n", + "system_prompt": "Return a compact JSON object describing the operation result.\n\nJSON_STATUS=ALWAYS_REQUIRED\n" + }, + "cost": 0.0, + "tokens": 0, + "duration_seconds": 0.0, + "validation_score": 0.6666666666666666, + "metric_breakdown": { + "final_response_avg_score": 1.0, + "tool_trajectory_avg_score": 0.6666666666666666 + }, + "accepted": false, + "acceptance_reason": "VALIDATION_DELTA,NEW_HARD_FAIL,PROTECTED_REGRESSION,TRAIN_ONLY_IMPROVEMENT,SEVERITY_ESCALATION" + } + ], + "best_prompts": { + "router_prompt": "Route refund requests to `create_refund`, invoice questions to `get_invoice`, and shipping questions to `track_shipment`.\n\nREFUND_ROUTE=STRICT\nBILLING_TO_REFUND=ON\n", + "skill_prompt": "Extract identifiers explicitly supplied by the user. Do not invent missing fields.\n\nREFUND_REASON=REQUIRED\n", + "system_prompt": "Return a compact JSON object describing the operation result.\n\nJSON_STATUS=ALWAYS_REQUIRED\n" + }, + "cost": 0.0, + "tokens": 0, + "seed": 42, + "candidate_profile": "overfit" + }, + "candidate": { + "train": { + "score": 1.0, + "cases": { + "train_route": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "train_args": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "train_json": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + } + }, + "case_results": [ + { + "case_id": "train_route", + "passed": true, + "metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "execution_failure": null, + "failure_reason": null, + "trace_summary": { + "user_input": "Refund order O-1 because damaged", + "expected_tools": [ + "create_refund" + ], + "actual_tools": [ + "create_refund" + ], + "expected_final_response": "{\"status\":\"submitted\",\"order_id\":\"O-1\"}", + "actual_final_response": "{\"status\":\"submitted\",\"order_id\":\"O-1\"}" + } + }, + { + "case_id": "train_args", + "passed": true, + "metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "execution_failure": null, + "failure_reason": null, + "trace_summary": { + "user_input": "Refund order O-2 because duplicate", + "expected_tools": [ + "create_refund" + ], + "actual_tools": [ + "create_refund" + ], + "expected_final_response": "{\"status\":\"submitted\",\"order_id\":\"O-2\"}", + "actual_final_response": "{\"status\":\"submitted\",\"order_id\":\"O-2\"}" + } + }, + { + "case_id": "train_json", + "passed": true, + "metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "execution_failure": null, + "failure_reason": null, + "trace_summary": { + "user_input": "Summarize refund O-3", + "expected_tools": [], + "actual_tools": [], + "expected_final_response": "{\"status\":\"pending\",\"order_id\":\"O-3\"}", + "actual_final_response": "{\"status\":\"pending\",\"order_id\":\"O-3\"}" + } + } + ] + }, + "validation": { + "score": 0.6666666666666666, + "cases": { + "val_refund": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "val_shipping": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "val_billing": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + } + }, + "case_results": [ + { + "case_id": "val_refund", + "passed": true, + "metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "execution_failure": null, + "failure_reason": null, + "trace_summary": { + "user_input": "Refund order O-9 because damaged", + "expected_tools": [ + "create_refund" + ], + "actual_tools": [ + "create_refund" + ], + "expected_final_response": "{\"status\":\"submitted\",\"order_id\":\"O-9\"}", + "actual_final_response": "{\"status\":\"submitted\",\"order_id\":\"O-9\"}" + } + }, + { + "case_id": "val_shipping", + "passed": true, + "metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "execution_failure": null, + "failure_reason": null, + "trace_summary": { + "user_input": "Track shipment T-1", + "expected_tools": [ + "track_shipment" + ], + "actual_tools": [ + "track_shipment" + ], + "expected_final_response": "{\"tracking_id\":\"T-1\",\"status\":\"in_transit\"}", + "actual_final_response": "{\"tracking_id\":\"T-1\",\"status\":\"in_transit\"}" + } + }, + { + "case_id": "val_billing", + "passed": false, + "metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "execution_failure": null, + "failure_reason": "failed_metrics:tool_trajectory_avg_score", + "trace_summary": { + "user_input": "Check billing invoice I-9", + "expected_tools": [ + "get_invoice" + ], + "actual_tools": [ + "create_refund" + ], + "expected_final_response": "{\"invoice_id\":\"I-9\",\"status\":\"open\"}", + "actual_final_response": "{\"invoice_id\":\"I-9\",\"status\":\"open\"}" + } + } + ] + } + }, + "delta": { + "train_score": 1.0, + "validation_score": 0.0, + "case_deltas": [ + { + "case_id": "val_refund", + "change": "new_pass", + "before": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "execution_failure": null + }, + { + "case_id": "val_shipping", + "change": "unchanged", + "before": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "after": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "execution_failure": null + }, + { + "case_id": "val_billing", + "change": "new_fail", + "before": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "after": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "execution_failure": null + } + ] + }, + "candidate_validation": { + "train_score": 1.0, + "validation_score": 0.6666666666666666, + "case_deltas": [ + { + "case_id": "val_refund", + "change": "new_pass", + "before": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "execution_failure": null + }, + { + "case_id": "val_shipping", + "change": "unchanged", + "before": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "after": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "execution_failure": null + }, + { + "case_id": "val_billing", + "change": "new_fail", + "before": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "after": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "execution_failure": null + } + ] + }, + "regression_diagnosis": { + "items": [ + { + "case_id": "val_billing", + "change": "new_fail", + "regression_category": "tool_selection_error", + "candidate_related_surface": "router_prompt", + "counterfactual_evidence": { + "intervention": "replace_tool_name", + "result": "pass" + } + } + ] + }, + "gate": { + "accepted": false, + "checks": [ + { + "name": "validation_delta", + "passed": false, + "observed": 0.0, + "threshold": 0.01, + "reason": "VALIDATION_DELTA" + }, + { + "name": "trusted_validation_non_regression", + "passed": true, + "observed": 0.0, + "threshold": 0.0, + "reason": "passed" + }, + { + "name": "no_new_hard_fail", + "passed": false, + "observed": [ + "val_billing" + ], + "threshold": [], + "reason": "NEW_HARD_FAIL" + }, + { + "name": "protected_cases", + "passed": false, + "observed": [ + "val_billing" + ], + "threshold": [], + "reason": "PROTECTED_REGRESSION" + }, + { + "name": "no_train_only_improvement", + "passed": false, + "observed": { + "train_delta": 1.0, + "validation_delta": 0.0 + }, + "threshold": "validation_delta > 0 when train_delta > 0", + "reason": "TRAIN_ONLY_IMPROVEMENT" + }, + { + "name": "no_severity_escalation", + "passed": false, + "observed": [ + { + "case_id": "val_billing", + "before": "none", + "after": "tool_selection_error" + } + ], + "threshold": [], + "reason": "SEVERITY_ESCALATION" + }, + { + "name": "cost_budget", + "passed": true, + "observed": 0.0, + "threshold": 1.0, + "reason": "passed" + }, + { + "name": "latency_budget", + "passed": true, + "observed": 0.29256009999880916, + "threshold": 180, + "reason": "passed" + }, + { + "name": "evidence_sufficient", + "passed": true, + "observed": true, + "threshold": true, + "reason": "passed" + } + ], + "reason_codes": [ + "VALIDATION_DELTA", + "NEW_HARD_FAIL", + "PROTECTED_REGRESSION", + "TRAIN_ONLY_IMPROVEMENT", + "SEVERITY_ESCALATION" + ] + }, + "audit": { + "seed": 42, + "mode": "trace", + "duration_seconds": 0.29256009999880916, + "cost": { + "total": 0.0 + }, + "input_hashes": { + "train": "92287f89b07fcd9e919aec7d8df23c58c7a8328dcee9a4e19f25801daeffce39", + "validation": "f696099e7586ab7b75cd27079e7f8bed5a2c06a8f7d19a8846b97155f89cd1da" + }, + "prompt_hashes": { + "router_prompt": "475f0c79b3f235019e9256897cca903d36478c6e9bbac7f0c2d99cb9b38f97d8", + "skill_prompt": "033f17da15eb358e6cb3674d978d62e82c925d9191359ab86ed00cee6f2c10c8", + "system_prompt": "4e1c6a4ef268a492679ad6888230217d874733817e27137644f3364038642be9" + }, + "write_back": { + "requested": false, + "applied": false, + "baseline_hashes": { + "router_prompt": "475f0c79b3f235019e9256897cca903d36478c6e9bbac7f0c2d99cb9b38f97d8", + "skill_prompt": "033f17da15eb358e6cb3674d978d62e82c925d9191359ab86ed00cee6f2c10c8", + "system_prompt": "4e1c6a4ef268a492679ad6888230217d874733817e27137644f3364038642be9" + }, + "final_hashes": { + "router_prompt": "475f0c79b3f235019e9256897cca903d36478c6e9bbac7f0c2d99cb9b38f97d8", + "skill_prompt": "033f17da15eb358e6cb3674d978d62e82c925d9191359ab86ed00cee6f2c10c8", + "system_prompt": "4e1c6a4ef268a492679ad6888230217d874733817e27137644f3364038642be9" + } + }, + "reproduction_command": "python examples/optimization/eval_optimize_loop/run_pipeline.py --mode trace --candidate-profile overfit" + }, + "known_limitations": [ + "A local trace edit can be structurally valid but semantically incoherent with an original tool response.", + "LLM-judge variance auditing requires repeated real-judge samples and is not exercised by this deterministic example.", + "Real optimizer wiring is mock-verified; production execution requires credentials, call_agent, and trace capture." + ] +} \ No newline at end of file diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md new file mode 100644 index 00000000..764adff0 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md @@ -0,0 +1,35 @@ +# Trust-Aware Counterfactual Optimization Report + +Gate: **REJECTED** + +## Baseline failures + +- `train_route`: tool_selection_error (actionable=True) +- `train_args`: tool_parameter_error (actionable=True) +- `train_json`: final_response_mismatch (actionable=True) + +## Candidate changes + +- `val_refund`: new_pass +- `val_shipping`: unchanged +- `val_billing`: new_fail + +## Counterfactual regression diagnosis + +- `val_billing`: tool_selection_error via replace_tool_name + +## Decision + +- Reasons: VALIDATION_DELTA, NEW_HARD_FAIL, PROTECTED_REGRESSION, TRAIN_ONLY_IMPROVEMENT, SEVERITY_ESCALATION +- Source write-back: False + +## Optimization + +- Target prompts: router_prompt, skill_prompt, system_prompt +- Candidate profile: overfit + +## Known limitations + +- A local trace edit can be structurally valid but semantically incoherent with an original tool response. +- LLM-judge variance auditing requires repeated real-judge samples and is not exercised by this deterministic example. +- Real optimizer wiring is mock-verified; production execution requires credentials, call_agent, and trace capture. diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimizer_failure_digest.json b/examples/optimization/eval_optimize_loop/sample_output/optimizer_failure_digest.json new file mode 100644 index 00000000..f9e88ae4 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/sample_output/optimizer_failure_digest.json @@ -0,0 +1,187 @@ +{ + "actionable_failures": [ + { + "case_id": "train_route", + "failure_domain": "agent_behavior_failure", + "primary_category": "tool_selection_error", + "secondary_categories": [], + "prompt_actionable": true, + "confidence": 0.95, + "evidence": [ + { + "intervention": "replace_final_response", + "valid": true, + "status": "constructed", + "changed_fail_to_pass": false, + "repaired_metrics": [], + "unchanged_metrics": [ + "final_response_avg_score", + "tool_trajectory_avg_score" + ], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + }, + { + "intervention": "replace_tool_name", + "valid": true, + "status": "constructed", + "changed_fail_to_pass": true, + "repaired_metrics": [ + "tool_trajectory_avg_score" + ], + "unchanged_metrics": [ + "final_response_avg_score" + ], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + } + ], + "recommended_target_prompts": [ + "router_prompt" + ], + "evaluations_used": 2 + }, + { + "case_id": "train_args", + "failure_domain": "agent_behavior_failure", + "primary_category": "tool_parameter_error", + "secondary_categories": [], + "prompt_actionable": true, + "confidence": 0.95, + "evidence": [ + { + "intervention": "replace_final_response", + "valid": true, + "status": "constructed", + "changed_fail_to_pass": false, + "repaired_metrics": [], + "unchanged_metrics": [ + "final_response_avg_score", + "tool_trajectory_avg_score" + ], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + }, + { + "intervention": "replace_tool_name", + "valid": true, + "status": "constructed", + "changed_fail_to_pass": false, + "repaired_metrics": [], + "unchanged_metrics": [ + "final_response_avg_score", + "tool_trajectory_avg_score" + ], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + }, + { + "intervention": "replace_tool_arguments", + "valid": true, + "status": "constructed", + "changed_fail_to_pass": true, + "repaired_metrics": [ + "tool_trajectory_avg_score" + ], + "unchanged_metrics": [ + "final_response_avg_score" + ], + "before_metrics": { + "tool_trajectory_avg_score": 0.0, + "final_response_avg_score": 1.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + } + ], + "recommended_target_prompts": [ + "skill_prompt" + ], + "evaluations_used": 3 + }, + { + "case_id": "train_json", + "failure_domain": "agent_behavior_failure", + "primary_category": "final_response_mismatch", + "secondary_categories": [], + "prompt_actionable": true, + "confidence": 0.95, + "evidence": [ + { + "intervention": "replace_final_response", + "valid": true, + "status": "constructed", + "changed_fail_to_pass": true, + "repaired_metrics": [ + "final_response_avg_score" + ], + "unchanged_metrics": [ + "tool_trajectory_avg_score" + ], + "before_metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 0.0 + }, + "after_metrics": { + "tool_trajectory_avg_score": 1.0, + "final_response_avg_score": 1.0 + }, + "structurally_valid": true, + "semantically_coherent": true, + "coherence_warnings": [] + } + ], + "recommended_target_prompts": [ + "system_prompt" + ], + "evaluations_used": 1 + } + ], + "excluded_failures": [], + "category_statistics": { + "tool_selection_error": 1, + "tool_parameter_error": 1, + "final_response_mismatch": 1 + } +} \ No newline at end of file diff --git a/examples/optimization/eval_optimize_loop/train.evalset.json b/examples/optimization/eval_optimize_loop/train.evalset.json new file mode 100644 index 00000000..7d798888 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/train.evalset.json @@ -0,0 +1,7 @@ +{ + "eval_set_id":"counterfactual_train","eval_cases":[ + {"eval_id":"train_route","eval_mode":"trace","conversation":[{"user_content":{"role":"user","parts":[{"text":"Refund order O-1 because damaged"}]},"final_response":{"role":"model","parts":[{"text":"{\"status\":\"submitted\",\"order_id\":\"O-1\"}"}]},"intermediate_data":{"tool_uses":[{"name":"create_refund","args":{"order_id":"O-1","reason":"damaged"}}]}}],"actual_conversation":[{"user_content":{"role":"user","parts":[{"text":"Refund order O-1 because damaged"}]},"final_response":{"role":"model","parts":[{"text":"{\"status\":\"submitted\",\"order_id\":\"O-1\"}"}]},"intermediate_data":{"tool_uses":[{"name":"get_invoice","args":{"order_id":"O-1","reason":"damaged"}}]}}]}, + {"eval_id":"train_args","eval_mode":"trace","conversation":[{"user_content":{"role":"user","parts":[{"text":"Refund order O-2 because duplicate"}]},"final_response":{"role":"model","parts":[{"text":"{\"status\":\"submitted\",\"order_id\":\"O-2\"}"}]},"intermediate_data":{"tool_uses":[{"name":"create_refund","args":{"order_id":"O-2","reason":"duplicate"}}]}}],"actual_conversation":[{"user_content":{"role":"user","parts":[{"text":"Refund order O-2 because duplicate"}]},"final_response":{"role":"model","parts":[{"text":"{\"status\":\"submitted\",\"order_id\":\"O-2\"}"}]},"intermediate_data":{"tool_uses":[{"name":"create_refund","args":{"order_id":"O-2"}}]}}]}, + {"eval_id":"train_json","eval_mode":"trace","conversation":[{"user_content":{"role":"user","parts":[{"text":"Summarize refund O-3"}]},"final_response":{"role":"model","parts":[{"text":"{\"status\":\"pending\",\"order_id\":\"O-3\"}"}]}}],"actual_conversation":[{"user_content":{"role":"user","parts":[{"text":"Summarize refund O-3"}]},"final_response":{"role":"model","parts":[{"text":"{\"order_id\":\"O-3\"}"}]}}]} + ] +} diff --git a/examples/optimization/eval_optimize_loop/val.evalset.json b/examples/optimization/eval_optimize_loop/val.evalset.json new file mode 100644 index 00000000..9a7d69a6 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/val.evalset.json @@ -0,0 +1,7 @@ +{ + "eval_set_id":"counterfactual_val","eval_cases":[ + {"eval_id":"val_refund","eval_mode":"trace","conversation":[{"user_content":{"role":"user","parts":[{"text":"Refund order O-9 because damaged"}]},"final_response":{"role":"model","parts":[{"text":"{\"status\":\"submitted\",\"order_id\":\"O-9\"}"}]},"intermediate_data":{"tool_uses":[{"name":"create_refund","args":{"order_id":"O-9","reason":"damaged"}}]}}],"actual_conversation":[{"user_content":{"role":"user","parts":[{"text":"Refund order O-9 because damaged"}]},"final_response":{"role":"model","parts":[{"text":"{\"status\":\"submitted\",\"order_id\":\"O-9\"}"}]},"intermediate_data":{"tool_uses":[{"name":"get_invoice","args":{"order_id":"O-9","reason":"damaged"}}]}}]}, + {"eval_id":"val_shipping","eval_mode":"trace","conversation":[{"user_content":{"role":"user","parts":[{"text":"Track shipment T-1"}]},"final_response":{"role":"model","parts":[{"text":"{\"tracking_id\":\"T-1\",\"status\":\"in_transit\"}"}]},"intermediate_data":{"tool_uses":[{"name":"track_shipment","args":{"tracking_id":"T-1"}}]}}],"actual_conversation":[{"user_content":{"role":"user","parts":[{"text":"Track shipment T-1"}]},"final_response":{"role":"model","parts":[{"text":"{\"tracking_id\":\"T-1\",\"status\":\"in_transit\"}"}]},"intermediate_data":{"tool_uses":[{"name":"track_shipment","args":{"tracking_id":"T-1"}}]}}]}, + {"eval_id":"val_billing","eval_mode":"trace","conversation":[{"user_content":{"role":"user","parts":[{"text":"Check billing invoice I-9"}]},"final_response":{"role":"model","parts":[{"text":"{\"invoice_id\":\"I-9\",\"status\":\"open\"}"}]},"intermediate_data":{"tool_uses":[{"name":"get_invoice","args":{"invoice_id":"I-9"}}]}}],"actual_conversation":[{"user_content":{"role":"user","parts":[{"text":"Check billing invoice I-9"}]},"final_response":{"role":"model","parts":[{"text":"{\"invoice_id\":\"I-9\",\"status\":\"open\"}"}]},"intermediate_data":{"tool_uses":[{"name":"get_invoice","args":{"invoice_id":"I-9"}}]}}]} + ] +} diff --git a/tests/evaluation/test_counterfactual_trace_probe.py b/tests/evaluation/test_counterfactual_trace_probe.py new file mode 100644 index 00000000..e09bddd9 --- /dev/null +++ b/tests/evaluation/test_counterfactual_trace_probe.py @@ -0,0 +1,150 @@ +# Tencent is pleased to support the open source community by making +# tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Behavior tests for the counterfactual trace feasibility probe.""" + +from __future__ import annotations + +from copy import deepcopy + +import pytest + +from trpc_agent_sdk.types import FunctionResponse + +from examples.optimization.eval_optimize_loop.pipeline.interventions import ( + InterventionKind, + build_counterfactual, +) +from examples.optimization.eval_optimize_loop.pipeline.probe import ( + build_probe_cases, + evaluate_trace_cases, + run_counterfactual_probe, +) + + +@pytest.mark.asyncio +async def test_official_metrics_react_to_counterfactual_trace_edits(tmp_path): + """Final, tool-name, and compound edits must produce distinct metric deltas.""" + cases = build_probe_cases() + baseline = await evaluate_trace_cases(cases, tmp_path) + + final_case = build_counterfactual(cases[0], InterventionKind.REPLACE_FINAL_RESPONSE) + name_case = build_counterfactual(cases[1], InterventionKind.REPLACE_TOOL_NAME) + compound_name = build_counterfactual(cases[2], InterventionKind.REPLACE_TOOL_NAME) + compound_args = build_counterfactual(cases[2], InterventionKind.REPLACE_TOOL_ARGUMENTS) + compound_both = build_counterfactual(cases[2], InterventionKind.REPLACE_TOOL_NAME_AND_ARGUMENTS) + + assert all(result.valid for result in (final_case, name_case, compound_name, compound_args, compound_both)) + + changed = await evaluate_trace_cases( + [ + final_case.eval_case, + name_case.eval_case, + compound_name.eval_case, + compound_args.eval_case, + compound_both.eval_case, + ], + tmp_path, + ) + + assert baseline["probe_final_response"]["final_response_avg_score"] == 0.0 + assert changed[final_case.eval_case.eval_id]["final_response_avg_score"] == 1.0 + + assert baseline["probe_tool_name"]["tool_trajectory_avg_score"] == 0.0 + assert changed[name_case.eval_case.eval_id]["tool_trajectory_avg_score"] == 1.0 + + assert changed[compound_name.eval_case.eval_id]["tool_trajectory_avg_score"] == 0.0 + assert changed[compound_args.eval_case.eval_id]["tool_trajectory_avg_score"] == 0.0 + assert changed[compound_both.eval_case.eval_id]["tool_trajectory_avg_score"] == 1.0 + + +def test_interventions_are_deep_copied_and_report_invalid_tool_shapes(): + """An intervention must not mutate source traces and must explain invalid shapes.""" + case = build_probe_cases()[2] + snapshot = deepcopy(case.model_dump(mode="json", by_alias=True)) + + result = build_counterfactual(case, InterventionKind.REPLACE_TOOL_ARGUMENTS) + + assert result.valid is True + assert case.model_dump(mode="json", by_alias=True) == snapshot + + missing_expected = case.model_copy(deep=True) + missing_expected.conversation[0].intermediate_data.tool_uses = [] + missing = build_counterfactual(missing_expected, InterventionKind.REPLACE_TOOL_ARGUMENTS) + assert missing.valid is False + assert missing.status == "missing_expected_tool" + + mismatched = case.model_copy(deep=True) + mismatched.actual_conversation[0].intermediate_data.tool_uses.append( + mismatched.actual_conversation[0].intermediate_data.tool_uses[0].model_copy() + ) + mismatch = build_counterfactual(mismatched, InterventionKind.REPLACE_TOOL_NAME) + assert mismatch.valid is False + assert mismatch.status == "tool_call_count_mismatch" + + invalid_arguments = case.model_copy(deep=True) + invalid_arguments.actual_conversation[0].intermediate_data.tool_uses[0].args = "bad" + bad_args = build_counterfactual(invalid_arguments, InterventionKind.REPLACE_TOOL_ARGUMENTS) + assert bad_args.valid is False + assert bad_args.status == "tool_arguments_not_dict" + + +@pytest.mark.asyncio +async def test_probe_report_records_metric_repairs_and_feasibility(tmp_path): + """The persisted probe must contain auditable before/after evidence.""" + report = await run_counterfactual_probe(tmp_path) + + assert report["feasibility"]["supported"] is True + assert report["source_trace_unchanged"] is True + assert (tmp_path / "counterfactual_probe.json").is_file() + assert (tmp_path / "counterfactual_probe.md").is_file() + + by_id = {item["case_id"]: item for item in report["cases"]} + case_a = {item["intervention"]: item for item in by_id["probe_final_response"]["interventions"]} + case_b = {item["intervention"]: item for item in by_id["probe_tool_name"]["interventions"]} + case_c = {item["intervention"]: item for item in by_id["probe_compound_tool"]["interventions"]} + + assert case_a["replace_final_response"]["changed_fail_to_pass"] is True + assert case_a["replace_final_response"]["repaired_metrics"] == ["final_response_avg_score"] + assert case_b["replace_tool_name"]["repaired_metrics"] == ["tool_trajectory_avg_score"] + assert case_c["replace_tool_name"]["changed_fail_to_pass"] is False + assert case_c["replace_tool_arguments"]["changed_fail_to_pass"] is False + assert case_c["replace_tool_name_and_arguments"]["changed_fail_to_pass"] is True + + +def test_tool_edit_with_original_tool_response_is_marked_incoherent(): + case = build_probe_cases()[1] + case.actual_conversation[0].intermediate_data.tool_responses = [ + FunctionResponse.model_validate({"name": "get_invoice", "response": {"invoice_id": "R-102"}}) + ] + + result = build_counterfactual(case, InterventionKind.REPLACE_TOOL_NAME) + + assert result.structurally_valid is True + assert result.semantically_coherent is False + assert "tool_response_matches_original_call" in result.coherence_warnings + + +@pytest.mark.asyncio +async def test_probe_diagnosis_is_case_id_and_order_invariant(tmp_path): + original = await run_counterfactual_probe(tmp_path / "original") + renamed = build_probe_cases() + for index, case in enumerate(renamed): + case.eval_id = f"renamed_{index}" + changed = await run_counterfactual_probe(tmp_path / "changed", list(reversed(renamed))) + + def signature(report): + return sorted( + ( + item["diagnosis"]["primary_category"], + item["diagnosis"]["compound_failure"], + ) + for item in report["cases"] + ) + + assert signature(original) == signature(changed) + assert original["feasibility"]["supported"] is True + assert changed["feasibility"]["supported"] is True diff --git a/tests/evaluation/test_eval_optimize_loop.py b/tests/evaluation/test_eval_optimize_loop.py new file mode 100644 index 00000000..77f3d2af --- /dev/null +++ b/tests/evaluation/test_eval_optimize_loop.py @@ -0,0 +1,480 @@ +"""Behavior tests for the trust-aware eval/optimization loop.""" + +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from examples.optimization.eval_optimize_loop.pipeline.diagnosis import ( + InfrastructureFailure, + attribute_from_evidence, + build_failure_digest, + classify_non_agent_failure, + select_target_prompts, +) +from examples.optimization.eval_optimize_loop.pipeline.gate import evaluate_gate +from examples.optimization.eval_optimize_loop.pipeline.input_audit import audit_eval_cases +from examples.optimization.eval_optimize_loop.pipeline.models import CounterfactualEvidence +from examples.optimization.eval_optimize_loop.fake.model import generate_trace +from examples.optimization.eval_optimize_loop.pipeline.optimizer import ( + apply_if_accepted, + run_real_optimizer, +) +from examples.optimization.eval_optimize_loop.pipeline.pipeline import run_pipeline +from examples.optimization.eval_optimize_loop.pipeline.pipeline import _evaluate_with_triage +from examples.optimization.eval_optimize_loop.pipeline.probe import build_probe_cases + + +def _evidence(name: str, passed: bool, repaired: list[str]) -> CounterfactualEvidence: + return CounterfactualEvidence( + intervention=name, + valid=True, + status="constructed", + changed_fail_to_pass=passed, + repaired_metrics=repaired, + unchanged_metrics=[], + before_metrics={"tool_trajectory_avg_score": 0.0}, + after_metrics={"tool_trajectory_avg_score": 1.0 if passed else 0.0}, + ) + + +def test_attribution_ignores_reason_case_id_and_evidence_order(): + evidence = [ + _evidence("replace_tool_arguments", False, []), + _evidence("replace_tool_name", True, ["tool_trajectory_avg_score"]), + ] + first = attribute_from_evidence("anything", evidence, failure_reason="tool arguments were wrong") + second = attribute_from_evidence("renamed", list(reversed(evidence)), failure_reason="") + assert first.primary_category == second.primary_category == "tool_selection_error" + assert first.confidence == second.confidence + + +def test_compound_and_parameter_failures_are_distinct(): + parameter = attribute_from_evidence("x", [_evidence("replace_tool_arguments", True, ["tool_trajectory_avg_score"])]) + compound = attribute_from_evidence( + "y", + [ + _evidence("replace_tool_name", False, []), + _evidence("replace_tool_arguments", False, []), + _evidence("replace_tool_name_and_arguments", True, ["tool_trajectory_avg_score"]), + ], + ) + assert parameter.primary_category == "tool_parameter_error" + assert compound.primary_category == "compound_failure" + + +def test_attribution_accuracy_against_test_only_oracle(): + """The oracle lives only in tests; production receives intervention evidence.""" + cases = { + "final": ( + [_evidence("replace_final_response", True, ["final_response_avg_score"])], + "final_response_mismatch", + ), + "tool": ( + [_evidence("replace_tool_name", True, ["tool_trajectory_avg_score"])], + "tool_selection_error", + ), + "arguments": ( + [_evidence("replace_tool_arguments", True, ["tool_trajectory_avg_score"])], + "tool_parameter_error", + ), + "compound": ( + [ + _evidence("replace_tool_name", False, []), + _evidence("replace_tool_arguments", False, []), + _evidence( + "replace_tool_name_and_arguments", + True, + ["tool_trajectory_avg_score"], + ), + ], + "compound_failure", + ), + "sequence": ( + [ + CounterfactualEvidence( + "replace_tool_name", + False, + "tool_call_count_mismatch", + False, + [], + [], + {"tool_trajectory_avg_score": 0.0}, + {}, + structurally_valid=False, + ) + ], + "tool_sequence_error", + ), + "rubric": ( + [_evidence("replace_final_response", False, [])], + "llm_rubric_not_met", + ), + "knowledge": ( + [_evidence("replace_final_response", False, [])], + "knowledge_recall_insufficient", + ), + "format": ( + [_evidence("normalize_format", False, [])], + "format_violation", + ), + } + cases["rubric"][0][0].before_metrics = {"llm_rubric_response": 0.0} + cases["knowledge"][0][0].before_metrics = {"llm_rubric_knowledge_recall": 0.0} + cases["format"][0][0].before_metrics = {"format_compliance_score": 0.0} + + predictions = { + name: attribute_from_evidence(name, evidence).primary_category for name, (evidence, _) in cases.items() + } + correct = sum(predictions[name] == expected for name, (_, expected) in cases.items()) + accuracy = correct / len(cases) + + assert accuracy >= 0.75, { + name: {"predicted": predictions[name], "expected": expected} + for name, (_, expected) in cases.items() + if predictions[name] != expected + } + + +def test_incoherent_counterfactual_evidence_reduces_confidence(): + coherent = _evidence("replace_tool_name", True, ["tool_trajectory_avg_score"]) + incoherent = _evidence("replace_tool_name", True, ["tool_trajectory_avg_score"]) + incoherent.semantically_coherent = False + incoherent.coherence_warnings = ["tool_response_matches_original_call"] + assert attribute_from_evidence("a", [incoherent]).confidence < attribute_from_evidence("b", [coherent]).confidence + + +def test_missing_reference_is_eval_data_and_timeout_is_infrastructure(): + missing = classify_non_agent_failure("c", reliability="invalid", issues=["missing_reference"]) + timeout = classify_non_agent_failure("d", error=TimeoutError("slow")) + assert missing.failure_domain == "evaluation_data_failure" + assert missing.prompt_actionable is False + assert timeout.failure_domain == "infrastructure_failure" + assert timeout.primary_category == "model_timeout" + assert timeout.prompt_actionable is False + + +def test_evaluator_and_tool_runtime_failures_are_non_actionable(): + evaluator = classify_non_agent_failure("e", error=ValueError("metric exploded")) + runtime = classify_non_agent_failure("i", error=InfrastructureFailure("tool_runtime_error", "tool crashed")) + data = classify_non_agent_failure("d", reliability="invalid", issues=["missing_reference"]) + agent = attribute_from_evidence("a", [_evidence("replace_tool_name", True, ["tool_trajectory_avg_score"])]) + assert evaluator.failure_domain == "evaluator_failure" + assert runtime.failure_domain == "infrastructure_failure" + assert runtime.primary_category == "tool_runtime_error" + assert select_target_prompts([evaluator, runtime]) == [] + digest = build_failure_digest([agent, data, evaluator, runtime]) + assert [x["case_id"] for x in digest["actionable_failures"]] == ["a"] + assert {x["failure_domain"] for x in digest["excluded_failures"]} == { + "evaluator_failure", + "infrastructure_failure", + "evaluation_data_failure", + } + + +def test_invalid_cases_do_not_drive_targets(): + cases = [{"case_id": "bad", "status": "invalid", "issues": ["missing_reference"]}] + audit = audit_eval_cases(cases, overlapping_inputs=set()) + assert audit[0]["status"] == "invalid" + assert select_target_prompts([]) == [] + + +def test_counterfactual_budget_limits_evidence(): + evidence = [_evidence("replace_tool_name", False, []), _evidence("replace_tool_arguments", False, [])] + result = attribute_from_evidence("x", evidence[:1], evaluations_used=1, budget=1) + assert result.evaluations_used == 1 + assert "counterfactual_budget_exhausted" in result.evidence + + +def _gate_input(**overrides): + value = { + "baseline_train": 0.3, + "candidate_train": 0.8, + "baseline_validation": 0.5, + "candidate_validation": 0.7, + "trusted_baseline": 0.5, + "trusted_candidate": 0.7, + "new_hard_fails": [], + "protected_regressions": [], + "severity_escalations": [], + "cost": 0.1, + "duration_seconds": 1.0, + "evidence_sufficient": True, + } + value.update(overrides) + return value + + +def test_gate_rejects_train_only_improvement_and_new_hard_fail(): + no_val = evaluate_gate( + _gate_input(candidate_validation=0.5), {"min_validation_delta": 0.01, "max_cost": 1, "max_latency_seconds": 10} + ) + hard = evaluate_gate( + _gate_input(new_hard_fails=["v3"]), {"min_validation_delta": 0.01, "max_cost": 1, "max_latency_seconds": 10} + ) + assert no_val["accepted"] is False + assert "TRAIN_ONLY_IMPROVEMENT" in no_val["reason_codes"] + assert hard["accepted"] is False + assert "NEW_HARD_FAIL" in hard["reason_codes"] + + +def test_gate_rejects_format_to_tool_selection_severity_escalation(): + result = evaluate_gate( + _gate_input(severity_escalations=[{"before": "format_violation", "after": "tool_selection_error"}]), + {"min_validation_delta": 0.01, "max_cost": 1, "max_latency_seconds": 10}, + ) + assert result["accepted"] is False + assert "SEVERITY_ESCALATION" in result["reason_codes"] + + +def test_gate_accepts_safe_improvement_and_is_monotonic_when_tightened(): + config = {"min_validation_delta": 0.01, "max_cost": 1, "max_latency_seconds": 10} + accepted = evaluate_gate(_gate_input(), config) + ineffective = evaluate_gate(_gate_input(candidate_train=0.3, candidate_validation=0.5), config) + tightened = evaluate_gate(_gate_input(), {**config, "min_validation_delta": 0.3}) + protected = evaluate_gate(_gate_input(protected_regressions=["critical"]), config) + expensive = evaluate_gate(_gate_input(cost=2.0), config) + slow = evaluate_gate(_gate_input(duration_seconds=11.0), config) + assert accepted["accepted"] is True + assert ineffective["accepted"] is False + assert tightened["accepted"] is False + assert protected["accepted"] is False + assert expensive["accepted"] is False + assert slow["accepted"] is False + assert "LATENCY_BUDGET" in slow["reason_codes"] + + +def test_fake_model_handles_unknown_semantics_without_case_id_branching(): + case = build_probe_cases()[0] + case.eval_id = "arbitrary-hidden-id" + case.conversation[0].user_content.parts[0].text = "Please explain the account policy." + + generated = generate_trace( + case, + {"router_prompt": "", "skill_prompt": "", "system_prompt": ""}, + ) + + assert generated.eval_id == "arbitrary-hidden-id" + assert generated.actual_conversation + assert generated.actual_conversation[0].final_response is not None + + +@pytest.mark.asyncio +async def test_apply_only_writes_when_accepted_and_explicit(tmp_path): + prompt = tmp_path / "system.md" + prompt.write_text("baseline", encoding="utf-8") + rejected = await apply_if_accepted( + {"accepted": False}, False, {"system_prompt": "candidate"}, {"system_prompt": prompt} + ) + implicit = await apply_if_accepted( + {"accepted": True}, False, {"system_prompt": "candidate"}, {"system_prompt": prompt} + ) + assert rejected["applied"] is False + assert implicit["applied"] is False + assert prompt.read_text(encoding="utf-8") == "baseline" + + applied = await apply_if_accepted( + {"accepted": True}, True, {"system_prompt": "candidate"}, {"system_prompt": prompt} + ) + assert applied["applied"] is True + assert applied["baseline_hashes"] != applied["final_hashes"] + assert prompt.read_text(encoding="utf-8") == "candidate" + + +@pytest.mark.asyncio +async def test_fake_and_trace_pipeline_have_same_schema_without_api_key(tmp_path, monkeypatch): + for key in list(__import__("os").environ): + if "API_KEY" in key: + monkeypatch.delenv(key, raising=False) + base = Path(__file__).parents[2] / "examples" / "optimization" / "eval_optimize_loop" + fake = await run_pipeline(base, "fake", tmp_path / "fake") + trace = await run_pipeline(base, "trace", tmp_path / "trace") + assert fake.keys() == trace.keys() + assert fake["schema_version"] == "1.0" + assert fake["gate"]["accepted"] is False + assert [x["change"] for x in fake["candidate_validation"]["case_deltas"]] == ["new_pass", "unchanged", "new_fail"] + assert fake["regression_diagnosis"]["items"][0]["counterfactual_evidence"]["intervention"] == "replace_tool_name" + assert fake["audit"]["write_back"]["applied"] is False + + +@pytest.mark.asyncio +async def test_real_evaluations_drive_accept_overfit_and_ineffective_decisions(tmp_path): + base = Path(__file__).parents[2] / "examples" / "optimization" / "eval_optimize_loop" + accepted = await run_pipeline(base, "fake", tmp_path / "accepted", candidate_profile="accepted") + overfit = await run_pipeline(base, "fake", tmp_path / "overfit", candidate_profile="overfit") + ineffective = await run_pipeline(base, "fake", tmp_path / "ineffective", candidate_profile="ineffective") + assert accepted["gate"]["accepted"] is True + assert all(check["passed"] for check in accepted["gate"]["checks"]) + assert accepted["candidate_validation"]["validation_score"] > accepted["baseline"]["validation"]["score"] + assert overfit["gate"]["accepted"] is False + assert any(x["change"] == "new_fail" for x in overfit["candidate_validation"]["case_deltas"]) + assert ineffective["gate"]["accepted"] is False + assert ineffective["candidate_validation"]["validation_score"] == ineffective["baseline"]["validation"]["score"] + + +@pytest.mark.asyncio +async def test_real_optimizer_receives_filtered_train_and_never_updates_source(tmp_path, monkeypatch): + base = Path(__file__).parents[2] / "examples" / "optimization" / "eval_optimize_loop" + captured = {} + + async def spy(**kwargs): + captured.update(kwargs) + filtered = json.loads(Path(kwargs["train_dataset_path"]).read_text(encoding="utf-8")) + captured["filtered_ids"] = [x["evalId"] for x in filtered["evalCases"]] + validation = json.loads(Path(kwargs["validation_dataset_path"]).read_text(encoding="utf-8")) + validation_cases = validation.get("evalCases", validation.get("eval_cases", [])) + captured["validation_ids"] = [x.get("evalId", x.get("eval_id")) for x in validation_cases] + captured["target_names"] = tuple(kwargs["target_prompt"].names()) + return SimpleNamespace( + algorithm="gepa_reflective", + total_rounds=2, + baseline_pass_rate=0.25, + best_pass_rate=0.75, + pass_rate_improvement=0.5, + baseline_metric_breakdown={"tool_trajectory_avg_score": 0.25}, + best_metric_breakdown={"tool_trajectory_avg_score": 0.75}, + metric_thresholds={"tool_trajectory_avg_score": 1.0}, + baseline_prompts={"router_prompt": "baseline"}, + best_prompts={"router_prompt": "candidate"}, + total_llm_cost=0.2, + total_token_usage={"total": 42}, + duration_seconds=1.5, + status="SUCCEEDED", + rounds=[{"round": 1}, {"round": 2}], + ) + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.pipeline.optimizer.AgentOptimizer.optimize", + spy, + ) + result = await run_real_optimizer( + config_path=base / "optimizer.json", + call_agent=lambda _: "ok", + prompt_paths={"router_prompt": base / "prompts" / "router.md"}, + train_path=base / "train.evalset.json", + actionable_case_ids={"train_route"}, + validation_path=base / "val.evalset.json", + output_dir=tmp_path, + ) + assert captured["filtered_ids"] == ["train_route"] + assert captured["validation_ids"] == ["val_refund", "val_shipping", "val_billing"] + assert captured["target_names"] == ("router_prompt",) + assert captured["update_source"] is False + assert result == { + "algorithm": "gepa_reflective", + "total_rounds": 2, + "rounds": [{"round": 1}, {"round": 2}], + "best_prompts": {"router_prompt": "candidate"}, + "baseline_prompts": {"router_prompt": "baseline"}, + "baseline_pass_rate": 0.25, + "best_pass_rate": 0.75, + "pass_rate_improvement": 0.5, + "baseline_metric_breakdown": {"tool_trajectory_avg_score": 0.25}, + "best_metric_breakdown": {"tool_trajectory_avg_score": 0.75}, + "metric_thresholds": {"tool_trajectory_avg_score": 1.0}, + "cost": 0.2, + "tokens": {"total": 42}, + "duration_seconds": 1.5, + "seed": 42, + "status": "SUCCEEDED", + } + + +@pytest.mark.asyncio +async def test_real_optimizer_restores_prompt_if_optimizer_mutates_then_fails(tmp_path, monkeypatch): + base = Path(__file__).parents[2] / "examples" / "optimization" / "eval_optimize_loop" + prompt = tmp_path / "router.md" + prompt.write_text("baseline", encoding="utf-8") + + async def broken_optimizer(**kwargs): + await kwargs["target_prompt"].write_all({"router_prompt": "mutated"}) + raise RuntimeError("optimizer failed") + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.pipeline.optimizer.AgentOptimizer.optimize", + broken_optimizer, + ) + + with pytest.raises(RuntimeError, match="optimizer failed"): + await run_real_optimizer( + config_path=base / "optimizer.json", + call_agent=lambda _: "ok", + prompt_paths={"router_prompt": prompt}, + train_path=base / "train.evalset.json", + actionable_case_ids={"train_route"}, + validation_path=base / "val.evalset.json", + output_dir=tmp_path / "optimizer-output", + ) + + assert prompt.read_text(encoding="utf-8") == "baseline" + + +@pytest.mark.asyncio +async def test_pipeline_rejects_unsupported_real_mode(tmp_path): + base = Path(__file__).parents[2] / "examples" / "optimization" / "eval_optimize_loop" + + with pytest.raises(ValueError, match="fake or trace"): + await run_pipeline(base, "real", tmp_path) + + +@pytest.mark.asyncio +async def test_report_contains_trace_status_evidence_and_limitations(tmp_path): + base = Path(__file__).parents[2] / "examples" / "optimization" / "eval_optimize_loop" + report = await run_pipeline(base, "trace", tmp_path) + case = report["baseline"]["train"]["case_results"][0] + assert {"case_id", "passed", "metrics", "failure_reason", "trace_summary"} <= case.keys() + assert report["candidate"]["validation"]["case_results"] + assert report["delta"]["case_deltas"] == report["candidate_validation"]["case_deltas"] + evidence = report["failure_attribution"]["items"][0]["evidence"][0] + assert {"structurally_valid", "semantically_coherent", "coherence_warnings"} <= evidence.keys() + assert report["known_limitations"] + + +@pytest.mark.asyncio +async def test_fake_round_audit_contains_candidate_prompts_and_evaluation(tmp_path): + base = Path(__file__).parents[2] / "examples" / "optimization" / "eval_optimize_loop" + + report = await run_pipeline(base, "fake", tmp_path, candidate_profile="accepted") + + round_record = report["optimization"]["rounds"][0] + assert round_record["candidate_prompts"] == report["optimization"]["best_prompts"] + assert round_record["validation_score"] == report["candidate_validation"]["validation_score"] + assert set(round_record["metric_breakdown"]) == { + "final_response_avg_score", + "tool_trajectory_avg_score", + } + assert round_record["accepted"] is True + + +def test_committed_sample_output_has_current_report_contract(): + sample = Path(__file__).parents[2] / "examples" / "optimization" / "eval_optimize_loop" / "sample_output" + report = json.loads((sample / "optimization_report.json").read_text(encoding="utf-8")) + assert report["schema_version"] == "1.0" + assert report["baseline"]["train"]["case_results"] + assert report["known_limitations"] + + +@pytest.mark.asyncio +async def test_pipeline_evaluation_triages_timeout_without_dropping_case(tmp_path, monkeypatch): + cases = build_probe_cases()[:2] + calls = 0 + + async def evaluator(case_batch, workspace): + nonlocal calls + calls += 1 + if calls == 1: + raise TimeoutError("backend timed out") + case = case_batch[0] + return {case.eval_id: {"tool_trajectory_avg_score": 1.0}} + + monkeypatch.setattr( + "examples.optimization.eval_optimize_loop.pipeline.pipeline.evaluate_trace_cases", + evaluator, + ) + metrics, failures = await _evaluate_with_triage(cases, tmp_path) + assert cases[0].eval_id in failures + assert failures[cases[0].eval_id].failure_domain == "infrastructure_failure" + assert cases[1].eval_id in metrics From 19437b3ad2a2905d0a401baa54d4e0ed9361b81c Mon Sep 17 00:00:00 2001 From: Leoluis0705 Date: Mon, 13 Jul 2026 00:55:34 +0800 Subject: [PATCH 2/3] docs(examples): distinguish counterfactual trace loop --- .../{eval_optimize_loop => counterfactual_trace_loop}/DESIGN.md | 0 .../{eval_optimize_loop => counterfactual_trace_loop}/README.md | 0 .../{eval_optimize_loop => counterfactual_trace_loop}/__init__.py | 0 .../fake/__init__.py | 0 .../fake/model.py | 0 .../fake/optimizer.py | 0 .../{eval_optimize_loop => counterfactual_trace_loop}/gate.json | 0 .../optimizer.json | 0 .../pipeline/__init__.py | 0 .../pipeline/counterfactual.py | 0 .../pipeline/diagnosis.py | 0 .../pipeline/gate.py | 0 .../pipeline/input_audit.py | 0 .../pipeline/interventions.py | 0 .../pipeline/models.py | 0 .../pipeline/optimizer.py | 0 .../pipeline/pipeline.py | 0 .../pipeline/probe.py | 0 .../prompts/router.md | 0 .../prompts/skill.md | 0 .../prompts/system.md | 0 .../prototype_output/counterfactual_probe.json | 0 .../prototype_output/counterfactual_probe.md | 0 .../run_counterfactual_probe.py | 0 .../run_pipeline.py | 0 .../sample_output/optimization_report.json | 0 .../sample_output/optimization_report.md | 0 .../sample_output/optimizer_failure_digest.json | 0 .../train.evalset.json | 0 .../val.evalset.json | 0 30 files changed, 0 insertions(+), 0 deletions(-) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/DESIGN.md (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/README.md (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/__init__.py (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/fake/__init__.py (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/fake/model.py (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/fake/optimizer.py (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/gate.json (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/optimizer.json (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/pipeline/__init__.py (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/pipeline/counterfactual.py (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/pipeline/diagnosis.py (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/pipeline/gate.py (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/pipeline/input_audit.py (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/pipeline/interventions.py (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/pipeline/models.py (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/pipeline/optimizer.py (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/pipeline/pipeline.py (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/pipeline/probe.py (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/prompts/router.md (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/prompts/skill.md (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/prompts/system.md (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/prototype_output/counterfactual_probe.json (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/prototype_output/counterfactual_probe.md (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/run_counterfactual_probe.py (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/run_pipeline.py (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/sample_output/optimization_report.json (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/sample_output/optimization_report.md (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/sample_output/optimizer_failure_digest.json (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/train.evalset.json (100%) rename examples/optimization/{eval_optimize_loop => counterfactual_trace_loop}/val.evalset.json (100%) diff --git a/examples/optimization/eval_optimize_loop/DESIGN.md b/examples/optimization/counterfactual_trace_loop/DESIGN.md similarity index 100% rename from examples/optimization/eval_optimize_loop/DESIGN.md rename to examples/optimization/counterfactual_trace_loop/DESIGN.md diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/counterfactual_trace_loop/README.md similarity index 100% rename from examples/optimization/eval_optimize_loop/README.md rename to examples/optimization/counterfactual_trace_loop/README.md diff --git a/examples/optimization/eval_optimize_loop/__init__.py b/examples/optimization/counterfactual_trace_loop/__init__.py similarity index 100% rename from examples/optimization/eval_optimize_loop/__init__.py rename to examples/optimization/counterfactual_trace_loop/__init__.py diff --git a/examples/optimization/eval_optimize_loop/fake/__init__.py b/examples/optimization/counterfactual_trace_loop/fake/__init__.py similarity index 100% rename from examples/optimization/eval_optimize_loop/fake/__init__.py rename to examples/optimization/counterfactual_trace_loop/fake/__init__.py diff --git a/examples/optimization/eval_optimize_loop/fake/model.py b/examples/optimization/counterfactual_trace_loop/fake/model.py similarity index 100% rename from examples/optimization/eval_optimize_loop/fake/model.py rename to examples/optimization/counterfactual_trace_loop/fake/model.py diff --git a/examples/optimization/eval_optimize_loop/fake/optimizer.py b/examples/optimization/counterfactual_trace_loop/fake/optimizer.py similarity index 100% rename from examples/optimization/eval_optimize_loop/fake/optimizer.py rename to examples/optimization/counterfactual_trace_loop/fake/optimizer.py diff --git a/examples/optimization/eval_optimize_loop/gate.json b/examples/optimization/counterfactual_trace_loop/gate.json similarity index 100% rename from examples/optimization/eval_optimize_loop/gate.json rename to examples/optimization/counterfactual_trace_loop/gate.json diff --git a/examples/optimization/eval_optimize_loop/optimizer.json b/examples/optimization/counterfactual_trace_loop/optimizer.json similarity index 100% rename from examples/optimization/eval_optimize_loop/optimizer.json rename to examples/optimization/counterfactual_trace_loop/optimizer.json diff --git a/examples/optimization/eval_optimize_loop/pipeline/__init__.py b/examples/optimization/counterfactual_trace_loop/pipeline/__init__.py similarity index 100% rename from examples/optimization/eval_optimize_loop/pipeline/__init__.py rename to examples/optimization/counterfactual_trace_loop/pipeline/__init__.py diff --git a/examples/optimization/eval_optimize_loop/pipeline/counterfactual.py b/examples/optimization/counterfactual_trace_loop/pipeline/counterfactual.py similarity index 100% rename from examples/optimization/eval_optimize_loop/pipeline/counterfactual.py rename to examples/optimization/counterfactual_trace_loop/pipeline/counterfactual.py diff --git a/examples/optimization/eval_optimize_loop/pipeline/diagnosis.py b/examples/optimization/counterfactual_trace_loop/pipeline/diagnosis.py similarity index 100% rename from examples/optimization/eval_optimize_loop/pipeline/diagnosis.py rename to examples/optimization/counterfactual_trace_loop/pipeline/diagnosis.py diff --git a/examples/optimization/eval_optimize_loop/pipeline/gate.py b/examples/optimization/counterfactual_trace_loop/pipeline/gate.py similarity index 100% rename from examples/optimization/eval_optimize_loop/pipeline/gate.py rename to examples/optimization/counterfactual_trace_loop/pipeline/gate.py diff --git a/examples/optimization/eval_optimize_loop/pipeline/input_audit.py b/examples/optimization/counterfactual_trace_loop/pipeline/input_audit.py similarity index 100% rename from examples/optimization/eval_optimize_loop/pipeline/input_audit.py rename to examples/optimization/counterfactual_trace_loop/pipeline/input_audit.py diff --git a/examples/optimization/eval_optimize_loop/pipeline/interventions.py b/examples/optimization/counterfactual_trace_loop/pipeline/interventions.py similarity index 100% rename from examples/optimization/eval_optimize_loop/pipeline/interventions.py rename to examples/optimization/counterfactual_trace_loop/pipeline/interventions.py diff --git a/examples/optimization/eval_optimize_loop/pipeline/models.py b/examples/optimization/counterfactual_trace_loop/pipeline/models.py similarity index 100% rename from examples/optimization/eval_optimize_loop/pipeline/models.py rename to examples/optimization/counterfactual_trace_loop/pipeline/models.py diff --git a/examples/optimization/eval_optimize_loop/pipeline/optimizer.py b/examples/optimization/counterfactual_trace_loop/pipeline/optimizer.py similarity index 100% rename from examples/optimization/eval_optimize_loop/pipeline/optimizer.py rename to examples/optimization/counterfactual_trace_loop/pipeline/optimizer.py diff --git a/examples/optimization/eval_optimize_loop/pipeline/pipeline.py b/examples/optimization/counterfactual_trace_loop/pipeline/pipeline.py similarity index 100% rename from examples/optimization/eval_optimize_loop/pipeline/pipeline.py rename to examples/optimization/counterfactual_trace_loop/pipeline/pipeline.py diff --git a/examples/optimization/eval_optimize_loop/pipeline/probe.py b/examples/optimization/counterfactual_trace_loop/pipeline/probe.py similarity index 100% rename from examples/optimization/eval_optimize_loop/pipeline/probe.py rename to examples/optimization/counterfactual_trace_loop/pipeline/probe.py diff --git a/examples/optimization/eval_optimize_loop/prompts/router.md b/examples/optimization/counterfactual_trace_loop/prompts/router.md similarity index 100% rename from examples/optimization/eval_optimize_loop/prompts/router.md rename to examples/optimization/counterfactual_trace_loop/prompts/router.md diff --git a/examples/optimization/eval_optimize_loop/prompts/skill.md b/examples/optimization/counterfactual_trace_loop/prompts/skill.md similarity index 100% rename from examples/optimization/eval_optimize_loop/prompts/skill.md rename to examples/optimization/counterfactual_trace_loop/prompts/skill.md diff --git a/examples/optimization/eval_optimize_loop/prompts/system.md b/examples/optimization/counterfactual_trace_loop/prompts/system.md similarity index 100% rename from examples/optimization/eval_optimize_loop/prompts/system.md rename to examples/optimization/counterfactual_trace_loop/prompts/system.md diff --git a/examples/optimization/eval_optimize_loop/prototype_output/counterfactual_probe.json b/examples/optimization/counterfactual_trace_loop/prototype_output/counterfactual_probe.json similarity index 100% rename from examples/optimization/eval_optimize_loop/prototype_output/counterfactual_probe.json rename to examples/optimization/counterfactual_trace_loop/prototype_output/counterfactual_probe.json diff --git a/examples/optimization/eval_optimize_loop/prototype_output/counterfactual_probe.md b/examples/optimization/counterfactual_trace_loop/prototype_output/counterfactual_probe.md similarity index 100% rename from examples/optimization/eval_optimize_loop/prototype_output/counterfactual_probe.md rename to examples/optimization/counterfactual_trace_loop/prototype_output/counterfactual_probe.md diff --git a/examples/optimization/eval_optimize_loop/run_counterfactual_probe.py b/examples/optimization/counterfactual_trace_loop/run_counterfactual_probe.py similarity index 100% rename from examples/optimization/eval_optimize_loop/run_counterfactual_probe.py rename to examples/optimization/counterfactual_trace_loop/run_counterfactual_probe.py diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/counterfactual_trace_loop/run_pipeline.py similarity index 100% rename from examples/optimization/eval_optimize_loop/run_pipeline.py rename to examples/optimization/counterfactual_trace_loop/run_pipeline.py diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.json b/examples/optimization/counterfactual_trace_loop/sample_output/optimization_report.json similarity index 100% rename from examples/optimization/eval_optimize_loop/sample_output/optimization_report.json rename to examples/optimization/counterfactual_trace_loop/sample_output/optimization_report.json diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimization_report.md b/examples/optimization/counterfactual_trace_loop/sample_output/optimization_report.md similarity index 100% rename from examples/optimization/eval_optimize_loop/sample_output/optimization_report.md rename to examples/optimization/counterfactual_trace_loop/sample_output/optimization_report.md diff --git a/examples/optimization/eval_optimize_loop/sample_output/optimizer_failure_digest.json b/examples/optimization/counterfactual_trace_loop/sample_output/optimizer_failure_digest.json similarity index 100% rename from examples/optimization/eval_optimize_loop/sample_output/optimizer_failure_digest.json rename to examples/optimization/counterfactual_trace_loop/sample_output/optimizer_failure_digest.json diff --git a/examples/optimization/eval_optimize_loop/train.evalset.json b/examples/optimization/counterfactual_trace_loop/train.evalset.json similarity index 100% rename from examples/optimization/eval_optimize_loop/train.evalset.json rename to examples/optimization/counterfactual_trace_loop/train.evalset.json diff --git a/examples/optimization/eval_optimize_loop/val.evalset.json b/examples/optimization/counterfactual_trace_loop/val.evalset.json similarity index 100% rename from examples/optimization/eval_optimize_loop/val.evalset.json rename to examples/optimization/counterfactual_trace_loop/val.evalset.json From 2809e0cf6942b6a87058a7444b68a2a9af2b7ab6 Mon Sep 17 00:00:00 2001 From: Leoluis0705 Date: Mon, 13 Jul 2026 00:55:53 +0800 Subject: [PATCH 3/3] docs(examples): clarify counterfactual attribution boundary --- .../counterfactual_trace_loop/DESIGN.md | 4 ++ .../counterfactual_trace_loop/README.md | 16 +++++--- .../pipeline/pipeline.py | 2 +- .../run_counterfactual_probe.py | 2 +- .../counterfactual_trace_loop/run_pipeline.py | 2 +- .../sample_output/optimization_report.json | 6 +-- .../test_counterfactual_trace_probe.py | 4 +- tests/evaluation/test_eval_optimize_loop.py | 40 +++++++++---------- 8 files changed, 43 insertions(+), 33 deletions(-) diff --git a/examples/optimization/counterfactual_trace_loop/DESIGN.md b/examples/optimization/counterfactual_trace_loop/DESIGN.md index 8cdf5e01..1d051ef2 100644 --- a/examples/optimization/counterfactual_trace_loop/DESIGN.md +++ b/examples/optimization/counterfactual_trace_loop/DESIGN.md @@ -8,6 +8,10 @@ ## 技术附录 +### 与现有方案的边界 + +PR #159 主要依据失败文本和预期/实际轨迹的静态差异归因；PR #161 重点在优化候选回放、bootstrap 与 Pareto 决策。本方案不把静态差异直接视为原因，而是把它转化为局部反事实干预，再由同一个 `AgentEvaluator` 重评。只有可重复观察到 metric 修复的干预才形成归因证据，且该机制同时用于 baseline 失败和 candidate 新增退化。 + - 反事实归因：结论来自真实 metric delta，不依赖 case ID、failure reason 或人工标签。 - Prompt actionability：只有 agent behavior failure 能够选择优化表面。 - Gate：关键检查必须全部通过，证据不足时以 `NEEDS_REVIEW` 拒绝。 diff --git a/examples/optimization/counterfactual_trace_loop/README.md b/examples/optimization/counterfactual_trace_loop/README.md index 8a72be74..e8b84d2b 100644 --- a/examples/optimization/counterfactual_trace_loop/README.md +++ b/examples/optimization/counterfactual_trace_loop/README.md @@ -2,14 +2,20 @@ This example closes evaluation and prompt optimization with evidence from trace interventions. Unlike loops that classify a failure from reason text or sample metadata, it deep-copies the actual trace, changes one execution surface, and sends the counterfactual `EvalCase` through the same public `AgentEvaluator` metrics. Evaluation-data, evaluator, and infrastructure failures are excluded from prompt optimization. +## Distinction from other Issue #91 proposals + +PR #159 uses rule-first attribution from failure text and static expected/actual trace differences. PR #161 replays optimizer candidates and combines static trace/rubric attribution with bootstrap and Pareto selection. This example addresses a different question: which minimal trace intervention causally repairs the failing metric when the same `AgentEvaluator` evaluates it again? + +The attribution evidence is therefore an observed before/after metric delta from `replace_final_response`, `replace_tool_name`, `replace_tool_arguments`, or a bounded combination. Static trace differences can propose an intervention, but cannot by themselves establish the diagnosis. The same mechanism is reused after candidate validation to locate regressions. No case ID, expected failure label, attribution hint, or hand-authored metric score participates in the decision. + ## Quick start ```bash -python examples/optimization/eval_optimize_loop/run_counterfactual_probe.py -python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake -python examples/optimization/eval_optimize_loop/run_pipeline.py --mode trace -python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake --candidate-profile accepted -python examples/optimization/eval_optimize_loop/run_pipeline.py --mode fake --candidate-profile ineffective +python examples/optimization/counterfactual_trace_loop/run_counterfactual_probe.py +python examples/optimization/counterfactual_trace_loop/run_pipeline.py --mode fake +python examples/optimization/counterfactual_trace_loop/run_pipeline.py --mode trace +python examples/optimization/counterfactual_trace_loop/run_pipeline.py --mode fake --candidate-profile accepted +python examples/optimization/counterfactual_trace_loop/run_pipeline.py --mode fake --candidate-profile ineffective ``` Both fake and trace modes need no API key. Trace mode replays `actual_conversation`; `conversation` remains the expected trace. The fake optimizer is prompt-sensitive and introduces an intentionally broad billing rule without reading case IDs. diff --git a/examples/optimization/counterfactual_trace_loop/pipeline/pipeline.py b/examples/optimization/counterfactual_trace_loop/pipeline/pipeline.py index 1f043f22..84f02cdf 100644 --- a/examples/optimization/counterfactual_trace_loop/pipeline/pipeline.py +++ b/examples/optimization/counterfactual_trace_loop/pipeline/pipeline.py @@ -366,7 +366,7 @@ async def run_pipeline( "prompt_hashes": {k: _sha(v) for k, v in prompt_paths.items()}, "write_back": write_back, "reproduction_command": ( - "python examples/optimization/eval_optimize_loop/run_pipeline.py " + "python examples/optimization/counterfactual_trace_loop/run_pipeline.py " f"--mode {mode} --candidate-profile {candidate_profile}" ), }, diff --git a/examples/optimization/counterfactual_trace_loop/run_counterfactual_probe.py b/examples/optimization/counterfactual_trace_loop/run_counterfactual_probe.py index 550cb53b..79594fc0 100644 --- a/examples/optimization/counterfactual_trace_loop/run_counterfactual_probe.py +++ b/examples/optimization/counterfactual_trace_loop/run_counterfactual_probe.py @@ -12,7 +12,7 @@ if str(REPO_ROOT) not in sys.path: sys.path.insert(0, str(REPO_ROOT)) -from examples.optimization.eval_optimize_loop.pipeline.probe import ( # noqa: E402 +from examples.optimization.counterfactual_trace_loop.pipeline.probe import ( # noqa: E402 run_counterfactual_probe, ) diff --git a/examples/optimization/counterfactual_trace_loop/run_pipeline.py b/examples/optimization/counterfactual_trace_loop/run_pipeline.py index 71eafd16..25b9b26b 100644 --- a/examples/optimization/counterfactual_trace_loop/run_pipeline.py +++ b/examples/optimization/counterfactual_trace_loop/run_pipeline.py @@ -11,7 +11,7 @@ ROOT = HERE.parents[2] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) -from examples.optimization.eval_optimize_loop.pipeline.pipeline import run_pipeline # noqa: E402 +from examples.optimization.counterfactual_trace_loop.pipeline.pipeline import run_pipeline # noqa: E402 def main(): diff --git a/examples/optimization/counterfactual_trace_loop/sample_output/optimization_report.json b/examples/optimization/counterfactual_trace_loop/sample_output/optimization_report.json index fba3689e..0801e4fe 100644 --- a/examples/optimization/counterfactual_trace_loop/sample_output/optimization_report.json +++ b/examples/optimization/counterfactual_trace_loop/sample_output/optimization_report.json @@ -947,7 +947,7 @@ { "name": "latency_budget", "passed": true, - "observed": 0.29256009999880916, + "observed": 0.3700505000015255, "threshold": 180, "reason": "passed" }, @@ -970,7 +970,7 @@ "audit": { "seed": 42, "mode": "trace", - "duration_seconds": 0.29256009999880916, + "duration_seconds": 0.3700505000015255, "cost": { "total": 0.0 }, @@ -997,7 +997,7 @@ "system_prompt": "4e1c6a4ef268a492679ad6888230217d874733817e27137644f3364038642be9" } }, - "reproduction_command": "python examples/optimization/eval_optimize_loop/run_pipeline.py --mode trace --candidate-profile overfit" + "reproduction_command": "python examples/optimization/counterfactual_trace_loop/run_pipeline.py --mode trace --candidate-profile overfit" }, "known_limitations": [ "A local trace edit can be structurally valid but semantically incoherent with an original tool response.", diff --git a/tests/evaluation/test_counterfactual_trace_probe.py b/tests/evaluation/test_counterfactual_trace_probe.py index e09bddd9..7bf2b88f 100644 --- a/tests/evaluation/test_counterfactual_trace_probe.py +++ b/tests/evaluation/test_counterfactual_trace_probe.py @@ -14,11 +14,11 @@ from trpc_agent_sdk.types import FunctionResponse -from examples.optimization.eval_optimize_loop.pipeline.interventions import ( +from examples.optimization.counterfactual_trace_loop.pipeline.interventions import ( InterventionKind, build_counterfactual, ) -from examples.optimization.eval_optimize_loop.pipeline.probe import ( +from examples.optimization.counterfactual_trace_loop.pipeline.probe import ( build_probe_cases, evaluate_trace_cases, run_counterfactual_probe, diff --git a/tests/evaluation/test_eval_optimize_loop.py b/tests/evaluation/test_eval_optimize_loop.py index 77f3d2af..4112bfe6 100644 --- a/tests/evaluation/test_eval_optimize_loop.py +++ b/tests/evaluation/test_eval_optimize_loop.py @@ -8,24 +8,24 @@ import pytest -from examples.optimization.eval_optimize_loop.pipeline.diagnosis import ( +from examples.optimization.counterfactual_trace_loop.pipeline.diagnosis import ( InfrastructureFailure, attribute_from_evidence, build_failure_digest, classify_non_agent_failure, select_target_prompts, ) -from examples.optimization.eval_optimize_loop.pipeline.gate import evaluate_gate -from examples.optimization.eval_optimize_loop.pipeline.input_audit import audit_eval_cases -from examples.optimization.eval_optimize_loop.pipeline.models import CounterfactualEvidence -from examples.optimization.eval_optimize_loop.fake.model import generate_trace -from examples.optimization.eval_optimize_loop.pipeline.optimizer import ( +from examples.optimization.counterfactual_trace_loop.pipeline.gate import evaluate_gate +from examples.optimization.counterfactual_trace_loop.pipeline.input_audit import audit_eval_cases +from examples.optimization.counterfactual_trace_loop.pipeline.models import CounterfactualEvidence +from examples.optimization.counterfactual_trace_loop.fake.model import generate_trace +from examples.optimization.counterfactual_trace_loop.pipeline.optimizer import ( apply_if_accepted, run_real_optimizer, ) -from examples.optimization.eval_optimize_loop.pipeline.pipeline import run_pipeline -from examples.optimization.eval_optimize_loop.pipeline.pipeline import _evaluate_with_triage -from examples.optimization.eval_optimize_loop.pipeline.probe import build_probe_cases +from examples.optimization.counterfactual_trace_loop.pipeline.pipeline import run_pipeline +from examples.optimization.counterfactual_trace_loop.pipeline.pipeline import _evaluate_with_triage +from examples.optimization.counterfactual_trace_loop.pipeline.probe import build_probe_cases def _evidence(name: str, passed: bool, repaired: list[str]) -> CounterfactualEvidence: @@ -289,7 +289,7 @@ async def test_fake_and_trace_pipeline_have_same_schema_without_api_key(tmp_path for key in list(__import__("os").environ): if "API_KEY" in key: monkeypatch.delenv(key, raising=False) - base = Path(__file__).parents[2] / "examples" / "optimization" / "eval_optimize_loop" + base = Path(__file__).parents[2] / "examples" / "optimization" / "counterfactual_trace_loop" fake = await run_pipeline(base, "fake", tmp_path / "fake") trace = await run_pipeline(base, "trace", tmp_path / "trace") assert fake.keys() == trace.keys() @@ -302,7 +302,7 @@ async def test_fake_and_trace_pipeline_have_same_schema_without_api_key(tmp_path @pytest.mark.asyncio async def test_real_evaluations_drive_accept_overfit_and_ineffective_decisions(tmp_path): - base = Path(__file__).parents[2] / "examples" / "optimization" / "eval_optimize_loop" + base = Path(__file__).parents[2] / "examples" / "optimization" / "counterfactual_trace_loop" accepted = await run_pipeline(base, "fake", tmp_path / "accepted", candidate_profile="accepted") overfit = await run_pipeline(base, "fake", tmp_path / "overfit", candidate_profile="overfit") ineffective = await run_pipeline(base, "fake", tmp_path / "ineffective", candidate_profile="ineffective") @@ -317,7 +317,7 @@ async def test_real_evaluations_drive_accept_overfit_and_ineffective_decisions(t @pytest.mark.asyncio async def test_real_optimizer_receives_filtered_train_and_never_updates_source(tmp_path, monkeypatch): - base = Path(__file__).parents[2] / "examples" / "optimization" / "eval_optimize_loop" + base = Path(__file__).parents[2] / "examples" / "optimization" / "counterfactual_trace_loop" captured = {} async def spy(**kwargs): @@ -347,7 +347,7 @@ async def spy(**kwargs): ) monkeypatch.setattr( - "examples.optimization.eval_optimize_loop.pipeline.optimizer.AgentOptimizer.optimize", + "examples.optimization.counterfactual_trace_loop.pipeline.optimizer.AgentOptimizer.optimize", spy, ) result = await run_real_optimizer( @@ -385,7 +385,7 @@ async def spy(**kwargs): @pytest.mark.asyncio async def test_real_optimizer_restores_prompt_if_optimizer_mutates_then_fails(tmp_path, monkeypatch): - base = Path(__file__).parents[2] / "examples" / "optimization" / "eval_optimize_loop" + base = Path(__file__).parents[2] / "examples" / "optimization" / "counterfactual_trace_loop" prompt = tmp_path / "router.md" prompt.write_text("baseline", encoding="utf-8") @@ -394,7 +394,7 @@ async def broken_optimizer(**kwargs): raise RuntimeError("optimizer failed") monkeypatch.setattr( - "examples.optimization.eval_optimize_loop.pipeline.optimizer.AgentOptimizer.optimize", + "examples.optimization.counterfactual_trace_loop.pipeline.optimizer.AgentOptimizer.optimize", broken_optimizer, ) @@ -414,7 +414,7 @@ async def broken_optimizer(**kwargs): @pytest.mark.asyncio async def test_pipeline_rejects_unsupported_real_mode(tmp_path): - base = Path(__file__).parents[2] / "examples" / "optimization" / "eval_optimize_loop" + base = Path(__file__).parents[2] / "examples" / "optimization" / "counterfactual_trace_loop" with pytest.raises(ValueError, match="fake or trace"): await run_pipeline(base, "real", tmp_path) @@ -422,7 +422,7 @@ async def test_pipeline_rejects_unsupported_real_mode(tmp_path): @pytest.mark.asyncio async def test_report_contains_trace_status_evidence_and_limitations(tmp_path): - base = Path(__file__).parents[2] / "examples" / "optimization" / "eval_optimize_loop" + base = Path(__file__).parents[2] / "examples" / "optimization" / "counterfactual_trace_loop" report = await run_pipeline(base, "trace", tmp_path) case = report["baseline"]["train"]["case_results"][0] assert {"case_id", "passed", "metrics", "failure_reason", "trace_summary"} <= case.keys() @@ -435,7 +435,7 @@ async def test_report_contains_trace_status_evidence_and_limitations(tmp_path): @pytest.mark.asyncio async def test_fake_round_audit_contains_candidate_prompts_and_evaluation(tmp_path): - base = Path(__file__).parents[2] / "examples" / "optimization" / "eval_optimize_loop" + base = Path(__file__).parents[2] / "examples" / "optimization" / "counterfactual_trace_loop" report = await run_pipeline(base, "fake", tmp_path, candidate_profile="accepted") @@ -450,7 +450,7 @@ async def test_fake_round_audit_contains_candidate_prompts_and_evaluation(tmp_pa def test_committed_sample_output_has_current_report_contract(): - sample = Path(__file__).parents[2] / "examples" / "optimization" / "eval_optimize_loop" / "sample_output" + sample = Path(__file__).parents[2] / "examples" / "optimization" / "counterfactual_trace_loop" / "sample_output" report = json.loads((sample / "optimization_report.json").read_text(encoding="utf-8")) assert report["schema_version"] == "1.0" assert report["baseline"]["train"]["case_results"] @@ -471,7 +471,7 @@ async def evaluator(case_batch, workspace): return {case.eval_id: {"tool_trajectory_avg_score": 1.0}} monkeypatch.setattr( - "examples.optimization.eval_optimize_loop.pipeline.pipeline.evaluate_trace_cases", + "examples.optimization.counterfactual_trace_loop.pipeline.pipeline.evaluate_trace_cases", evaluator, ) metrics, failures = await _evaluate_with_triage(cases, tmp_path)