From 3e3831be4aac1bf6928f3be894ae5cf079c9e21a Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Thu, 23 Jul 2026 22:46:22 +0800 Subject: [PATCH 1/2] chore: make RAG canary variability diagnosable Persist structured answer-quality reports with run context and add stable content, provider, and latency comparison fingerprints. No retrieval or answer behaviour changes. --- .github/workflows/eval-canary.yml | 2 +- docs/observability-slos.md | 17 ++- scripts/eval-quality.ts | 17 +++ scripts/eval-trend.mjs | 161 ++++++++++++++++++++++++++++- tests/eval-canary-workflow.test.ts | 1 + tests/eval-quality.test.ts | 26 +++++ tests/eval-trend.test.ts | 107 ++++++++++++++++++- 7 files changed, 323 insertions(+), 8 deletions(-) diff --git a/.github/workflows/eval-canary.yml b/.github/workflows/eval-canary.yml index bf75b24ca..e2d950161 100644 --- a/.github/workflows/eval-canary.yml +++ b/.github/workflows/eval-canary.yml @@ -169,7 +169,7 @@ jobs: fi mkdir -p .local/eval-canary set -o pipefail - npm run eval:quality -- --rag-only --limit "$ANSWER_CASE_LIMIT" --fail-on-threshold 2>&1 | tee .local/eval-canary/answer-quality.log + npm run eval:quality -- --rag-only --limit "$ANSWER_CASE_LIMIT" --output-dir .local/eval-canary/quality-reports --fail-on-threshold 2>&1 | tee .local/eval-canary/answer-quality.log # Phase E baseline/paired-run instrument: the five answer-quality metrics (relevance, # readability, artifact leaks, intent coverage, fail-closed) + per-intent structural diff --git a/docs/observability-slos.md b/docs/observability-slos.md index e6a501bab..f5b9a0063 100644 --- a/docs/observability-slos.md +++ b/docs/observability-slos.md @@ -140,9 +140,11 @@ What it does, in order: corpus. This is the eval CI never runs on PRs (it needs live Supabase + OpenAI keys); the canary makes it a standing weekly guard instead of a manual pre-merge step that can be skipped. -3. `npm run eval:quality -- --rag-only --limit 8 --fail-on-threshold` — a - small answer-quality subset (grounding, citations, unsupported-correctness) - to bound OpenAI spend while still catching generation-side regressions. +3. `npm run eval:quality -- --rag-only --limit 44 --fail-on-threshold` — the + full committed answer-quality set (grounding, citations, + unsupported-correctness). The workflow writes the structured JSON and + Markdown reports into the `eval-canary-output` artifact with the tested Git + SHA and run identity. Failing loudly: @@ -174,7 +176,14 @@ Operational notes: scheduled run's OpenAI rate limit mid-run. - Evals write telemetry rows (`rag_queries`) but mutate no content. - Cost bound: 36 committed retrieval cases plus any captured cases (embedding - calls only on forced-vector probes) + 8 generated answers per week. + calls only on forced-vector probes) + 44 generated answers per week. +- To compare two or more downloaded structured answer reports without treating + raw millisecond jitter as a content regression, run + `npm run eval:trend -- --answer-quality `. + Add `--case ` for the per-run route, latency, and diagnostic + signature of one case. A `same_tree_content_variability` result means the + reports share a Git SHA but differ in content/citation outcomes; provider and + latency-threshold variability are reported separately. - **The schedule only runs from `main`.** After merging, trigger one `workflow_dispatch` run and confirm it goes green before trusting the weekly cadence (repo gate for this workflow). diff --git a/scripts/eval-quality.ts b/scripts/eval-quality.ts index 150f86209..c0ace8d11 100644 --- a/scripts/eval-quality.ts +++ b/scripts/eval-quality.ts @@ -118,6 +118,22 @@ export type QualityFailureCategory = export type EvalQualityReport = ReturnType; +export type EvalQualityRunContext = { + git_sha: string | null; + github_run_id: string | null; + github_run_attempt: string | null; + latency_context: string; +}; + +export function evalQualityRunContext(env: Record = process.env): EvalQualityRunContext { + return { + git_sha: env.EVAL_GIT_SHA?.trim() || env.GITHUB_SHA?.trim() || null, + github_run_id: env.GITHUB_RUN_ID?.trim() || null, + github_run_attempt: env.GITHUB_RUN_ATTEMPT?.trim() || null, + latency_context: env.EVAL_LATENCY_CONTEXT?.trim() || "default", + }; +} + export type SourceMetadataDebtAcceptance = { path?: string; accepted_by: string; @@ -703,6 +719,7 @@ export function buildEvalQualityReport(args: { return { generated_at: args.generatedAt ?? new Date().toISOString(), + run_context: evalQualityRunContext(), provider: { ...providerEvidence, passed: diff --git a/scripts/eval-trend.mjs b/scripts/eval-trend.mjs index 7ce6aa51f..5535e9dd3 100644 --- a/scripts/eval-trend.mjs +++ b/scripts/eval-trend.mjs @@ -10,6 +10,7 @@ // providers, no repo state — the durable trend record without new infrastructure // (docs/observability-slos.md §3.1). import { readFileSync } from "node:fs"; +import { createHash } from "node:crypto"; /** One trend row per artifact payload; exported for tests. */ export function buildTrendRows(payloads) { @@ -46,6 +47,153 @@ export function buildCaseTrend(payloads, caseId) { }); } +function answerResults(payload) { + return Array.isArray(payload?.rag?.results) ? payload.rag.results : []; +} + +function answerFailureCategory(message) { + const normalized = String(message ?? "").toLowerCase(); + if (normalized.includes("citation")) return "citation"; + if (normalized.includes("latency")) return "latency"; + if (normalized.includes("grounded answer")) return "grounding"; + if (normalized.includes("unsupported answer") || normalized.includes("false-positive")) { + return "unsupported_correctness"; + } + if (normalized.includes("expected document") || normalized.includes("retrieved sources")) return "expected_source"; + if (normalized.includes("expected content")) return "expected_content"; + if (normalized.includes("numeric") || normalized.includes("faithfulness") || normalized.includes("verify against")) { + return "numeric_grounding"; + } + if ( + normalized.includes("source governance") || + normalized.includes("outdated") || + normalized.includes("unverified") + ) { + return "source_governance"; + } + return "other"; +} + +function sortedUnique(values) { + return [...new Set(values)].sort(); +} + +function stableSignature(value) { + return createHash("sha256").update(JSON.stringify(value)).digest("hex").slice(0, 16); +} + +function answerFingerprints(result) { + const categories = sortedUnique((result.failures ?? []).map(answerFailureCategory)); + const contentCategories = categories.filter((category) => category !== "latency"); + const routingReason = String(result.routingReason ?? ""); + const providerReason = /provider|generation_(?:fallback|failed|quality)|max_output_tokens|model_timeout/i.test( + routingReason, + ) + ? routingReason + : "none"; + const content = { + categories: contentCategories, + grounded: Boolean(result.grounded), + expectedHit: Boolean(result.expectedHit), + citations: Number(result.citations ?? 0), + missingFiles: sortedUnique(result.missingFiles ?? []), + sourceDangerWarningCount: Number(result.sourceDangerWarningCount ?? 0), + unverifiedNumericTokenCount: Number(result.unverifiedNumericTokenCount ?? 0), + hasFaithfulnessWarning: Boolean(result.hasFaithfulnessWarning), + }; + const provider = { + route: result.route ?? "none", + latencyRoute: result.latencyRoute ?? "none", + providerReason, + model: result.model ?? null, + requestIdsPresent: (result.openAIRequestIds?.length ?? 0) > 0, + }; + const latency = { + latencyFailure: categories.includes("latency"), + routeCeilingExceeded: Boolean(result.routeCeilingExceeded), + routeDeadlineExceeded: Boolean(result.timings?.routeDeadlineExceeded), + budgetExhaustedByRetrieval: Boolean(result.timings?.budgetExhaustedByRetrieval), + latencyRoute: result.latencyRoute ?? "none", + }; + return { + content, + provider, + latency, + diagnosticSignature: stableSignature({ content, provider, latency }), + }; +} + +function allEqual(values) { + return values.length > 0 && values.every((value) => value === values[0]); +} + +function reportsUseSameTree(payloads) { + const shas = payloads.map(({ payload }) => payload?.run_context?.git_sha ?? null); + return shas.length > 1 && shas.every(Boolean) && allEqual(shas); +} + +/** + * Compare structured eval:quality reports without treating raw millisecond drift as a regression. + * Content, provider/route, and latency-threshold outcomes receive separate stable fingerprints. + */ +export function buildAnswerQualityVariabilityRows(payloads) { + const ids = sortedUnique(payloads.flatMap(({ payload }) => answerResults(payload).map((result) => result.id))); + const sameTree = reportsUseSameTree(payloads); + + return ids.map((caseId) => { + const matches = payloads.map(({ payload }) => answerResults(payload).find((result) => result.id === caseId)); + if (matches.some((result) => !result)) { + return { + case: caseId, + classification: "missing_case", + same_tree: sameTree, + runs: matches.filter(Boolean).length, + baseline_signature: matches[0] ? answerFingerprints(matches[0]).diagnosticSignature : null, + latest_signature: matches.at(-1) ? answerFingerprints(matches.at(-1)).diagnosticSignature : null, + }; + } + + const fingerprints = matches.map(answerFingerprints); + const contentSignatures = fingerprints.map(({ content }) => stableSignature(content)); + const providerSignatures = fingerprints.map(({ provider }) => stableSignature(provider)); + const latencySignatures = fingerprints.map(({ latency }) => stableSignature(latency)); + const repeatedContentFailure = + allEqual(contentSignatures) && fingerprints.every(({ content }) => content.categories.length > 0); + let classification = "stable"; + if (!allEqual(contentSignatures)) classification = sameTree ? "same_tree_content_variability" : "content_change"; + else if (repeatedContentFailure) classification = "repeated_content_failure"; + else if (!allEqual(providerSignatures)) classification = "provider_route_variability"; + else if (!allEqual(latencySignatures)) classification = "latency_variability"; + + return { + case: caseId, + classification, + same_tree: sameTree, + runs: matches.length, + baseline_signature: fingerprints[0].diagnosticSignature, + latest_signature: fingerprints.at(-1).diagnosticSignature, + }; + }); +} + +/** Per-run answer diagnostics for one case; exported for focused offline investigation. */ +export function buildAnswerQualityCaseTrend(payloads, caseId) { + return payloads.map(({ label, payload }) => { + const match = answerResults(payload).find((result) => result.id === caseId); + const fingerprints = match ? answerFingerprints(match) : null; + return { + label, + git_sha: payload?.run_context?.git_sha ?? null, + found: Boolean(match), + passed: match ? (match.failures?.length ?? 0) === 0 : null, + route: match?.route ?? null, + latency_route: match?.latencyRoute ?? null, + total_ms: match?.latencyMs ?? null, + diagnostic_signature: fingerprints?.diagnosticSignature ?? null, + }; + }); +} + function formatTable(rows) { if (!rows.length) return "(no rows)"; const keys = Object.keys(rows[0]); @@ -61,14 +209,23 @@ function formatTable(rows) { function main() { const argv = process.argv.slice(2); + const answerQuality = argv.includes("--answer-quality"); const caseFlag = argv.indexOf("--case"); const caseId = caseFlag >= 0 ? argv[caseFlag + 1] : undefined; - const files = argv.filter((arg, index) => arg !== "--case" && index !== caseFlag + 1); + const files = argv.filter((arg, index) => arg !== "--case" && arg !== "--answer-quality" && index !== caseFlag + 1); if (!files.length) { - console.error("Usage: eval-trend [--case ] (oldest first)"); + console.error("Usage: eval-trend [--answer-quality] [--case ] (oldest first)"); process.exit(2); } const payloads = files.map((file) => ({ label: file, payload: JSON.parse(readFileSync(file, "utf8")) })); + if (answerQuality) { + console.log(formatTable(buildAnswerQualityVariabilityRows(payloads))); + if (caseId) { + console.log(`\nAnswer case trend: ${caseId}`); + console.log(formatTable(buildAnswerQualityCaseTrend(payloads, caseId))); + } + return; + } console.log(formatTable(buildTrendRows(payloads))); if (caseId) { console.log(`\nCase trend: ${caseId}`); diff --git a/tests/eval-canary-workflow.test.ts b/tests/eval-canary-workflow.test.ts index ca97a9430..6f53e3a86 100644 --- a/tests/eval-canary-workflow.test.ts +++ b/tests/eval-canary-workflow.test.ts @@ -31,6 +31,7 @@ describe("eval canary workflow input", () => { expect(workflow).toContain("set -o pipefail"); expect(workflow).toContain("tee .local/eval-canary/golden-retrieval.log"); expect(workflow).toContain("tee .local/eval-canary/answer-quality.log"); + expect(workflow).toContain("--output-dir .local/eval-canary/quality-reports"); expect(workflow).toContain( "await import(pathToFileURL(`${process.env.GITHUB_WORKSPACE}/scripts/productivity-core.mjs`).href)", ); diff --git a/tests/eval-quality.test.ts b/tests/eval-quality.test.ts index 9f4c311e5..7375b0eb9 100644 --- a/tests/eval-quality.test.ts +++ b/tests/eval-quality.test.ts @@ -4,6 +4,7 @@ import { buildEvalQualityReport, configureEvalProviderEnvironment, deliveredGroundedAfterSourceGovernancePolicy, + evalQualityRunContext, qualityFailureCategory, ragAnswerTimingDiagnostics, renderEvalQualityMarkdown, @@ -14,6 +15,31 @@ import { } from "../scripts/eval-quality"; import { evaluateGoldenRetrievalCase, type GoldenRetrievalResult } from "../scripts/eval-retrieval"; +describe("eval quality run context", () => { + it("records stable run identity without requiring GitHub Actions", () => { + expect( + evalQualityRunContext({ + EVAL_GIT_SHA: " candidate-sha ", + GITHUB_SHA: "ignored-fallback", + GITHUB_RUN_ID: "123", + GITHUB_RUN_ATTEMPT: "2", + EVAL_LATENCY_CONTEXT: "cross-region-runner", + }), + ).toEqual({ + git_sha: "candidate-sha", + github_run_id: "123", + github_run_attempt: "2", + latency_context: "cross-region-runner", + }); + expect(evalQualityRunContext({})).toEqual({ + git_sha: null, + github_run_id: null, + github_run_attempt: null, + latency_context: "default", + }); + }); +}); + function retrievalResult(overrides: Partial = {}): GoldenRetrievalResult { const base: GoldenRetrievalResult = { id: "retrieval-1", diff --git a/tests/eval-trend.test.ts b/tests/eval-trend.test.ts index b07cfa53d..166d7517f 100644 --- a/tests/eval-trend.test.ts +++ b/tests/eval-trend.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from "vitest"; -import { buildCaseTrend, buildTrendRows } from "../scripts/eval-trend.mjs"; +import { + buildAnswerQualityCaseTrend, + buildAnswerQualityVariabilityRows, + buildCaseTrend, + buildTrendRows, +} from "../scripts/eval-trend.mjs"; const payload = (overrides: Record = {}) => ({ label: "run-a.json", @@ -58,3 +63,103 @@ describe("eval-trend aggregation", () => { expect(missing[0]).toMatchObject({ found: false, rr_at_10: null, passed: null }); }); }); + +const answerResult = (overrides: Record = {}) => ({ + id: "neuroleptic-side-effect-escalation", + failures: [], + grounded: true, + expectedHit: true, + citations: 3, + missingFiles: [], + sourceDangerWarningCount: 0, + unverifiedNumericTokenCount: 0, + hasFaithfulnessWarning: false, + route: "fast", + latencyRoute: "fast", + routingReason: "clinical_risk_or_complex_query", + model: "gpt-test", + openAIRequestIds: ["req-test"], + routeCeilingExceeded: false, + latencyMs: 1200, + timings: { + routeDeadlineExceeded: false, + budgetExhaustedByRetrieval: false, + }, + ...overrides, +}); + +const answerReport = (label: string, sha: string, result: Record) => ({ + label, + payload: { + run_context: { git_sha: sha }, + rag: { results: [result] }, + }, +}); + +describe("eval-trend answer-quality variability", () => { + it("marks pass-to-content-failure changes on the same tree as variability, not a deterministic regression", () => { + const rows = buildAnswerQualityVariabilityRows([ + answerReport("baseline", "same-sha", answerResult()), + answerReport( + "confirmation", + "same-sha", + answerResult({ failures: ["citation count below required minimum"], citations: 1 }), + ), + ]); + + expect(rows).toEqual([ + expect.objectContaining({ + case: "neuroleptic-side-effect-escalation", + classification: "same_tree_content_variability", + same_tree: true, + runs: 2, + }), + ]); + }); + + it("separates latency and provider-route variability from content changes", () => { + const latencyRows = buildAnswerQualityVariabilityRows([ + answerReport("baseline", "same-sha", answerResult()), + answerReport( + "slow", + "same-sha", + answerResult({ + failures: ["route latency ceiling exceeded: 5000ms total"], + routeCeilingExceeded: true, + latencyMs: 5000, + timings: { routeDeadlineExceeded: true, budgetExhaustedByRetrieval: false }, + }), + ), + ]); + expect(latencyRows[0]).toMatchObject({ classification: "latency_variability", same_tree: true }); + + const providerRows = buildAnswerQualityVariabilityRows([ + answerReport("baseline", "same-sha", answerResult()), + answerReport( + "fallback", + "same-sha", + answerResult({ + route: "extractive", + latencyRoute: "fallback", + routingReason: "generation_fallback:provider_timeout", + model: null, + }), + ), + ]); + expect(providerRows[0]).toMatchObject({ classification: "provider_route_variability", same_tree: true }); + }); + + it("identifies a repeated content failure and emits stable per-case signatures", () => { + const failed = answerResult({ failures: ["citation count below required minimum"], citations: 1 }); + const payloads = [answerReport("run-a", "same-sha", failed), answerReport("run-b", "same-sha", failed)]; + expect(buildAnswerQualityVariabilityRows(payloads)[0]).toMatchObject({ + classification: "repeated_content_failure", + same_tree: true, + }); + + const trend = buildAnswerQualityCaseTrend(payloads, "neuroleptic-side-effect-escalation"); + expect(trend[0]).toMatchObject({ found: true, passed: false, git_sha: "same-sha" }); + expect(trend[0].diagnostic_signature).toMatch(/^[a-f0-9]{16}$/); + expect(trend[1].diagnostic_signature).toBe(trend[0].diagnostic_signature); + }); +}); From 6f863b0a0a9bf436a419d73ba6e047342725b1ed Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Thu, 23 Jul 2026 22:51:54 +0800 Subject: [PATCH 2/2] fix: record the evaluated canary tree Export the checked-out SHA so optional-ref canary artifacts cannot be mislabeled as same-tree comparisons. --- .github/workflows/eval-canary.yml | 4 ++++ tests/eval-canary-workflow.test.ts | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/eval-canary.yml b/.github/workflows/eval-canary.yml index e2d950161..238a24043 100644 --- a/.github/workflows/eval-canary.yml +++ b/.github/workflows/eval-canary.yml @@ -102,6 +102,10 @@ jobs: # Empty ref = checkout's default (the triggering sha) — schedule runs unaffected. ref: ${{ github.event.inputs.ref || '' }} + - name: Record evaluated Git SHA + shell: bash + run: echo "EVAL_GIT_SHA=$(git rev-parse HEAD)" >> "$GITHUB_ENV" + # Fail loudly on a malformed override: resolveRankingConfig (src/lib/ranking-config.ts) # silently falls back to production defaults on bad JSON, which would turn a staged-weights # eval into an accidental baseline and corrupt the pair comparison. diff --git a/tests/eval-canary-workflow.test.ts b/tests/eval-canary-workflow.test.ts index 6f53e3a86..ecedea3d5 100644 --- a/tests/eval-canary-workflow.test.ts +++ b/tests/eval-canary-workflow.test.ts @@ -18,6 +18,11 @@ describe("eval canary workflow input", () => { expect(workflow).not.toMatch(/run:.*github\.event\.inputs\.answer_case_limit/); }); + it("records the actual checked-out tree for optional-ref comparisons", () => { + expect(workflow).toContain("ref: ${{ github.event.inputs.ref || '' }}"); + expect(workflow).toContain('echo "EVAL_GIT_SHA=$(git rev-parse HEAD)" >> "$GITHUB_ENV"'); + }); + it("distinguishes provider outages from retrieval regressions in the failure issue", () => { expect(workflow).toContain('title: "Eval canary failure: weekly evaluation did not complete"'); expect(workflow).toContain("Resolve provider quota/auth/config failures before rerunning");