RAG canary: distinguish content, provider, and latency variability

.github/workflows/eval-canary.yml

@@ -102,6 +102,10 @@ jobs:
           # Empty ref = checkout's default (the triggering sha) — schedule runs unaffected.
           ref: ${{ github.event.inputs.ref || '' }}
 
+      - name: Record evaluated Git SHA
+        shell: bash
+        run: echo "EVAL_GIT_SHA=$(git rev-parse HEAD)" >> "$GITHUB_ENV"
+
       # Fail loudly on a malformed override: resolveRankingConfig (src/lib/ranking-config.ts)
       # silently falls back to production defaults on bad JSON, which would turn a staged-weights
       # eval into an accidental baseline and corrupt the pair comparison.
@@ -169,7 +173,7 @@ jobs:
           fi
           mkdir -p .local/eval-canary
           set -o pipefail
-          npm run eval:quality -- --rag-only --limit "$ANSWER_CASE_LIMIT" --fail-on-threshold 2>&1 | tee .local/eval-canary/answer-quality.log
+          npm run eval:quality -- --rag-only --limit "$ANSWER_CASE_LIMIT" --output-dir .local/eval-canary/quality-reports --fail-on-threshold 2>&1 | tee .local/eval-canary/answer-quality.log
 
       # Phase E baseline/paired-run instrument: the five answer-quality metrics (relevance,
       # readability, artifact leaks, intent coverage, fail-closed) + per-intent structural

docs/observability-slos.md

@@ -140,9 +140,11 @@ What it does, in order:
    corpus. This is the eval CI never runs on PRs (it needs live Supabase +
    OpenAI keys); the canary makes it a standing weekly guard instead of a
    manual pre-merge step that can be skipped.
-3. `npm run eval:quality -- --rag-only --limit 8 --fail-on-threshold` — a
-   small answer-quality subset (grounding, citations, unsupported-correctness)
-   to bound OpenAI spend while still catching generation-side regressions.
+3. `npm run eval:quality -- --rag-only --limit 44 --fail-on-threshold` — the
+   full committed answer-quality set (grounding, citations,
+   unsupported-correctness). The workflow writes the structured JSON and
+   Markdown reports into the `eval-canary-output` artifact with the tested Git
+   SHA and run identity.
 
 Failing loudly:
 
@@ -174,7 +176,14 @@ Operational notes:
   scheduled run's OpenAI rate limit mid-run.
 - Evals write telemetry rows (`rag_queries`) but mutate no content.
 - Cost bound: 36 committed retrieval cases plus any captured cases (embedding
-  calls only on forced-vector probes) + 8 generated answers per week.
+  calls only on forced-vector probes) + 44 generated answers per week.
+- To compare two or more downloaded structured answer reports without treating
+  raw millisecond jitter as a content regression, run
+  `npm run eval:trend -- --answer-quality <oldest-report.json> <newest-report.json>`.
+  Add `--case <case-id>` for the per-run route, latency, and diagnostic
+  signature of one case. A `same_tree_content_variability` result means the
+  reports share a Git SHA but differ in content/citation outcomes; provider and
+  latency-threshold variability are reported separately.
 - **The schedule only runs from `main`.** After merging, trigger one
   `workflow_dispatch` run and confirm it goes green before trusting the
   weekly cadence (repo gate for this workflow).

scripts/eval-quality.ts

@@ -118,6 +118,22 @@ export type QualityFailureCategory =
 
 export type EvalQualityReport = ReturnType<typeof buildEvalQualityReport>;
 
+export type EvalQualityRunContext = {
+  git_sha: string | null;
+  github_run_id: string | null;
+  github_run_attempt: string | null;
+  latency_context: string;
+};
+
+export function evalQualityRunContext(env: Record<string, string | undefined> = process.env): EvalQualityRunContext {
+  return {
+    git_sha: env.EVAL_GIT_SHA?.trim() || env.GITHUB_SHA?.trim() || null,
+    github_run_id: env.GITHUB_RUN_ID?.trim() || null,
+    github_run_attempt: env.GITHUB_RUN_ATTEMPT?.trim() || null,
+    latency_context: env.EVAL_LATENCY_CONTEXT?.trim() || "default",
+  };
+}
+
 export type SourceMetadataDebtAcceptance = {
   path?: string;
   accepted_by: string;
@@ -703,6 +719,7 @@ export function buildEvalQualityReport(args: {
 
   return {
     generated_at: args.generatedAt ?? new Date().toISOString(),
+    run_context: evalQualityRunContext(),
     provider: {
       ...providerEvidence,
       passed:

scripts/eval-trend.mjs

@@ -10,6 +10,7 @@
 // providers, no repo state — the durable trend record without new infrastructure
 // (docs/observability-slos.md §3.1).
 import { readFileSync } from "node:fs";
+import { createHash } from "node:crypto";
 
 /** One trend row per artifact payload; exported for tests. */
 export function buildTrendRows(payloads) {
@@ -46,6 +47,153 @@ export function buildCaseTrend(payloads, caseId) {
   });
 }
 
+function answerResults(payload) {
+  return Array.isArray(payload?.rag?.results) ? payload.rag.results : [];
+}
+
+function answerFailureCategory(message) {
+  const normalized = String(message ?? "").toLowerCase();
+  if (normalized.includes("citation")) return "citation";
+  if (normalized.includes("latency")) return "latency";
+  if (normalized.includes("grounded answer")) return "grounding";
+  if (normalized.includes("unsupported answer") || normalized.includes("false-positive")) {
+    return "unsupported_correctness";
+  }
+  if (normalized.includes("expected document") || normalized.includes("retrieved sources")) return "expected_source";
+  if (normalized.includes("expected content")) return "expected_content";
+  if (normalized.includes("numeric") || normalized.includes("faithfulness") || normalized.includes("verify against")) {
+    return "numeric_grounding";
+  }
+  if (
+    normalized.includes("source governance") ||
+    normalized.includes("outdated") ||
+    normalized.includes("unverified")
+  ) {
+    return "source_governance";
+  }
+  return "other";
+}
+
+function sortedUnique(values) {
+  return [...new Set(values)].sort();
+}
+
+function stableSignature(value) {
+  return createHash("sha256").update(JSON.stringify(value)).digest("hex").slice(0, 16);
+}
+
+function answerFingerprints(result) {
+  const categories = sortedUnique((result.failures ?? []).map(answerFailureCategory));
+  const contentCategories = categories.filter((category) => category !== "latency");
+  const routingReason = String(result.routingReason ?? "");
+  const providerReason = /provider|generation_(?:fallback|failed|quality)|max_output_tokens|model_timeout/i.test(
+    routingReason,
+  )
+    ? routingReason
+    : "none";
+  const content = {
+    categories: contentCategories,
+    grounded: Boolean(result.grounded),
+    expectedHit: Boolean(result.expectedHit),
+    citations: Number(result.citations ?? 0),
+    missingFiles: sortedUnique(result.missingFiles ?? []),
+    sourceDangerWarningCount: Number(result.sourceDangerWarningCount ?? 0),
+    unverifiedNumericTokenCount: Number(result.unverifiedNumericTokenCount ?? 0),
+    hasFaithfulnessWarning: Boolean(result.hasFaithfulnessWarning),
+  };
+  const provider = {
+    route: result.route ?? "none",
+    latencyRoute: result.latencyRoute ?? "none",
+    providerReason,
+    model: result.model ?? null,
+    requestIdsPresent: (result.openAIRequestIds?.length ?? 0) > 0,
+  };
+  const latency = {
+    latencyFailure: categories.includes("latency"),
+    routeCeilingExceeded: Boolean(result.routeCeilingExceeded),
+    routeDeadlineExceeded: Boolean(result.timings?.routeDeadlineExceeded),
+    budgetExhaustedByRetrieval: Boolean(result.timings?.budgetExhaustedByRetrieval),
+    latencyRoute: result.latencyRoute ?? "none",
+  };
+  return {
+    content,
+    provider,
+    latency,
+    diagnosticSignature: stableSignature({ content, provider, latency }),
+  };
+}
+
+function allEqual(values) {
+  return values.length > 0 && values.every((value) => value === values[0]);
+}
+
+function reportsUseSameTree(payloads) {
+  const shas = payloads.map(({ payload }) => payload?.run_context?.git_sha ?? null);
+  return shas.length > 1 && shas.every(Boolean) && allEqual(shas);
+}
+
+/**
+ * Compare structured eval:quality reports without treating raw millisecond drift as a regression.
+ * Content, provider/route, and latency-threshold outcomes receive separate stable fingerprints.
+ */
+export function buildAnswerQualityVariabilityRows(payloads) {
+  const ids = sortedUnique(payloads.flatMap(({ payload }) => answerResults(payload).map((result) => result.id)));
+  const sameTree = reportsUseSameTree(payloads);
+
+  return ids.map((caseId) => {
+    const matches = payloads.map(({ payload }) => answerResults(payload).find((result) => result.id === caseId));
+    if (matches.some((result) => !result)) {
+      return {
+        case: caseId,
+        classification: "missing_case",
+        same_tree: sameTree,
+        runs: matches.filter(Boolean).length,
+        baseline_signature: matches[0] ? answerFingerprints(matches[0]).diagnosticSignature : null,
+        latest_signature: matches.at(-1) ? answerFingerprints(matches.at(-1)).diagnosticSignature : null,
+      };
+    }
+
+    const fingerprints = matches.map(answerFingerprints);
+    const contentSignatures = fingerprints.map(({ content }) => stableSignature(content));
+    const providerSignatures = fingerprints.map(({ provider }) => stableSignature(provider));
+    const latencySignatures = fingerprints.map(({ latency }) => stableSignature(latency));
+    const repeatedContentFailure =
+      allEqual(contentSignatures) && fingerprints.every(({ content }) => content.categories.length > 0);
+    let classification = "stable";
+    if (!allEqual(contentSignatures)) classification = sameTree ? "same_tree_content_variability" : "content_change";
+    else if (repeatedContentFailure) classification = "repeated_content_failure";
+    else if (!allEqual(providerSignatures)) classification = "provider_route_variability";
+    else if (!allEqual(latencySignatures)) classification = "latency_variability";
+
+    return {
+      case: caseId,
+      classification,
+      same_tree: sameTree,
+      runs: matches.length,
+      baseline_signature: fingerprints[0].diagnosticSignature,
+      latest_signature: fingerprints.at(-1).diagnosticSignature,
+    };
+  });
+}
+
+/** Per-run answer diagnostics for one case; exported for focused offline investigation. */
+export function buildAnswerQualityCaseTrend(payloads, caseId) {
+  return payloads.map(({ label, payload }) => {
+    const match = answerResults(payload).find((result) => result.id === caseId);
+    const fingerprints = match ? answerFingerprints(match) : null;
+    return {
+      label,
+      git_sha: payload?.run_context?.git_sha ?? null,
+      found: Boolean(match),
+      passed: match ? (match.failures?.length ?? 0) === 0 : null,
+      route: match?.route ?? null,
+      latency_route: match?.latencyRoute ?? null,
+      total_ms: match?.latencyMs ?? null,
+      diagnostic_signature: fingerprints?.diagnosticSignature ?? null,
+    };
+  });
+}
+
 function formatTable(rows) {
   if (!rows.length) return "(no rows)";
   const keys = Object.keys(rows[0]);
@@ -61,14 +209,23 @@ function formatTable(rows) {
 
 function main() {
   const argv = process.argv.slice(2);
+  const answerQuality = argv.includes("--answer-quality");
   const caseFlag = argv.indexOf("--case");
   const caseId = caseFlag >= 0 ? argv[caseFlag + 1] : undefined;
-  const files = argv.filter((arg, index) => arg !== "--case" && index !== caseFlag + 1);
+  const files = argv.filter((arg, index) => arg !== "--case" && arg !== "--answer-quality" && index !== caseFlag + 1);
   if (!files.length) {
-    console.error("Usage: eval-trend [--case <golden-case-id>] <golden-retrieval.json...>  (oldest first)");
+    console.error("Usage: eval-trend [--answer-quality] [--case <case-id>] <report.json...>  (oldest first)");
     process.exit(2);
   }
   const payloads = files.map((file) => ({ label: file, payload: JSON.parse(readFileSync(file, "utf8")) }));
+  if (answerQuality) {
+    console.log(formatTable(buildAnswerQualityVariabilityRows(payloads)));
+    if (caseId) {
+      console.log(`\nAnswer case trend: ${caseId}`);
+      console.log(formatTable(buildAnswerQualityCaseTrend(payloads, caseId)));
+    }
+    return;
+  }
   console.log(formatTable(buildTrendRows(payloads)));
   if (caseId) {
     console.log(`\nCase trend: ${caseId}`);

tests/eval-canary-workflow.test.ts

@@ -18,6 +18,11 @@ describe("eval canary workflow input", () => {
     expect(workflow).not.toMatch(/run:.*github\.event\.inputs\.answer_case_limit/);
   });
 
+  it("records the actual checked-out tree for optional-ref comparisons", () => {
+    expect(workflow).toContain("ref: ${{ github.event.inputs.ref || '' }}");
+    expect(workflow).toContain('echo "EVAL_GIT_SHA=$(git rev-parse HEAD)" >> "$GITHUB_ENV"');
+  });
+
   it("distinguishes provider outages from retrieval regressions in the failure issue", () => {
     expect(workflow).toContain('title: "Eval canary failure: weekly evaluation did not complete"');
     expect(workflow).toContain("Resolve provider quota/auth/config failures before rerunning");
@@ -31,6 +36,7 @@ describe("eval canary workflow input", () => {
     expect(workflow).toContain("set -o pipefail");
     expect(workflow).toContain("tee .local/eval-canary/golden-retrieval.log");
     expect(workflow).toContain("tee .local/eval-canary/answer-quality.log");
+    expect(workflow).toContain("--output-dir .local/eval-canary/quality-reports");
     expect(workflow).toContain(
       "await import(pathToFileURL(`${process.env.GITHUB_WORKSPACE}/scripts/productivity-core.mjs`).href)",
     );

tests/eval-quality.test.ts

@@ -4,6 +4,7 @@ import {
   buildEvalQualityReport,
   configureEvalProviderEnvironment,
   deliveredGroundedAfterSourceGovernancePolicy,
+  evalQualityRunContext,
   qualityFailureCategory,
   ragAnswerTimingDiagnostics,
   renderEvalQualityMarkdown,
@@ -14,6 +15,31 @@ import {
 } from "../scripts/eval-quality";
 import { evaluateGoldenRetrievalCase, type GoldenRetrievalResult } from "../scripts/eval-retrieval";
 
+describe("eval quality run context", () => {
+  it("records stable run identity without requiring GitHub Actions", () => {
+    expect(
+      evalQualityRunContext({
+        EVAL_GIT_SHA: " candidate-sha ",
+        GITHUB_SHA: "ignored-fallback",
+        GITHUB_RUN_ID: "123",
+        GITHUB_RUN_ATTEMPT: "2",
+        EVAL_LATENCY_CONTEXT: "cross-region-runner",
+      }),
+    ).toEqual({
+      git_sha: "candidate-sha",
+      github_run_id: "123",
+      github_run_attempt: "2",
+      latency_context: "cross-region-runner",
+    });
+    expect(evalQualityRunContext({})).toEqual({
+      git_sha: null,
+      github_run_id: null,
+      github_run_attempt: null,
+      latency_context: "default",
+    });
+  });
+});
+
 function retrievalResult(overrides: Partial<GoldenRetrievalResult> = {}): GoldenRetrievalResult {
   const base: GoldenRetrievalResult = {
     id: "retrieval-1",