From 6adf2c333e5b550e60a75786b70e58f6ab325081 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 24 Jul 2026 06:43:41 +0800 Subject: [PATCH] fix: populate canary source governance report --- .github/workflows/eval-canary.yml | 2 +- docs/observability-slos.md | 12 +++++---- docs/outstanding-issues.md | 2 +- scripts/eval-quality.ts | 39 +++++++++++++++++++++++++++++- tests/eval-canary-workflow.test.ts | 1 + tests/eval-quality.test.ts | 37 ++++++++++++++++++++++++++++ 6 files changed, 85 insertions(+), 8 deletions(-) diff --git a/.github/workflows/eval-canary.yml b/.github/workflows/eval-canary.yml index 238a2404..6f5f7b12 100644 --- a/.github/workflows/eval-canary.yml +++ b/.github/workflows/eval-canary.yml @@ -173,7 +173,7 @@ jobs: fi mkdir -p .local/eval-canary set -o pipefail - npm run eval:quality -- --rag-only --limit "$ANSWER_CASE_LIMIT" --output-dir .local/eval-canary/quality-reports --fail-on-threshold 2>&1 | tee .local/eval-canary/answer-quality.log + npm run eval:quality -- --rag-only --limit "$ANSWER_CASE_LIMIT" --source-governance-results .local/eval-canary/golden-retrieval.json --output-dir .local/eval-canary/quality-reports --fail-on-threshold 2>&1 | tee .local/eval-canary/answer-quality.log # Phase E baseline/paired-run instrument: the five answer-quality metrics (relevance, # readability, artifact leaks, intent coverage, fail-closed) + per-intent structural diff --git a/docs/observability-slos.md b/docs/observability-slos.md index f5b9a006..a53b3874 100644 --- a/docs/observability-slos.md +++ b/docs/observability-slos.md @@ -140,11 +140,13 @@ What it does, in order: corpus. This is the eval CI never runs on PRs (it needs live Supabase + OpenAI keys); the canary makes it a standing weekly guard instead of a manual pre-merge step that can be skipped. -3. `npm run eval:quality -- --rag-only --limit 44 --fail-on-threshold` — the - full committed answer-quality set (grounding, citations, - unsupported-correctness). The workflow writes the structured JSON and - Markdown reports into the `eval-canary-output` artifact with the tested Git - SHA and run identity. +3. The answer-quality step runs `eval:quality` with `--rag-only --limit 44`, + supplies `.local/eval-canary/golden-retrieval.json` through + `--source-governance-results`, and enables `--fail-on-threshold`. The + separate retrieval artifact populates the Source Governance table without + rerunning retrieval or changing answer-gate thresholds. The workflow writes + the structured JSON and Markdown reports into the `eval-canary-output` + artifact with the tested Git SHA and run identity. Failing loudly: diff --git a/docs/outstanding-issues.md b/docs/outstanding-issues.md index dff179da..1d2db23a 100644 --- a/docs/outstanding-issues.md +++ b/docs/outstanding-issues.md @@ -61,7 +61,6 @@ Durable, cross-session memory of everything still outstanding for this repo: ope | #028 | P3 | rec | Runtime error tracking (Sentry or similar) | No error tracking in the repo — production exceptions on `psychiatry.tools`, including how often `RAG_PROVIDER_MODE=auto` silently degrades to source-only, are invisible. Weigh adding `@sentry/nextjs` (dependency + DSN secret + instrumentation) vs cost; alert → chat/issue. Provider-backed; needs explicit sign-off before adding the dependency. | session 2026-07-22 webhook review | 2026-07-22 | | #029 | P2 | issue | 12 of 30 answer-quality cases return the fallback stub | run #61 --dump-answers: 12/30 quality cases emit the source_backed_review_fallback boilerplate with answer_sections: [], all grounded with 4-6 citations. Some still PASS targeting because the stub echoes query keywords (the contraindication/document_lookup matchers need only a keyword), so the targeting metric MASKS the problem for those intents. Superset of #018 — fix in the extractive composer, validate with the provider-backed answer eval. | run #61 dump artifact; session 2026-07-22 | 2026-07-22 | | #030 | P3 | issue | Wide-tier alias lets one doc satisfy both comparison slots | In src/lib/eval-document-matching.ts, "Admission to Discharge for Mental Health Inpatients" appears in BOTH the AdmissionCommunityPts and Discharge alias lists, so a single document can satisfy both expectedFiles slots and make allHit true — a latent false-pass on admission-discharge cases. Not firing today (that doc is not in the failing top-5) but it would mask a real miss. Tighten the tables so one doc cannot fill both sides. | src/lib/eval-document-matching.ts:32-65; session 2026-07-22 | 2026-07-22 | -| #031 | P3 | issue | Canary Source Governance table reports all zeros | Reconfirmed on run `30018289898`: the Answer Metrics section reported source-governance warning rate 0.8409, while the Source Governance table still reported zero top results and zero for every status bucket. Root boundary: the workflow runs `eval:quality --rag-only` separately from the golden retrieval command, so `topResultGovernanceCounts` receives no retrieval results even though `golden-retrieval.json` contains the metadata. Wire that artifact into the answer report rather than duplicating or recomputing retrieval. | runs #61 and `30018289898`; `scripts/eval-quality.ts` | 2026-07-22 | | #032 | P3 | rec | Governance ranking weighting: REFUTED, not debt | The source-governance audit (PR #1051) flagged three "gaps": `review_due` carries no ranking penalty, `unknownCurrentnessPenalty` ships at 0, and `selectBestSourceRecommendation` ignores governance metadata. **These are deliberate, measured decisions — do NOT implement them as written.** Blanket metadata boosts/penalties in selection ordering were measured on 2026-07-02 to regress the golden retrieval eval to 16/23 (doc-recall@5 1.0→0.76, mrr 0.75→0.64). Two corpus facts make it unsafe: scores saturate at the clamp so stacked boosts fully override lexical relevance, and the corpus is only partially metadata-enriched while `normalizeSourceMetadata` coerces unenriched docs to `unknown`/`unverified` — so "unknown" ≠ "bad" and blanket weighting swings ranking approx. 0.35 for reasons unrelated to relevance. Even governance-as-tiebreak buried correct unenriched docs (3 designs bisected). Next action: none — treat as a guardrail. If ever revisited, RC8 (source-strength as a _filter_) is the tracked path, gated on `eval:retrieval:quality` 36/36 plus a live canary pair. | PR #118; `docs/rag-behaviour/refuted-approaches.md`; PR #1051 items 4/5/6 | 2026-07-22 | | #033 | P3 | rec | Source governance metadata absent from the LLM prompt | `buildRagSourceBlock` omits `document_status`, `clinical_validation_status`, and `extraction_quality`, so the model cannot self-caveat during generation and governance is enforced only post-hoc. Generation-surface change: needs `eval:rag` plus `eval:quality --rag-only` (grounded-supported must not drop, citation-failure 0) and explicit approval. Carries the same "unknown ≠ bad" hazard as #032 — on a partially-enriched corpus the model would likely over-caveat correct sources, so design the prompt wording before spending an eval. | `src/lib/rag/rag-source-block.ts:126-198`; PR #1051 audit item 8 | 2026-07-22 | | #034 | P3 | issue | Answer cache can serve stale governance metadata | `cacheIndexingVersion` derives the version from `updated_at` / `indexed_at` / `index_generation_id`, so a metadata-only `document_status` flip that bumps none of those is invisible to the passive guard. **Already mitigated**: every known status-write path calls `invalidateRagCachesForOwner` or `invalidateRagCachesForDocumentMutation`. Residual risk only — a future write path that omits the invalidator would serve stale governance until TTL. Next action: add a regression test pinning the invalidator call on status-mutating routes (cheaper and safer than touching the protected cache key). | `src/lib/rag/rag-cache.ts:382-438`; PR #1051 audit item 10 | 2026-07-22 | @@ -79,6 +78,7 @@ Move resolved rows here with the resolution date and a one-line outcome. Keep th | ID | Type | Summary | Outcome | Resolved | | ---- | ----- | ---------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | +| #031 | issue | Populate canary Source Governance table | The answer-quality step now consumes the preceding `golden-retrieval.json` only for source-governance reporting. Offline replay of run `30018289898` populated 338 top results, including 202 review-required entries, while retaining zero retrieval cases and no additional threshold failures. Retrieval and ranking behavior are unchanged. | 2026-07-24 | | #020 | task | Validate eval:quality cost readout post-fix | Confirmed on merged-main canary run `30018289898`: Answer Metrics reported 9 nonzero-cost cases and an estimated answer cost of `$0.234736`; the structured report retained the same value. The PR #1050 estimator fix is operationally proven. | 2026-07-23 | | #003 | task | Staging tenancy release evidence outstanding | Ran GitHub Action and validated isolation | 2026-07-21 | | #002 | task | Process-ownership fix not yet isolated on `main` | Fixed process isolation using child.pid termination | 2026-07-21 | diff --git a/scripts/eval-quality.ts b/scripts/eval-quality.ts index c0ace8d1..148ddb59 100644 --- a/scripts/eval-quality.ts +++ b/scripts/eval-quality.ts @@ -38,6 +38,7 @@ export type EvalQualityProviderMode = "openai" | "offline"; type EvalQualityArgs = { fixture: string; + sourceGovernanceResults?: string; ownerEmail?: string; ownerId?: string; limit?: number; @@ -293,6 +294,7 @@ function parseArgs(argv: string[]): EvalQualityArgs { index += 1; if (token === "--fixture") args.fixture = value; + if (token === "--source-governance-results") args.sourceGovernanceResults = value; if (token === "--owner-email") args.ownerEmail = value; if (token === "--owner-id") args.ownerId = value; if (token === "--limit") args.limit = Number.parseInt(value, 10); @@ -597,6 +599,7 @@ function summarizeRagQualityResults(results: RagQualityResult[], providerMode: E export function buildEvalQualityReport(args: { generatedAt?: string; retrievalResults: GoldenRetrievalResult[]; + sourceGovernanceResults?: GoldenRetrievalResult[]; ragResults: RagQualityResult[]; sourceMetadataDebtAcceptance?: SourceMetadataDebtAcceptance; providerMode?: EvalQualityProviderMode; @@ -604,7 +607,11 @@ export function buildEvalQualityReport(args: { const providerMode = args.providerMode ?? "openai"; const retrievalSummary = summarizeGoldenRetrievalResults(args.retrievalResults); const ragSummary = summarizeRagQualityResults(args.ragResults, providerMode); - const governance = topResultGovernanceCounts(args.retrievalResults); + // `--rag-only` intentionally leaves retrieval metrics and gates empty. The + // canary can still supply the preceding golden-retrieval artifact so this + // report renders its source-governance metadata without rerunning retrieval + // or changing the answer gate's pass/fail contract. + const governance = topResultGovernanceCounts(args.sourceGovernanceResults ?? args.retrievalResults); const thresholdFailures: string[] = []; const providerEvidence = { mode: providerMode, @@ -1195,6 +1202,32 @@ function asRecord(value: unknown, label: string) { return value as Record; } +export function sourceGovernanceResultsFromArtifact(value: unknown): GoldenRetrievalResult[] { + const artifact = asRecord(value, "source governance results artifact"); + if (!Array.isArray(artifact.results)) { + throw new Error("source governance results artifact results must be an array."); + } + + return artifact.results.map((result, resultIndex) => { + const record = asRecord(result, `source governance results[${resultIndex}]`); + if (typeof record.id !== "string" || !record.id.trim()) { + throw new Error(`source governance results[${resultIndex}] id must be a non-empty string.`); + } + if (!Array.isArray(record.topResults)) { + throw new Error(`source governance results[${resultIndex}] topResults must be an array.`); + } + record.topResults.forEach((topResult, topResultIndex) => { + asRecord(topResult, `source governance results[${resultIndex}].topResults[${topResultIndex}]`); + }); + return record as unknown as GoldenRetrievalResult; + }); +} + +async function loadSourceGovernanceResults(path: string) { + const parsed = JSON.parse(await readFile(path, "utf8")) as unknown; + return sourceGovernanceResultsFromArtifact(parsed); +} + function requiredString(record: Record, key: string) { const value = record[key]; if (typeof value !== "string" || !value.trim()) throw new Error(`source metadata debt ${key} is required.`); @@ -1256,12 +1289,16 @@ async function main() { const sourceMetadataDebtAcceptance = args.sourceMetadataDebt ? await loadSourceMetadataDebtAcceptance(args.sourceMetadataDebt) : undefined; + const sourceGovernanceResults = args.sourceGovernanceResults + ? await loadSourceGovernanceResults(args.sourceGovernanceResults) + : undefined; const ownerId = await resolveEvalOwnerId(supabase, args); const retrievalResults = args.ragOnly ? [] : await runRetrievalQualityCases({ ...args, ownerId, supabase }); const ragResults = args.retrievalOnly ? [] : await runRagQualityCases({ ...args, ownerId, supabase }); const report = buildEvalQualityReport({ retrievalResults, + sourceGovernanceResults, ragResults, sourceMetadataDebtAcceptance, providerMode: args.providerMode, diff --git a/tests/eval-canary-workflow.test.ts b/tests/eval-canary-workflow.test.ts index ecedea3d..bc38b8c4 100644 --- a/tests/eval-canary-workflow.test.ts +++ b/tests/eval-canary-workflow.test.ts @@ -37,6 +37,7 @@ describe("eval canary workflow input", () => { expect(workflow).toContain("tee .local/eval-canary/golden-retrieval.log"); expect(workflow).toContain("tee .local/eval-canary/answer-quality.log"); expect(workflow).toContain("--output-dir .local/eval-canary/quality-reports"); + expect(workflow).toContain("--source-governance-results .local/eval-canary/golden-retrieval.json"); expect(workflow).toContain( "await import(pathToFileURL(`${process.env.GITHUB_WORKSPACE}/scripts/productivity-core.mjs`).href)", ); diff --git a/tests/eval-quality.test.ts b/tests/eval-quality.test.ts index 7375b0eb..3379ed15 100644 --- a/tests/eval-quality.test.ts +++ b/tests/eval-quality.test.ts @@ -9,6 +9,7 @@ import { ragAnswerTimingDiagnostics, renderEvalQualityMarkdown, retrievalCasesForProviderMode, + sourceGovernanceResultsFromArtifact, sourceGovernanceDangerFailuresForAnswer, sourceWarningsForRagQualityAnswer, type RagQualityResult, @@ -256,6 +257,42 @@ describe("eval quality reporting", () => { ); }); + it("reports governance from a separate retrieval artifact without enabling retrieval gates", () => { + const governanceResult = retrievalResult({ + topResults: [ + { + ...retrievalResult().topResults[0], + document_status: "outdated", + clinical_validation_status: "unverified", + extraction_quality: "poor", + }, + ], + }); + const parsedResults = sourceGovernanceResultsFromArtifact({ results: [governanceResult] }); + const report = buildEvalQualityReport({ + generatedAt: "2026-07-24T00:00:00.000Z", + retrievalResults: [], + sourceGovernanceResults: parsedResults, + ragResults: [], + }); + + expect(report.retrieval.summary.case_count).toBe(0); + expect(report.retrieval.source_governance).toMatchObject({ + total_top_results: 1, + stale_top_results: 1, + unverified_top_results: 1, + poor_extraction_top_results: 1, + review_required_top_results: 1, + }); + expect(report.threshold_failures).not.toEqual(expect.arrayContaining([expect.stringContaining("top-result")])); + }); + + it("rejects a malformed source-governance retrieval artifact", () => { + expect(() => sourceGovernanceResultsFromArtifact({ results: [{ id: "missing-top-results" }] })).toThrow( + "topResults must be an array", + ); + }); + it("counts an acceptSourceOnly source-only answer as grounded-supported only when expected docs are cited", () => { const acceptedSourceOnly = buildEvalQualityReport({ generatedAt: "2026-07-13T00:00:00.000Z",