diff --git a/scripts/eval-trend.mjs b/scripts/eval-trend.mjs index 5535e9dd..d40d32b2 100644 --- a/scripts/eval-trend.mjs +++ b/scripts/eval-trend.mjs @@ -158,12 +158,22 @@ export function buildAnswerQualityVariabilityRows(payloads) { const providerSignatures = fingerprints.map(({ provider }) => stableSignature(provider)); const latencySignatures = fingerprints.map(({ latency }) => stableSignature(latency)); const repeatedContentFailure = - allEqual(contentSignatures) && fingerprints.every(({ content }) => content.categories.length > 0); - let classification = "stable"; - if (!allEqual(contentSignatures)) classification = sameTree ? "same_tree_content_variability" : "content_change"; - else if (repeatedContentFailure) classification = "repeated_content_failure"; - else if (!allEqual(providerSignatures)) classification = "provider_route_variability"; - else if (!allEqual(latencySignatures)) classification = "latency_variability"; + matches.length > 1 && + allEqual(contentSignatures) && + fingerprints.every(({ content }) => content.categories.length > 0); + let classification = + matches.length === 1 + ? fingerprints[0].content.categories.length > 0 + ? "observed_content_failure" + : fingerprints[0].latency.latencyFailure || fingerprints[0].latency.routeCeilingExceeded + ? "observed_latency_failure" + : "single_run" + : "stable"; + if (matches.length > 1 && !allEqual(contentSignatures)) { + classification = sameTree ? "same_tree_content_variability" : "content_change"; + } else if (repeatedContentFailure) classification = "repeated_content_failure"; + else if (matches.length > 1 && !allEqual(providerSignatures)) classification = "provider_route_variability"; + else if (matches.length > 1 && !allEqual(latencySignatures)) classification = "latency_variability"; return { case: caseId, diff --git a/tests/eval-trend.test.ts b/tests/eval-trend.test.ts index 166d7517..57171c0e 100644 --- a/tests/eval-trend.test.ts +++ b/tests/eval-trend.test.ts @@ -97,6 +97,18 @@ const answerReport = (label: string, sha: string, result: Record { + it("does not call a single observed failure repeated", () => { + const failed = answerResult({ failures: ["citation count below required minimum"], citations: 1 }); + expect(buildAnswerQualityVariabilityRows([answerReport("first-run", "same-sha", failed)])[0]).toMatchObject({ + classification: "observed_content_failure", + same_tree: false, + runs: 1, + }); + expect(buildAnswerQualityVariabilityRows([answerReport("first-run", "same-sha", answerResult())])[0]).toMatchObject( + { classification: "single_run", runs: 1 }, + ); + }); + it("marks pass-to-content-failure changes on the same tree as variability, not a deterministic regression", () => { const rows = buildAnswerQualityVariabilityRows([ answerReport("baseline", "same-sha", answerResult()),