From 0874517903fcb567c5be57f5e134cd2a9596869e Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Tue, 14 Jul 2026 12:08:53 -0700 Subject: [PATCH] fix(drift): stop gemini-interactions provider-mode false positive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The daily Drift Tests / Fix Drift jobs failed every run because the Gemini model-availability check scrapes `gemini-*` tokens from source files (including README.md) with a greedy regex, then verifies each against Google's live model list. Commit 4753feb documented the aimock `gemini-interactions` provider mode in README.md as a bare token; the scraper grabbed it as if it were a Gemini model id, the availability check failed (it is not a real Google model), and drift-report-collector.ts crashed as an "unparseable" failure — firing a daily false-positive "providers changed response formats" Slack alert with no real drift. Fix (two layers of defense): - Extend the Gemini stable filter to exclude aimock provider-mode names via an explicit AIMOCK_GEMINI_PROVIDER_MODES set, so a scraped provider-mode token is never drift-checked (the real fix). - Reword the README so the provider mode is no longer written as a bare model-id token, so the greedy scraper does not pick it up in the first place. Extract scrapeModels / GEMINI_MODEL_PATTERN / sourceFiles / filterStableGeminiModels as exports and add models-scrape.drift.ts, a regression guard that exercises the real scrape + filter surface (no fakes) so a future doc mention re-introducing this false positive is caught here instead of in the live nightly job. --- README.md | 2 +- src/__tests__/drift/models-scrape.drift.ts | 48 ++++++++++++++++++++++ src/__tests__/drift/models.drift.ts | 43 +++++++++++++++---- 3 files changed, 84 insertions(+), 9 deletions(-) create mode 100644 src/__tests__/drift/models-scrape.drift.ts diff --git a/README.md b/README.md index 410cd936..687ca280 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ In record or `--proxy-only` mode, aimock forwards the caller's auth header to th | `AIMOCK_PROVIDER_ELEVENLABS_KEY` | ElevenLabs | `xi-api-key: ` | | `AIMOCK_PROVIDER_FAL_KEY` | fal.ai | `Authorization: Key ` | -`gemini-interactions` reuses `AIMOCK_PROVIDER_GEMINI_KEY` (same upstream API as Gemini). An empty-string value is treated as unset. +The Gemini interactions provider mode reuses `AIMOCK_PROVIDER_GEMINI_KEY` (same upstream API as Gemini). An empty-string value is treated as unset. This is opt-in and backward-compatible: with no key configured the feature is inert and the caller's header passes through as-is. Injection fires only when the caller sent no credential **or** a dummy credential prefixed with `sk-aimock-` (overridable via `AIMOCK_DUMMY_KEY_MARKER`); a real caller key never starting with that marker is always forwarded verbatim, so the caller overrides aimock. Signed and exchanged credentials — AWS Bedrock (SigV4) and Vertex AI (OAuth) — are never rewritten and always forwarded unchanged. (Azure's static `api-key` is injected; a real Microsoft Entra ID `Authorization: Bearer` token from the caller is never dummy-prefixed, so it too passes through verbatim.) diff --git a/src/__tests__/drift/models-scrape.drift.ts b/src/__tests__/drift/models-scrape.drift.ts new file mode 100644 index 00000000..d6b00896 --- /dev/null +++ b/src/__tests__/drift/models-scrape.drift.ts @@ -0,0 +1,48 @@ +/** + * Regression guard for the model-id scraper used by the drift checks. + * + * The Gemini availability check scrapes `gemini-*` tokens from source files — + * including README.md — with a greedy regex, then checks each against Google's + * live model list. aimock also has "provider modes" such as `gemini-interactions` + * that route to the Gemini upstream API but are NOT Gemini model ids. When the + * README documented such a mode as a bare `gemini-interactions` token, the + * greedy scraper grabbed it, the availability check failed (it is not a real + * Google model), and the drift collector crashed as an "unparseable" failure — + * firing a daily false-positive drift alert with no real drift. + * + * Two layers of defense are guarded here against the REAL scrape + filter + * surface (real README, real regex, real filter): + * 1. The stable filter drops aimock provider-mode names even if scraped. + * 2. The README no longer spells the mode as a bare model-id token, so the + * scraper never picks it up in the first place. + */ + +import { describe, it, expect } from "vitest"; +import { + scrapeModels, + filterStableGeminiModels, + sourceFiles, + GEMINI_MODEL_PATTERN, +} from "./models.drift.js"; + +describe("Gemini model scrape does not flag aimock provider-mode names", () => { + it("stable filter drops the gemini-interactions provider-mode token", () => { + // Layer 1 (the real fix): even if a `gemini-interactions` token is scraped + // from anywhere, the stable filter must exclude it so it is never checked + // for drift. This is independent of any doc wording. + const stable = filterStableGeminiModels([ + "gemini-2.5-flash", + "gemini-interactions", + "gemini-1.5-pro", + ]); + expect(stable).toEqual(["gemini-2.5-flash", "gemini-1.5-pro"]); + }); + + it("real source-file scrape produces no provider-mode false positives", () => { + // Layer 2: run the exact scrape + filter the drift check uses over the + // real source files. The result must contain no aimock provider-mode names. + const referenced = scrapeModels(GEMINI_MODEL_PATTERN, sourceFiles); + const stable = filterStableGeminiModels(referenced); + expect(stable).not.toContain("gemini-interactions"); + }); +}); diff --git a/src/__tests__/drift/models.drift.ts b/src/__tests__/drift/models.drift.ts index 73e4f4e0..1e5fa556 100644 --- a/src/__tests__/drift/models.drift.ts +++ b/src/__tests__/drift/models.drift.ts @@ -14,7 +14,7 @@ import { listOpenAIModels, listAnthropicModels, listGeminiModels } from "./provi const PROJECT_ROOT = path.resolve(import.meta.dirname, "..", "..", ".."); -function scrapeModels(pattern: RegExp, files: string[]): string[] { +export function scrapeModels(pattern: RegExp, files: string[]): string[] { const models = new Set(); for (const file of files) { const filePath = path.join(PROJECT_ROOT, file); @@ -29,7 +29,7 @@ function scrapeModels(pattern: RegExp, files: string[]): string[] { return [...models]; } -const sourceFiles = [ +export const sourceFiles = [ "src/__tests__/api-conformance.test.ts", "src/__tests__/ws-api-conformance.test.ts", "README.md", @@ -38,6 +38,35 @@ const sourceFiles = [ "fixtures/example-tool-call.json", ]; +// Regex used to scrape Gemini model ids from the source files above. Greedy +// on purpose so we catch versioned/dated ids (e.g. gemini-2.5-flash), but that +// greed also grabs any `gemini-*` token appearing in prose — see the stable +// filter below for what gets excluded. +export const GEMINI_MODEL_PATTERN = /\b(gemini-(?:[\w.-]+))\b/g; + +// aimock exposes "provider modes" — internal names that route to a real +// upstream API but are NOT themselves model ids exposed by that provider. The +// README documents them (e.g. `gemini-interactions` reuses the Gemini upstream +// key), so the greedy scraper above grabs them as if they were Gemini models. +// They will never appear in Google's model list, so checking them for drift is +// a guaranteed false positive. Exclude them explicitly. +const AIMOCK_GEMINI_PROVIDER_MODES = new Set(["gemini-interactions"]); + +// Narrow a raw scrape of `gemini-*` tokens down to real, checkable model ids by +// dropping (a) experimental/live/preview ids, (b) markdown anchor-link +// fragments, and (c) aimock provider-mode names that are documentation prose, +// not provider model ids. Exported so the regression suite can exercise the +// exact filtering the drift check relies on. +export function filterStableGeminiModels(referenced: string[]): string[] { + return referenced.filter( + (m) => + !m.includes("-exp") && + !m.includes("-live") && + !m.includes("bidigeneratecontent") && + !AIMOCK_GEMINI_PROVIDER_MODES.has(m), + ); +} + // --------------------------------------------------------------------------- // OpenAI // --------------------------------------------------------------------------- @@ -85,15 +114,13 @@ describe.skipIf(!process.env.ANTHROPIC_API_KEY)("Anthropic model availability", describe.skipIf(!process.env.GOOGLE_API_KEY)("Gemini model availability", () => { it("models used in aimock tests are still available", async () => { const models = await listGeminiModels(process.env.GOOGLE_API_KEY!); - const referenced = scrapeModels(/\b(gemini-(?:[\w.-]+))\b/g, sourceFiles); + const referenced = scrapeModels(GEMINI_MODEL_PATTERN, sourceFiles); if (referenced.length === 0) return; - // Skip experimental models, live-only models, and anchor-link fragments - // scraped from markdown (e.g., "gemini-live-bidigeneratecontent") - const stable = referenced.filter( - (m) => !m.includes("-exp") && !m.includes("-live") && !m.includes("bidigeneratecontent"), - ); + // Drop experimental/live ids, markdown anchor fragments, and aimock + // provider-mode names (see filterStableGeminiModels). + const stable = filterStableGeminiModels(referenced); for (const m of stable) { const found = models.some((available) => available === m || available.startsWith(m));