fix(drift): stop gemini-interactions provider-mode false positive

README.md

@@ -145,7 +145,7 @@ In record or `--proxy-only` mode, aimock forwards the caller's auth header to th
 | `AIMOCK_PROVIDER_ELEVENLABS_KEY` | ElevenLabs                       | `xi-api-key: <key>`           |
 | `AIMOCK_PROVIDER_FAL_KEY`        | fal.ai                           | `Authorization: Key <key>`    |
 
-`gemini-interactions` reuses `AIMOCK_PROVIDER_GEMINI_KEY` (same upstream API as Gemini). An empty-string value is treated as unset.
+The Gemini interactions provider mode reuses `AIMOCK_PROVIDER_GEMINI_KEY` (same upstream API as Gemini). An empty-string value is treated as unset.
 
 This is opt-in and backward-compatible: with no key configured the feature is inert and the caller's header passes through as-is. Injection fires only when the caller sent no credential **or** a dummy credential prefixed with `sk-aimock-` (overridable via `AIMOCK_DUMMY_KEY_MARKER`); a real caller key never starting with that marker is always forwarded verbatim, so the caller overrides aimock. Signed and exchanged credentials — AWS Bedrock (SigV4) and Vertex AI (OAuth) — are never rewritten and always forwarded unchanged. (Azure's static `api-key` is injected; a real Microsoft Entra ID `Authorization: Bearer` token from the caller is never dummy-prefixed, so it too passes through verbatim.)
 

src/__tests__/drift/models-scrape.drift.ts

@@ -0,0 +1,48 @@
+/**
+ * Regression guard for the model-id scraper used by the drift checks.
+ *
+ * The Gemini availability check scrapes `gemini-*` tokens from source files —
+ * including README.md — with a greedy regex, then checks each against Google's
+ * live model list. aimock also has "provider modes" such as `gemini-interactions`
+ * that route to the Gemini upstream API but are NOT Gemini model ids. When the
+ * README documented such a mode as a bare `gemini-interactions` token, the
+ * greedy scraper grabbed it, the availability check failed (it is not a real
+ * Google model), and the drift collector crashed as an "unparseable" failure —
+ * firing a daily false-positive drift alert with no real drift.
+ *
+ * Two layers of defense are guarded here against the REAL scrape + filter
+ * surface (real README, real regex, real filter):
+ *   1. The stable filter drops aimock provider-mode names even if scraped.
+ *   2. The README no longer spells the mode as a bare model-id token, so the
+ *      scraper never picks it up in the first place.
+ */
+
+import { describe, it, expect } from "vitest";
+import {
+  scrapeModels,
+  filterStableGeminiModels,
+  sourceFiles,
+  GEMINI_MODEL_PATTERN,
+} from "./models.drift.js";
+
+describe("Gemini model scrape does not flag aimock provider-mode names", () => {
+  it("stable filter drops the gemini-interactions provider-mode token", () => {
+    // Layer 1 (the real fix): even if a `gemini-interactions` token is scraped
+    // from anywhere, the stable filter must exclude it so it is never checked
+    // for drift. This is independent of any doc wording.
+    const stable = filterStableGeminiModels([
+      "gemini-2.5-flash",
+      "gemini-interactions",
+      "gemini-1.5-pro",
+    ]);
+    expect(stable).toEqual(["gemini-2.5-flash", "gemini-1.5-pro"]);
+  });
+
+  it("real source-file scrape produces no provider-mode false positives", () => {
+    // Layer 2: run the exact scrape + filter the drift check uses over the
+    // real source files. The result must contain no aimock provider-mode names.
+    const referenced = scrapeModels(GEMINI_MODEL_PATTERN, sourceFiles);
+    const stable = filterStableGeminiModels(referenced);
+    expect(stable).not.toContain("gemini-interactions");
+  });
+});

src/__tests__/drift/models.drift.ts

@@ -14,7 +14,7 @@ import { listOpenAIModels, listAnthropicModels, listGeminiModels } from "./provi
 
 const PROJECT_ROOT = path.resolve(import.meta.dirname, "..", "..", "..");
 
-function scrapeModels(pattern: RegExp, files: string[]): string[] {
+export function scrapeModels(pattern: RegExp, files: string[]): string[] {
   const models = new Set<string>();
   for (const file of files) {
     const filePath = path.join(PROJECT_ROOT, file);
@@ -29,7 +29,7 @@ function scrapeModels(pattern: RegExp, files: string[]): string[] {
   return [...models];
 }
 
-const sourceFiles = [
+export const sourceFiles = [
   "src/__tests__/api-conformance.test.ts",
   "src/__tests__/ws-api-conformance.test.ts",
   "README.md",
@@ -38,6 +38,35 @@ const sourceFiles = [
   "fixtures/example-tool-call.json",
 ];
 
+// Regex used to scrape Gemini model ids from the source files above. Greedy
+// on purpose so we catch versioned/dated ids (e.g. gemini-2.5-flash), but that
+// greed also grabs any `gemini-*` token appearing in prose — see the stable
+// filter below for what gets excluded.
+export const GEMINI_MODEL_PATTERN = /\b(gemini-(?:[\w.-]+))\b/g;
+
+// aimock exposes "provider modes" — internal names that route to a real
+// upstream API but are NOT themselves model ids exposed by that provider. The
+// README documents them (e.g. `gemini-interactions` reuses the Gemini upstream
+// key), so the greedy scraper above grabs them as if they were Gemini models.
+// They will never appear in Google's model list, so checking them for drift is
+// a guaranteed false positive. Exclude them explicitly.
+const AIMOCK_GEMINI_PROVIDER_MODES = new Set(["gemini-interactions"]);
+
+// Narrow a raw scrape of `gemini-*` tokens down to real, checkable model ids by
+// dropping (a) experimental/live/preview ids, (b) markdown anchor-link
+// fragments, and (c) aimock provider-mode names that are documentation prose,
+// not provider model ids. Exported so the regression suite can exercise the
+// exact filtering the drift check relies on.
+export function filterStableGeminiModels(referenced: string[]): string[] {
+  return referenced.filter(
+    (m) =>
+      !m.includes("-exp") &&
+      !m.includes("-live") &&
+      !m.includes("bidigeneratecontent") &&
+      !AIMOCK_GEMINI_PROVIDER_MODES.has(m),
+  );
+}
+
 // ---------------------------------------------------------------------------
 // OpenAI
 // ---------------------------------------------------------------------------
@@ -85,15 +114,13 @@ describe.skipIf(!process.env.ANTHROPIC_API_KEY)("Anthropic model availability",
 describe.skipIf(!process.env.GOOGLE_API_KEY)("Gemini model availability", () => {
   it("models used in aimock tests are still available", async () => {
     const models = await listGeminiModels(process.env.GOOGLE_API_KEY!);
-    const referenced = scrapeModels(/\b(gemini-(?:[\w.-]+))\b/g, sourceFiles);
+    const referenced = scrapeModels(GEMINI_MODEL_PATTERN, sourceFiles);
 
     if (referenced.length === 0) return;
 
-    // Skip experimental models, live-only models, and anchor-link fragments
-    // scraped from markdown (e.g., "gemini-live-bidigeneratecontent")
-    const stable = referenced.filter(
-      (m) => !m.includes("-exp") && !m.includes("-live") && !m.includes("bidigeneratecontent"),
-    );
+    // Drop experimental/live ids, markdown anchor fragments, and aimock
+    // provider-mode names (see filterStableGeminiModels).
+    const stable = filterStableGeminiModels(referenced);
 
     for (const m of stable) {
       const found = models.some((available) => available === m || available.startsWith(m));