From 305d3036a3528072ce3399d49d68d519876ccdcc Mon Sep 17 00:00:00 2001 From: skullcmd Date: Sun, 5 Apr 2026 05:31:04 +0000 Subject: [PATCH 1/4] feat(api): stabilize native responses history --- .gitignore | 3 + apps/api/.test-data/models.json | 9398 -------------------------- apps/api/dev/runTestServer.sh | 7 +- apps/api/dev/testApi.ts | 6 +- apps/api/dev/testOpenAISdk.ts | 90 +- apps/api/dev/testSetup.ts | 211 +- apps/api/dev/testWs.ts | 4 +- apps/api/modules/providerIdentity.ts | 138 + apps/api/modules/responsesHistory.ts | 20 + apps/api/package.json | 4 +- apps/api/routes/nativeProviders.ts | 1784 +++++ apps/api/server.ts | 4 + 12 files changed, 2126 insertions(+), 9543 deletions(-) delete mode 100644 apps/api/.test-data/models.json create mode 100644 apps/api/modules/providerIdentity.ts create mode 100644 apps/api/routes/nativeProviders.ts diff --git a/.gitignore b/.gitignore index f39f747..ff16cc3 100644 --- a/.gitignore +++ b/.gitignore @@ -56,6 +56,9 @@ apps/api/keys.json apps/api/providers.json apps/api/models.json apps/api/tiers.json +apps/api/.test-data/providers.json +apps/api/.test-data/keys.json +apps/api/.test-data/models.json apps/api/.test-data/*.backup apps/api/keys.json.invalid-2026-03-01T19-09-59-257Z.bak # Excluded errors (if generated) diff --git a/apps/api/.test-data/models.json b/apps/api/.test-data/models.json deleted file mode 100644 index bdc7aa7..0000000 --- a/apps/api/.test-data/models.json +++ /dev/null @@ -1,9398 +0,0 @@ -{ - "object": "list", - "data": [ - { - "id": "gpt-3.5-turbo", - "object": "model", - "created": 1773822267, - "owned_by": "openai-mock", - "providers": 113, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.017475, - "output": 0.052425, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5.4", - "object": "model", - "created": 1773822267, - "owned_by": "openai-mock", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0.047931, - "output": 0.287589, - "unit": "per_million_tokens" - } - }, - { - "id": "deepseek-chat", - "object": "model", - "created": 1774530909676, - "owned_by": "deepseek", - "providers": 65, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0.008266, - "output": 0.033674, - "unit": "per_million_tokens" - } - }, - { - "id": "deepseek-reasoner", - "object": "model", - "created": 1774530909676, - "owned_by": "deepseek", - "providers": 65, - "throughput": 50, - "capabilities": [ - "text", - "image_output", - "audio_output" - ], - "pricing": { - "input": 0.014031, - "output": 0.055869, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-2.5-flash", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input", - "tool_calling" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-2.5-pro", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input", - "tool_calling" - ], - "pricing": { - "input": 0.01864, - "output": 0.14912, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-2.0-flash", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-2.0-flash-001", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-2.0-flash-lite-001", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-2.0-flash-lite", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-2.5-flash-preview-tts", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 210, - "throughput": 50, - "capabilities": [ - "text", - "audio_output" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "audio_output": 0.6, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-2.5-pro-preview-tts", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 210, - "throughput": 50, - "capabilities": [ - "text", - "audio_output" - ], - "pricing": { - "input": 0.01864, - "output": 0.14912, - "audio_output": 1, - "unit": "per_million_tokens" - } - }, - { - "id": "gemma-3-1b-it", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "gemma-3-4b-it", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "gemma-3-12b-it", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "gemma-3-27b-it", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "gemma-3n-e4b-it", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "gemma-3n-e2b-it", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-flash-latest", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-flash-lite-latest", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-pro-latest", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.01864, - "output": 0.14912, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-2.5-flash-lite", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-2.5-flash-image", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "image_input", - "image_output" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "per_image": 0.008, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-2.5-flash-lite-preview-09-2025", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 199, - "throughput": 111, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-3-pro-preview", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.01864, - "output": 0.14912, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-3-flash-preview", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 211, - "throughput": 44, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-3.1-pro-preview", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input", - "tool_calling" - ], - "pricing": { - "input": 0.01864, - "output": 0.14912, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-3.1-pro-preview-customtools", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input", - "tool_calling" - ], - "pricing": { - "input": 0.01864, - "output": 0.14912, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-3.1-flash-lite-preview", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input", - "tool_calling" - ], - "pricing": { - "input": 0.005991, - "output": 0.035949, - "per_image": 5e-8, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-3-pro-image-preview", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "image_input", - "image_output" - ], - "pricing": { - "input": 0.01864, - "output": 0.14912, - "per_image": 0.008, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-3.1-flash-image-preview", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "image_input", - "image_output" - ], - "pricing": { - "input": 0.009986, - "output": 0.059914, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-robotics-er-1.5-preview", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input" - ], - "pricing": { - "input": 0.033552, - "output": 0.134208, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-2.5-computer-use-preview-10-2025", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.01864, - "output": 0.14912, - "unit": "per_million_tokens" - } - }, - { - "id": "deep-research-pro-preview-12-2025", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.033552, - "output": 0.134208, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-embedding-001", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-embedding-2-preview", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "aqa", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "imagen-4.0-generate-001", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "image_output" - ], - "pricing": { - "per_image": 0.008, - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "imagen-4.0-ultra-generate-001", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "image_output" - ], - "pricing": { - "per_image": 0.016, - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "imagen-4.0-fast-generate-001", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "image_output" - ], - "pricing": { - "per_image": 0.004, - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "veo-2.0-generate-001", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "image_output" - ], - "pricing": { - "per_request": 0.07, - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "veo-3.0-generate-001", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "image_output" - ], - "pricing": { - "per_image": 0.0125, - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "veo-3.0-fast-generate-001", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "image_output" - ], - "pricing": { - "per_image": 0.00625, - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "veo-3.1-generate-preview", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "image_output" - ], - "pricing": { - "per_image": 0.0125, - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "veo-3.1-fast-generate-preview", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "image_output" - ], - "pricing": { - "per_image": 0.00625, - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-2.5-flash-native-audio-latest", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "audio_input", - "audio_output" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "audio_input": 0.05, - "audio_output": 0.6, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-2.5-flash-native-audio-preview-09-2025", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "audio_input", - "audio_output" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "audio_input": 0.05, - "audio_output": 0.6, - "unit": "per_million_tokens" - } - }, - { - "id": "gemini-2.5-flash-native-audio-preview-12-2025", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 213, - "throughput": 50, - "capabilities": [ - "text", - "audio_input", - "audio_output" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "audio_input": 0.05, - "audio_output": 0.6, - "unit": "per_million_tokens" - } - }, - { - "id": "nano-banana-pro-preview", - "object": "model", - "created": 1774530909677, - "owned_by": "google", - "providers": 115, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "per_image": 0.008, - "unit": "per_million_tokens" - } - }, - { - "id": "lyria-3-clip-preview", - "object": "model", - "created": 1774530909677, - "owned_by": "unknown", - "providers": 17, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "lyria-3-pro-preview", - "object": "model", - "created": 1774530909677, - "owned_by": "unknown", - "providers": 19, - "throughput": 50, - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o-mini", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 112, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-image-1", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 104, - "throughput": 50, - "capabilities": [ - "text", - "image_output" - ], - "pricing": { - "per_image": 0.008, - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4-turbo", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 111, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.17475, - "output": 0.52425, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5-mini-2025-08-07", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 109, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.01398, - "output": 0.05592, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5-mini", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 109, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.01398, - "output": 0.05592, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o-mini-2024-07-18", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 111, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o-2024-08-06", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 111, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.033552, - "output": 0.134208, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o-2024-11-20", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 111, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.033552, - "output": 0.134208, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 113, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.033552, - "output": 0.134208, - "image_input": 0.0003826, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5.2", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 109, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.033552, - "output": 0.134208, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4-0613", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.233, - "output": 0.466, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.233, - "output": 0.466, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5.4-mini", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0.009986, - "output": 0.059914, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5.4-nano-2026-03-17", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0.005785, - "output": 0.036155, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5.4-nano", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0.005785, - "output": 0.036155, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5.4-mini-2026-03-17", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0.009986, - "output": 0.059914, - "unit": "per_million_tokens" - } - }, - { - "id": "davinci-002", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.03495, - "output": 0.03495, - "unit": "per_million_tokens" - } - }, - { - "id": "babbage-002", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.016776, - "output": 0.016776, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-3.5-turbo-instruct", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.029957, - "output": 0.039943, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-3.5-turbo-instruct-0914", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.029957, - "output": 0.039943, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4-1106-preview", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.17475, - "output": 0.52425, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-3.5-turbo-1106", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.0233, - "output": 0.0466, - "unit": "per_million_tokens" - } - }, - { - "id": "tts-1-hd", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "audio_output" - ], - "pricing": { - "input": 0.16776, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "tts-1-1106", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "audio_output" - ], - "pricing": { - "input": 0.08388, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "tts-1-hd-1106", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "audio_output" - ], - "pricing": { - "input": 0.16776, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "text-embedding-3-small", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.016776, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "text-embedding-3-large", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.016776, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4-0125-preview", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.17475, - "output": 0.52425, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4-turbo-preview", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.17475, - "output": 0.52425, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-3.5-turbo-0125", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 112, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.017475, - "output": 0.052425, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4-turbo-2024-04-09", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.17475, - "output": 0.52425, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o-2024-05-13", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.08388, - "output": 0.25164, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o-audio-preview", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0.033552, - "output": 0.134208, - "audio_input": 2, - "audio_output": 4, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o-realtime-preview", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input", - "audio_output" - ], - "pricing": { - "input": 0.067104, - "output": 0.268416, - "audio_input": 2, - "audio_output": 4, - "unit": "per_million_tokens" - } - }, - { - "id": "omni-moderation-latest", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "omni-moderation-2024-09-26", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o-realtime-preview-2024-12-17", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input", - "audio_output" - ], - "pricing": { - "input": 0.067104, - "output": 0.268416, - "audio_input": 2, - "audio_output": 4, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o-audio-preview-2024-12-17", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0.033552, - "output": 0.134208, - "audio_input": 2, - "audio_output": 4, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o-mini-realtime-preview-2024-12-17", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input", - "audio_output" - ], - "pricing": { - "input": 0.01398, - "output": 0.05592, - "audio_input": 0.5, - "audio_output": 1, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o-mini-audio-preview-2024-12-17", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "audio_input": 0.5, - "audio_output": 1, - "unit": "per_million_tokens" - } - }, - { - "id": "o1-2024-12-17", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.1398, - "output": 0.5592, - "unit": "per_million_tokens" - } - }, - { - "id": "o1", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.1398, - "output": 0.5592, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o-mini-realtime-preview", - "object": "model", - "created": 1774530909677, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input", - "audio_output" - ], - "pricing": { - "input": 0.01398, - "output": 0.05592, - "audio_input": 0.5, - "audio_output": 1, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o-mini-audio-preview", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "audio_input": 0.5, - "audio_output": 1, - "unit": "per_million_tokens" - } - }, - { - "id": "o3-mini", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.01398, - "output": 0.05592, - "unit": "per_million_tokens" - } - }, - { - "id": "o3-mini-2025-01-31", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.01398, - "output": 0.05592, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o-mini-search-preview-2025-03-11", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o-mini-search-preview", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o-transcribe", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "audio_input" - ], - "pricing": { - "input": 0.033552, - "output": 0.134208, - "audio_input": 0.3, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o-mini-transcribe", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "audio_input" - ], - "pricing": { - "audio_input": 0.15, - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "o1-pro-2025-03-19", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.16776, - "output": 0.67104, - "unit": "per_million_tokens" - } - }, - { - "id": "o1-pro", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.16776, - "output": 0.67104, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o-mini-tts", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "audio_output" - ], - "pricing": { - "audio_output": 0.6, - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "o3-2025-04-16", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.1398, - "output": 0.5592, - "unit": "per_million_tokens" - } - }, - { - "id": "o4-mini-2025-04-16", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.01398, - "output": 0.05592, - "unit": "per_million_tokens" - } - }, - { - "id": "o3", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.1398, - "output": 0.5592, - "unit": "per_million_tokens" - } - }, - { - "id": "o4-mini", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.01398, - "output": 0.05592, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4.1-2025-04-14", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.033552, - "output": 0.134208, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4.1", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.033552, - "output": 0.134208, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4.1-mini-2025-04-14", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.01398, - "output": 0.05592, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4.1-mini", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.01398, - "output": 0.05592, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4.1-nano-2025-04-14", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4.1-nano", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o-realtime-preview-2025-06-03", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input", - "audio_output" - ], - "pricing": { - "input": 0.067104, - "output": 0.268416, - "audio_input": 2, - "audio_output": 4, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o-audio-preview-2025-06-03", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0.033552, - "output": 0.134208, - "audio_input": 2, - "audio_output": 4, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o-transcribe-diarize", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input" - ], - "pricing": { - "input": 0.033552, - "output": 0.134208, - "audio_input": 0.3, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5-chat-latest", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.067104, - "output": 0.268416, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5-2025-08-07", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.067104, - "output": 0.268416, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5-nano-2025-08-07", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 109, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5-nano", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-audio-2025-08-28", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_output", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0.033552, - "output": 0.134208, - "audio_input": 2, - "audio_output": 4, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-realtime", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "audio_input", - "audio_output" - ], - "pricing": { - "input": 0.067104, - "output": 0.268416, - "audio_input": 2, - "audio_output": 4, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-realtime-2025-08-28", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "audio_input", - "audio_output" - ], - "pricing": { - "input": 0.067104, - "output": 0.268416, - "audio_input": 2, - "audio_output": 4, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-audio", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_output", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0.033552, - "output": 0.134208, - "audio_input": 2, - "audio_output": 4, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5-codex", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.067104, - "output": 0.268416, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-image-1-mini", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 103, - "throughput": 50, - "capabilities": [ - "text", - "image_output" - ], - "pricing": { - "per_image": 0.004, - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5-pro-2025-10-06", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.1398, - "output": 0.5592, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5-pro", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.1398, - "output": 0.5592, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-audio-mini", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_output", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "audio_input": 0.5, - "audio_output": 1, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-audio-mini-2025-10-06", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_output", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "audio_input": 0.5, - "audio_output": 1, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5-search-api", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.067104, - "output": 0.268416, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-realtime-mini", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "audio_input", - "audio_output" - ], - "pricing": { - "input": 0.01398, - "output": 0.05592, - "audio_input": 0.5, - "audio_output": 1, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-realtime-mini-2025-10-06", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "audio_input", - "audio_output" - ], - "pricing": { - "input": 0.01398, - "output": 0.05592, - "audio_input": 0.5, - "audio_output": 1, - "unit": "per_million_tokens" - } - }, - { - "id": "sora-2", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_output" - ], - "pricing": { - "per_image": 0.000625, - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "sora-2-pro", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_output" - ], - "pricing": { - "per_image": 0.04, - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5-search-api-2025-10-14", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.067104, - "output": 0.268416, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5.1-chat-latest", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.067104, - "output": 0.268416, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5.1-2025-11-13", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.067104, - "output": 0.268416, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5.1", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.067104, - "output": 0.268416, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5.1-codex", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.067104, - "output": 0.268416, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5.1-codex-mini", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.01398, - "output": 0.05592, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5.1-codex-max", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.1398, - "output": 0.5592, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-image-1.5", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 103, - "throughput": 50, - "capabilities": [ - "text", - "image_output" - ], - "pricing": { - "per_image": 0.008, - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5.2-2025-12-11", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.033552, - "output": 0.134208, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5.2-pro-2025-12-11", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.1398, - "output": 0.5592, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5.2-pro", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.1398, - "output": 0.5592, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5.2-chat-latest", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.033552, - "output": 0.134208, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o-mini-transcribe-2025-12-15", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "audio_input" - ], - "pricing": { - "audio_input": 0.15, - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o-mini-transcribe-2025-03-20", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "audio_input" - ], - "pricing": { - "audio_input": 0.15, - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o-mini-tts-2025-03-20", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "audio_output" - ], - "pricing": { - "audio_output": 0.6, - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o-mini-tts-2025-12-15", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "audio_output" - ], - "pricing": { - "audio_output": 0.6, - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-realtime-mini-2025-12-15", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "audio_input", - "audio_output" - ], - "pricing": { - "input": 0.01398, - "output": 0.05592, - "audio_input": 0.5, - "audio_output": 1, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-audio-mini-2025-12-15", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_output", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0.00671, - "output": 0.026842, - "audio_input": 0.5, - "audio_output": 1, - "unit": "per_million_tokens" - } - }, - { - "id": "chatgpt-image-latest", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 103, - "throughput": 50, - "capabilities": [ - "text", - "image_output" - ], - "pricing": { - "per_image": 0.008, - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5.2-codex", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.033552, - "output": 0.134208, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5.3-codex", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.033552, - "output": 0.134208, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-realtime-1.5", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "audio_input", - "audio_output" - ], - "pricing": { - "input": 0.067104, - "output": 0.268416, - "audio_input": 1.5, - "audio_output": 3, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-audio-1.5", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_output", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0.033552, - "output": 0.134208, - "audio_input": 1.5, - "audio_output": 3, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o-search-preview", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.033552, - "output": 0.134208, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4o-search-preview-2025-03-11", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.033552, - "output": 0.134208, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5.3-chat-latest", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0.067104, - "output": 0.268416, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5.4-2026-03-05", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0.047931, - "output": 0.287589, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5.4-pro", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 105, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0.119829, - "output": 0.718971, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5.4-pro-2026-03-05", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0.119829, - "output": 0.718971, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-3.5-turbo-16k", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.071897, - "output": 0.095863, - "unit": "per_million_tokens" - } - }, - { - "id": "tts-1", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text", - "audio_output" - ], - "pricing": { - "input": 0.08388, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "whisper-1", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 110, - "throughput": 50, - "capabilities": [ - "text", - "audio_input" - ], - "pricing": { - "audio_input": 0.0003, - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "text-embedding-ada-002", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 108, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.016776, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-5", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 55, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.067104, - "output": 0.268416, - "unit": "per_million_tokens" - } - }, - { - "id": "o4-mini-deep-research", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 78, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.033552, - "output": 0.134208, - "unit": "per_million_tokens" - } - }, - { - "id": "o4-mini-deep-research-2025-06-26", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 78, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.033552, - "output": 0.134208, - "unit": "per_million_tokens" - } - }, - { - "id": "o3-pro", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 26, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.16776, - "output": 0.67104, - "unit": "per_million_tokens" - } - }, - { - "id": "o3-pro-2025-06-10", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 26, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.16776, - "output": 0.67104, - "unit": "per_million_tokens" - } - }, - { - "id": "o3-deep-research", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 26, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.1398, - "output": 0.5592, - "unit": "per_million_tokens" - } - }, - { - "id": "o3-deep-research-2025-06-26", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 26, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.1398, - "output": 0.5592, - "unit": "per_million_tokens" - } - }, - { - "id": "computer-use-preview", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 26, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.033552, - "output": 0.134208, - "unit": "per_million_tokens" - } - }, - { - "id": "computer-use-preview-2025-03-11", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 26, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.033552, - "output": 0.134208, - "unit": "per_million_tokens" - } - }, - { - "id": "gpt-4-0314", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 2, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.29125, - "output": 0.5825, - "unit": "per_million_tokens" - } - }, - { - "id": "reka/reka-edge", - "object": "model", - "created": 1774530909678, - "owned_by": "reka", - "providers": 4, - "throughput": 50, - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "xiaomi/mimo-v2-omni", - "object": "model", - "created": 1774530909678, - "owned_by": "xiaomi", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input" - ], - "pricing": { - "input": 0.012621, - "output": 0.063104, - "unit": "per_million_tokens" - } - }, - { - "id": "xiaomi/mimo-v2-pro", - "object": "model", - "created": 1774530909678, - "owned_by": "xiaomi", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.018931, - "output": 0.056794, - "unit": "per_million_tokens" - } - }, - { - "id": "minimax/minimax-m2.7", - "object": "model", - "created": 1774530909678, - "owned_by": "minimax", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.009087, - "output": 0.036348, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-5.4-nano", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.006267, - "output": 0.039168, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-5.4-mini", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.010818, - "output": 0.064907, - "unit": "per_million_tokens" - } - }, - { - "id": "mistralai/mistral-small-2603", - "object": "model", - "created": 1774530909678, - "owned_by": "mistral.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "z-ai/glm-5-turbo", - "object": "model", - "created": 1774530909678, - "owned_by": "z.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.017475, - "output": 0.05825, - "unit": "per_million_tokens" - } - }, - { - "id": "x-ai/grok-4.20-multi-agent-beta", - "object": "model", - "created": 1774530909678, - "owned_by": "xai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.045435, - "output": 0.136305, - "unit": "per_million_tokens" - } - }, - { - "id": "x-ai/grok-4.20-beta", - "object": "model", - "created": 1774530909678, - "owned_by": "xai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.045435, - "output": 0.136305, - "unit": "per_million_tokens" - } - }, - { - "id": "nvidia/nemotron-3-super-120b-a12b:free", - "object": "model", - "created": 1774530909678, - "owned_by": "nvidia", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "nvidia/nemotron-3-super-120b-a12b", - "object": "model", - "created": 1774530909678, - "owned_by": "nvidia", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.006058, - "output": 0.03029, - "unit": "per_million_tokens" - } - }, - { - "id": "bytedance-seed/seed-2.0-lite", - "object": "model", - "created": 1774530909678, - "owned_by": "bytedance", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.008414, - "output": 0.067311, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3.5-9b", - "object": "model", - "created": 1774530909678, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.009087, - "output": 0.027261, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-5.4-pro", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.129814, - "output": 0.778886, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-5.4", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.051926, - "output": 0.311554, - "unit": "per_million_tokens" - } - }, - { - "id": "inception/mercury-2", - "object": "model", - "created": 1774530909678, - "owned_by": "inception", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.009087, - "output": 0.027261, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-5.3-chat", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.020193, - "output": 0.161547, - "unit": "per_million_tokens" - } - }, - { - "id": "google/gemini-3.1-flash-lite-preview", - "object": "model", - "created": 1774530909678, - "owned_by": "google", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input", - "tool_calling" - ], - "pricing": { - "input": 0.006491, - "output": 0.038944, - "per_image": 5e-8, - "unit": "per_million_tokens" - } - }, - { - "id": "bytedance-seed/seed-2.0-mini", - "object": "model", - "created": 1774530909678, - "owned_by": "bytedance", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "google/gemini-3.1-flash-image-preview", - "object": "model", - "created": 1774530909678, - "owned_by": "google", - "providers": 8, - "throughput": 50, - "capabilities": [ - "image_input", - "image_output" - ], - "pricing": { - "input": 0.010818, - "output": 0.064907, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3.5-35b-a3b", - "object": "model", - "created": 1774530909678, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.005048, - "output": 0.040387, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3.5-27b", - "object": "model", - "created": 1774530909678, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.005048, - "output": 0.040387, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3.5-122b-a10b", - "object": "model", - "created": 1774530909678, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.008414, - "output": 0.067311, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3.5-flash-02-23", - "object": "model", - "created": 1774530909678, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "liquid/lfm-2-24b-a2b", - "object": "model", - "created": 1774530909678, - "owned_by": "liquid", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "google/gemini-3.1-pro-preview-customtools", - "object": "model", - "created": 1774530909678, - "owned_by": "google", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input", - "tool_calling" - ], - "pricing": { - "input": 0.020193, - "output": 0.161547, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-5.3-codex", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.036348, - "output": 0.145392, - "unit": "per_million_tokens" - } - }, - { - "id": "aion-labs/aion-2.0", - "object": "model", - "created": 1774530909678, - "owned_by": "aion-labs", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.025242, - "output": 0.050483, - "unit": "per_million_tokens" - } - }, - { - "id": "google/gemini-3.1-pro-preview", - "object": "model", - "created": 1774530909678, - "owned_by": "google", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input", - "tool_calling" - ], - "pricing": { - "input": 0.020193, - "output": 0.161547, - "unit": "per_million_tokens" - } - }, - { - "id": "anthropic/claude-sonnet-4.6", - "object": "model", - "created": 1774530909678, - "owned_by": "anthropic", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.06058, - "output": 0.3029, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3.5-plus-02-15", - "object": "model", - "created": 1774530909678, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.006491, - "output": 0.038944, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3.5-397b-a17b", - "object": "model", - "created": 1774530909678, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.010818, - "output": 0.064907, - "unit": "per_million_tokens" - } - }, - { - "id": "minimax/minimax-m2.5:free", - "object": "model", - "created": 1774530909678, - "owned_by": "minimax", - "providers": 8, - "throughput": 50, - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "minimax/minimax-m2.5", - "object": "model", - "created": 1774530909678, - "owned_by": "minimax", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.006633, - "output": 0.038802, - "unit": "per_million_tokens" - } - }, - { - "id": "z-ai/glm-5", - "object": "model", - "created": 1774530909678, - "owned_by": "z.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.018054, - "output": 0.057671, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-max-thinking", - "object": "model", - "created": 1774530909678, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.012621, - "output": 0.063104, - "unit": "per_million_tokens" - } - }, - { - "id": "anthropic/claude-opus-4.6", - "object": "model", - "created": 1774530909678, - "owned_by": "anthropic", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.06524, - "output": 0.3262, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-coder-next", - "object": "model", - "created": 1774530909678, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.005014, - "output": 0.031334, - "unit": "per_million_tokens" - } - }, - { - "id": "openrouter/free", - "object": "model", - "created": 1774530909678, - "owned_by": "openrouter", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "stepfun/step-3.5-flash:free", - "object": "model", - "created": 1774530909678, - "owned_by": "stepfun", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "stepfun/step-3.5-flash", - "object": "model", - "created": 1774530909678, - "owned_by": "stepfun", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.009087, - "output": 0.027261, - "unit": "per_million_tokens" - } - }, - { - "id": "arcee-ai/trinity-large-preview:free", - "object": "model", - "created": 1774530909678, - "owned_by": "arcee", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "moonshotai/kimi-k2.5", - "object": "model", - "created": 1774530909678, - "owned_by": "moonshot", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.012859, - "output": 0.062866, - "unit": "per_million_tokens" - } - }, - { - "id": "upstage/solar-pro-3", - "object": "model", - "created": 1774530909678, - "owned_by": "upstage", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "minimax/minimax-m2-her", - "object": "model", - "created": 1774530909678, - "owned_by": "minimax", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.009087, - "output": 0.036348, - "unit": "per_million_tokens" - } - }, - { - "id": "writer/palmyra-x5", - "object": "model", - "created": 1774530909678, - "owned_by": "writer", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.016522, - "output": 0.165218, - "unit": "per_million_tokens" - } - }, - { - "id": "liquid/lfm-2.5-1.2b-thinking:free", - "object": "model", - "created": 1774530909678, - "owned_by": "liquid", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "liquid/lfm-2.5-1.2b-instruct:free", - "object": "model", - "created": 1774530909678, - "owned_by": "liquid", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-audio", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "audio_input" - ], - "pricing": { - "input": 0.036348, - "output": 0.145392, - "audio_input": 2, - "audio_output": 4, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-audio-mini", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "audio_input" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "audio_input": 0.5, - "audio_output": 1, - "unit": "per_million_tokens" - } - }, - { - "id": "z-ai/glm-4.7-flash", - "object": "model", - "created": 1774530909678, - "owned_by": "z.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.004741, - "output": 0.031607, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-5.2-codex", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.036348, - "output": 0.145392, - "unit": "per_million_tokens" - } - }, - { - "id": "allenai/olmo-3.1-32b-instruct", - "object": "model", - "created": 1774530909678, - "owned_by": "allenai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.009087, - "output": 0.027261, - "unit": "per_million_tokens" - } - }, - { - "id": "bytedance-seed/seed-1.6-flash", - "object": "model", - "created": 1774530909678, - "owned_by": "bytedance", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "bytedance-seed/seed-1.6", - "object": "model", - "created": 1774530909678, - "owned_by": "bytedance", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.008414, - "output": 0.067311, - "unit": "per_million_tokens" - } - }, - { - "id": "minimax/minimax-m2.1", - "object": "model", - "created": 1774530909678, - "owned_by": "minimax", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.010055, - "output": 0.03538, - "unit": "per_million_tokens" - } - }, - { - "id": "z-ai/glm-4.7", - "object": "model", - "created": 1774530909678, - "owned_by": "z.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.0138, - "output": 0.061925, - "unit": "per_million_tokens" - } - }, - { - "id": "google/gemini-3-flash-preview", - "object": "model", - "created": 1774530909678, - "owned_by": "google", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input", - "tool_calling" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "mistralai/mistral-small-creative", - "object": "model", - "created": 1774530909678, - "owned_by": "mistral.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.009087, - "output": 0.027261, - "unit": "per_million_tokens" - } - }, - { - "id": "allenai/olmo-3.1-32b-think", - "object": "model", - "created": 1774530909678, - "owned_by": "allenai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.008388, - "output": 0.02796, - "unit": "per_million_tokens" - } - }, - { - "id": "xiaomi/mimo-v2-flash", - "object": "model", - "created": 1774530909678, - "owned_by": "xiaomi", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.008609, - "output": 0.027739, - "unit": "per_million_tokens" - } - }, - { - "id": "nvidia/nemotron-3-nano-30b-a3b:free", - "object": "model", - "created": 1774530909678, - "owned_by": "nvidia", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "nvidia/nemotron-3-nano-30b-a3b", - "object": "model", - "created": 1774530909678, - "owned_by": "nvidia", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-5.2-chat", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.020193, - "output": 0.161547, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-5.2-pro", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.15145, - "output": 0.6058, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-5.2", - "object": "model", - "created": 1774530909678, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.036348, - "output": 0.145392, - "unit": "per_million_tokens" - } - }, - { - "id": "mistralai/devstral-2512", - "object": "model", - "created": 1774530909678, - "owned_by": "mistral.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.012621, - "output": 0.063104, - "unit": "per_million_tokens" - } - }, - { - "id": "relace/relace-search", - "object": "model", - "created": 1774530909678, - "owned_by": "relace", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.018931, - "output": 0.056794, - "unit": "per_million_tokens" - } - }, - { - "id": "z-ai/glm-4.6v", - "object": "model", - "created": 1774530909678, - "owned_by": "z.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.011359, - "output": 0.034076, - "unit": "per_million_tokens" - } - }, - { - "id": "nex-agi/deepseek-v3.1-nex-n1", - "object": "model", - "created": 1774530909678, - "owned_by": "nex-agi", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.009659, - "output": 0.035776, - "unit": "per_million_tokens" - } - }, - { - "id": "essentialai/rnj-1-instruct", - "object": "model", - "created": 1774530909678, - "owned_by": "essentialai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.018174, - "output": 0.018174, - "unit": "per_million_tokens" - } - }, - { - "id": "openrouter/bodybuilder", - "object": "model", - "created": 1774530909678, - "owned_by": "openrouter", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-5.1-codex-max", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.15145, - "output": 0.6058, - "unit": "per_million_tokens" - } - }, - { - "id": "amazon/nova-2-lite-v1", - "object": "model", - "created": 1774530909679, - "owned_by": "amazon", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.008113, - "output": 0.067612, - "unit": "per_million_tokens" - } - }, - { - "id": "mistralai/ministral-14b-2512", - "object": "model", - "created": 1774530909679, - "owned_by": "mistral.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.018174, - "output": 0.018174, - "unit": "per_million_tokens" - } - }, - { - "id": "mistralai/ministral-8b-2512", - "object": "model", - "created": 1774530909679, - "owned_by": "mistral.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.018174, - "output": 0.018174, - "unit": "per_million_tokens" - } - }, - { - "id": "mistralai/ministral-3b-2512", - "object": "model", - "created": 1774530909679, - "owned_by": "mistral.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.018174, - "output": 0.018174, - "unit": "per_million_tokens" - } - }, - { - "id": "mistralai/mistral-large-2512", - "object": "model", - "created": 1774530909679, - "owned_by": "mistral.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.011359, - "output": 0.034076, - "unit": "per_million_tokens" - } - }, - { - "id": "arcee-ai/trinity-mini:free", - "object": "model", - "created": 1774530909679, - "owned_by": "arcee", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "arcee-ai/trinity-mini", - "object": "model", - "created": 1774530909679, - "owned_by": "arcee", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.008388, - "output": 0.02796, - "unit": "per_million_tokens" - } - }, - { - "id": "deepseek/deepseek-v3.2-speciale", - "object": "model", - "created": 1774530909679, - "owned_by": "deepseek", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.011359, - "output": 0.034076, - "unit": "per_million_tokens" - } - }, - { - "id": "deepseek/deepseek-v3.2", - "object": "model", - "created": 1774530909679, - "owned_by": "deepseek", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.014766, - "output": 0.021582, - "unit": "per_million_tokens" - } - }, - { - "id": "prime-intellect/intellect-3", - "object": "model", - "created": 1774530909679, - "owned_by": "prime-intellect", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.00699, - "output": 0.038445, - "unit": "per_million_tokens" - } - }, - { - "id": "anthropic/claude-opus-4.5", - "object": "model", - "created": 1774530909679, - "owned_by": "anthropic", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.06058, - "output": 0.3029, - "unit": "per_million_tokens" - } - }, - { - "id": "allenai/olmo-3-32b-think", - "object": "model", - "created": 1774530909679, - "owned_by": "allenai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.008388, - "output": 0.02796, - "unit": "per_million_tokens" - } - }, - { - "id": "google/gemini-3-pro-image-preview", - "object": "model", - "created": 1774530909679, - "owned_by": "google", - "providers": 8, - "throughput": 50, - "capabilities": [ - "image_input", - "image_output" - ], - "pricing": { - "input": 0.020193, - "output": 0.161547, - "per_image": 0.008, - "unit": "per_million_tokens" - } - }, - { - "id": "x-ai/grok-4.1-fast", - "object": "model", - "created": 1774530909679, - "owned_by": "xai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.010385, - "output": 0.025963, - "unit": "per_million_tokens" - } - }, - { - "id": "deepcogito/cogito-v2.1-671b", - "object": "model", - "created": 1774530909679, - "owned_by": "deepcogito", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.037863, - "output": 0.037863, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-5.1", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.072696, - "output": 0.290784, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-5.1-chat", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.020193, - "output": 0.161547, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-5.1-codex", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.072696, - "output": 0.290784, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-5.1-codex-mini", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.015145, - "output": 0.06058, - "unit": "per_million_tokens" - } - }, - { - "id": "kwaipilot/kat-coder-pro", - "object": "model", - "created": 1774530909679, - "owned_by": "kwaipilot", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "moonshotai/kimi-k2-thinking", - "object": "model", - "created": 1774530909679, - "owned_by": "moonshot", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.014409, - "output": 0.061316, - "unit": "per_million_tokens" - } - }, - { - "id": "amazon/nova-premier-v1", - "object": "model", - "created": 1774530909679, - "owned_by": "amazon", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.03029, - "output": 0.15145, - "unit": "per_million_tokens" - } - }, - { - "id": "perplexity/sonar-pro-search", - "object": "model", - "created": 1774530909679, - "owned_by": "perplexity", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.06058, - "output": 0.3029, - "unit": "per_million_tokens" - } - }, - { - "id": "mistralai/voxtral-small-24b-2507", - "object": "model", - "created": 1774530909679, - "owned_by": "mistral.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "audio_input", - "tool_calling" - ], - "pricing": { - "input": 0.009087, - "output": 0.027261, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-oss-safeguard-20b", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "nvidia/nemotron-nano-12b-v2-vl:free", - "object": "model", - "created": 1774530909679, - "owned_by": "nvidia", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "nvidia/nemotron-nano-12b-v2-vl", - "object": "model", - "created": 1774530909679, - "owned_by": "nvidia", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.009087, - "output": 0.027261, - "unit": "per_million_tokens" - } - }, - { - "id": "minimax/minimax-m2", - "object": "model", - "created": 1774530909679, - "owned_by": "minimax", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.009232, - "output": 0.036203, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-vl-32b-instruct", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "liquid/lfm2-8b-a1b", - "object": "model", - "created": 1774530909679, - "owned_by": "liquid", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.012116, - "output": 0.024232, - "unit": "per_million_tokens" - } - }, - { - "id": "liquid/lfm-2.2-6b", - "object": "model", - "created": 1774530909679, - "owned_by": "liquid", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.012116, - "output": 0.024232, - "unit": "per_million_tokens" - } - }, - { - "id": "ibm-granite/granite-4.0-h-micro", - "object": "model", - "created": 1774530909679, - "owned_by": "ibm", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.004865, - "output": 0.031483, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-5-image-mini", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 7, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output" - ], - "pricing": { - "per_image": 0.004, - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "anthropic/claude-haiku-4.5", - "object": "model", - "created": 1774530909679, - "owned_by": "anthropic", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.012621, - "output": 0.063104, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-vl-8b-thinking", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.003587, - "output": 0.041848, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-vl-8b-instruct", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.005014, - "output": 0.031334, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-5-image", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output" - ], - "pricing": { - "per_image": 0.008, - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/o3-deep-research", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.15145, - "output": 0.6058, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/o4-mini-deep-research", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.036348, - "output": 0.145392, - "unit": "per_million_tokens" - } - }, - { - "id": "nvidia/llama-3.3-nemotron-super-49b-v1.5", - "object": "model", - "created": 1774530909679, - "owned_by": "nvidia", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "baidu/ernie-4.5-21b-a3b-thinking", - "object": "model", - "created": 1774530909679, - "owned_by": "baidu", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "google/gemini-2.5-flash-image", - "object": "model", - "created": 1774530909679, - "owned_by": "google", - "providers": 8, - "throughput": 50, - "capabilities": [ - "image_input", - "image_output" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "per_image": 0.008, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-vl-30b-a3b-thinking", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.003495, - "output": 0.04194, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-vl-30b-a3b-instruct", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-5-pro", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.15145, - "output": 0.6058, - "unit": "per_million_tokens" - } - }, - { - "id": "z-ai/glm-4.6", - "object": "model", - "created": 1774530909679, - "owned_by": "z.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.012896, - "output": 0.062829, - "unit": "per_million_tokens" - } - }, - { - "id": "anthropic/claude-sonnet-4.5", - "object": "model", - "created": 1774530909679, - "owned_by": "anthropic", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.06058, - "output": 0.3029, - "unit": "per_million_tokens" - } - }, - { - "id": "deepseek/deepseek-v3.2-exp", - "object": "model", - "created": 1774530909679, - "owned_by": "deepseek", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.014432, - "output": 0.021916, - "unit": "per_million_tokens" - } - }, - { - "id": "thedrummer/cydonia-24b-v4.1", - "object": "model", - "created": 1774530909679, - "owned_by": "thedrummer", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.013631, - "output": 0.022718, - "unit": "per_million_tokens" - } - }, - { - "id": "relace/relace-apply-3", - "object": "model", - "created": 1774530909679, - "owned_by": "relace", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.01839, - "output": 0.027045, - "unit": "per_million_tokens" - } - }, - { - "id": "google/gemini-2.5-flash-lite-preview-09-2025", - "object": "model", - "created": 1774530909679, - "owned_by": "google", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input", - "tool_calling" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-vl-235b-a22b-thinking", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.006884, - "output": 0.068841, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-vl-235b-a22b-instruct", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.008414, - "output": 0.037021, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-max", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.012621, - "output": 0.063104, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-coder-plus", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.012621, - "output": 0.063104, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-5-codex", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.072696, - "output": 0.290784, - "unit": "per_million_tokens" - } - }, - { - "id": "deepseek/deepseek-v3.1-terminus", - "object": "model", - "created": 1774530909679, - "owned_by": "deepseek", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.007633, - "output": 0.028715, - "unit": "per_million_tokens" - } - }, - { - "id": "x-ai/grok-4-fast", - "object": "model", - "created": 1774530909679, - "owned_by": "xai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.010385, - "output": 0.025963, - "unit": "per_million_tokens" - } - }, - { - "id": "alibaba/tongyi-deepresearch-30b-a3b", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.006058, - "output": 0.03029, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-coder-flash", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.007573, - "output": 0.037863, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-next-80b-a3b-thinking", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.004039, - "output": 0.032309, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-next-80b-a3b-instruct:free", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-next-80b-a3b-instruct", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.003436, - "output": 0.041999, - "unit": "per_million_tokens" - } - }, - { - "id": "meituan/longcat-flash-chat", - "object": "model", - "created": 1774530909679, - "owned_by": "meituan", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen-plus-2025-07-28:thinking", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.009087, - "output": 0.027261, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen-plus-2025-07-28", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.009087, - "output": 0.027261, - "unit": "per_million_tokens" - } - }, - { - "id": "nvidia/nemotron-nano-9b-v2:free", - "object": "model", - "created": 1774530909679, - "owned_by": "nvidia", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "nvidia/nemotron-nano-9b-v2", - "object": "model", - "created": 1774530909679, - "owned_by": "nvidia", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "moonshotai/kimi-k2-0905", - "object": "model", - "created": 1774530909679, - "owned_by": "moonshot", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.012621, - "output": 0.063104, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-30b-a3b-thinking-2507", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.006058, - "output": 0.03029, - "unit": "per_million_tokens" - } - }, - { - "id": "x-ai/grok-code-fast-1", - "object": "model", - "created": 1774530909679, - "owned_by": "xai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.015145, - "output": 0.06058, - "unit": "per_million_tokens" - } - }, - { - "id": "nousresearch/hermes-4-70b", - "object": "model", - "created": 1774530909679, - "owned_by": "nous", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.008916, - "output": 0.027432, - "unit": "per_million_tokens" - } - }, - { - "id": "nousresearch/hermes-4-405b", - "object": "model", - "created": 1774530909679, - "owned_by": "nous", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.018931, - "output": 0.056794, - "unit": "per_million_tokens" - } - }, - { - "id": "deepseek/deepseek-chat-v3.1", - "object": "model", - "created": 1774530909679, - "owned_by": "deepseek", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.006058, - "output": 0.03029, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-4o-audio-preview", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "audio_input" - ], - "pricing": { - "input": 0.036348, - "output": 0.145392, - "audio_input": 2, - "audio_output": 4, - "unit": "per_million_tokens" - } - }, - { - "id": "mistralai/mistral-medium-3.1", - "object": "model", - "created": 1774530909679, - "owned_by": "mistral.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.012621, - "output": 0.063104, - "unit": "per_million_tokens" - } - }, - { - "id": "baidu/ernie-4.5-21b-a3b", - "object": "model", - "created": 1774530909679, - "owned_by": "baidu", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "baidu/ernie-4.5-vl-28b-a3b", - "object": "model", - "created": 1774530909679, - "owned_by": "baidu", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "z-ai/glm-4.5v", - "object": "model", - "created": 1774530909679, - "owned_by": "z.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.018931, - "output": 0.056794, - "unit": "per_million_tokens" - } - }, - { - "id": "ai21/jamba-large-1.7", - "object": "model", - "created": 1774530909679, - "owned_by": "ai21", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.036348, - "output": 0.145392, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-5-chat", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.020193, - "output": 0.161547, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-5", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.072696, - "output": 0.290784, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-5-mini", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.015145, - "output": 0.06058, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-5-nano", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-oss-120b:free", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "tool_calling" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-oss-120b", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.00619, - "output": 0.030158, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-oss-20b:free", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-oss-20b", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.007789, - "output": 0.028559, - "unit": "per_million_tokens" - } - }, - { - "id": "anthropic/claude-opus-4.1", - "object": "model", - "created": 1774530909679, - "owned_by": "anthropic", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.126208, - "output": 0.631042, - "unit": "per_million_tokens" - } - }, - { - "id": "mistralai/codestral-2508", - "object": "model", - "created": 1774530909679, - "owned_by": "mistral.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.011359, - "output": 0.034076, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-coder-30b-a3b-instruct", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.007483, - "output": 0.028865, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-30b-a3b-instruct-2507", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.008388, - "output": 0.02796, - "unit": "per_million_tokens" - } - }, - { - "id": "z-ai/glm-4.5", - "object": "model", - "created": 1774530909679, - "owned_by": "z.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.016227, - "output": 0.059498, - "unit": "per_million_tokens" - } - }, - { - "id": "z-ai/glm-4.5-air:free", - "object": "model", - "created": 1774530909679, - "owned_by": "z.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "z-ai/glm-4.5-air", - "object": "model", - "created": 1774530909679, - "owned_by": "z.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.004822, - "output": 0.031526, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-235b-a22b-thinking-2507", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.00413, - "output": 0.041305, - "unit": "per_million_tokens" - } - }, - { - "id": "z-ai/glm-4-32b", - "object": "model", - "created": 1774530909679, - "owned_by": "z.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.018174, - "output": 0.018174, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-coder:free", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "tool_calling" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-coder", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.008193, - "output": 0.037242, - "unit": "per_million_tokens" - } - }, - { - "id": "bytedance/ui-tars-1.5-7b", - "object": "model", - "created": 1774530909679, - "owned_by": "bytedance", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.012116, - "output": 0.024232, - "unit": "per_million_tokens" - } - }, - { - "id": "google/gemini-2.5-flash-lite", - "object": "model", - "created": 1774530909679, - "owned_by": "google", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input", - "tool_calling" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-235b-a22b-2507", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.015092, - "output": 0.021256, - "unit": "per_million_tokens" - } - }, - { - "id": "switchpoint/router", - "object": "model", - "created": 1774530909679, - "owned_by": "switchpoint", - "providers": 8, - "throughput": 50, - "pricing": { - "input": 0.015145, - "output": 0.06058, - "unit": "per_million_tokens" - } - }, - { - "id": "moonshotai/kimi-k2", - "object": "model", - "created": 1774530909679, - "owned_by": "moonshot", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.015145, - "output": 0.06058, - "unit": "per_million_tokens" - } - }, - { - "id": "mistralai/devstral-medium", - "object": "model", - "created": 1774530909679, - "owned_by": "mistral.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.012621, - "output": 0.063104, - "unit": "per_million_tokens" - } - }, - { - "id": "mistralai/devstral-small", - "object": "model", - "created": 1774530909679, - "owned_by": "mistral.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.009087, - "output": 0.027261, - "unit": "per_million_tokens" - } - }, - { - "id": "cognitivecomputations/dolphin-mistral-24b-venice-edition:free", - "object": "model", - "created": 1774530909679, - "owned_by": "cognitivecomputations", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "x-ai/grok-4", - "object": "model", - "created": 1774530909679, - "owned_by": "xai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.06524, - "output": 0.3262, - "unit": "per_million_tokens" - } - }, - { - "id": "google/gemma-3n-e2b-it:free", - "object": "model", - "created": 1774530909679, - "owned_by": "google", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "tencent/hunyuan-a13b-instruct", - "object": "model", - "created": 1774530909679, - "owned_by": "tencent", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.007167, - "output": 0.029181, - "unit": "per_million_tokens" - } - }, - { - "id": "tngtech/deepseek-r1t2-chimera", - "object": "model", - "created": 1774530909679, - "owned_by": "tngtech", - "providers": 8, - "throughput": 50, - "pricing": { - "input": 0.009736, - "output": 0.035699, - "unit": "per_million_tokens" - } - }, - { - "id": "morph/morph-v3-large", - "object": "model", - "created": 1774530909679, - "owned_by": "morph", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.02434, - "output": 0.051385, - "unit": "per_million_tokens" - } - }, - { - "id": "morph/morph-v3-fast", - "object": "model", - "created": 1774530909679, - "owned_by": "morph", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.018174, - "output": 0.027261, - "unit": "per_million_tokens" - } - }, - { - "id": "baidu/ernie-4.5-vl-424b-a47b", - "object": "model", - "created": 1774530909679, - "owned_by": "baidu", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.011427, - "output": 0.034008, - "unit": "per_million_tokens" - } - }, - { - "id": "baidu/ernie-4.5-300b-a47b", - "object": "model", - "created": 1774530909679, - "owned_by": "baidu", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.009219, - "output": 0.036216, - "unit": "per_million_tokens" - } - }, - { - "id": "inception/mercury", - "object": "model", - "created": 1774530909679, - "owned_by": "inception", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.009087, - "output": 0.027261, - "unit": "per_million_tokens" - } - }, - { - "id": "mistralai/mistral-small-3.2-24b-instruct", - "object": "model", - "created": 1774530909679, - "owned_by": "mistral.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.009913, - "output": 0.026435, - "unit": "per_million_tokens" - } - }, - { - "id": "minimax/minimax-m1", - "object": "model", - "created": 1774530909679, - "owned_by": "minimax", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.01165, - "output": 0.064075, - "unit": "per_million_tokens" - } - }, - { - "id": "google/gemini-2.5-flash", - "object": "model", - "created": 1774530909679, - "owned_by": "google", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input", - "tool_calling" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "google/gemini-2.5-pro", - "object": "model", - "created": 1774530909679, - "owned_by": "google", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input", - "tool_calling" - ], - "pricing": { - "input": 0.020193, - "output": 0.161547, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/o3-pro", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.18174, - "output": 0.72696, - "unit": "per_million_tokens" - } - }, - { - "id": "x-ai/grok-3-mini", - "object": "model", - "created": 1774530909679, - "owned_by": "xai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.013631, - "output": 0.022718, - "unit": "per_million_tokens" - } - }, - { - "id": "x-ai/grok-3", - "object": "model", - "created": 1774530909679, - "owned_by": "xai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.06058, - "output": 0.3029, - "unit": "per_million_tokens" - } - }, - { - "id": "google/gemini-2.5-pro-preview", - "object": "model", - "created": 1774530909679, - "owned_by": "google", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input", - "tool_calling" - ], - "pricing": { - "input": 0.020193, - "output": 0.161547, - "per_image": 2.4e-7, - "unit": "per_million_tokens" - } - }, - { - "id": "deepseek/deepseek-r1-0528", - "object": "model", - "created": 1774530909679, - "owned_by": "deepseek", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.013106, - "output": 0.062619, - "unit": "per_million_tokens" - } - }, - { - "id": "anthropic/claude-opus-4", - "object": "model", - "created": 1774530909679, - "owned_by": "anthropic", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.126208, - "output": 0.631042, - "unit": "per_million_tokens" - } - }, - { - "id": "anthropic/claude-sonnet-4", - "object": "model", - "created": 1774530909679, - "owned_by": "anthropic", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.06058, - "output": 0.3029, - "unit": "per_million_tokens" - } - }, - { - "id": "google/gemma-3n-e4b-it:free", - "object": "model", - "created": 1774530909679, - "owned_by": "google", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "google/gemma-3n-e4b-it", - "object": "model", - "created": 1774530909679, - "owned_by": "google", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "mistralai/mistral-medium-3", - "object": "model", - "created": 1774530909679, - "owned_by": "mistral.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.012621, - "output": 0.063104, - "unit": "per_million_tokens" - } - }, - { - "id": "google/gemini-2.5-pro-preview-05-06", - "object": "model", - "created": 1774530909679, - "owned_by": "google", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input", - "tool_calling" - ], - "pricing": { - "input": 0.020193, - "output": 0.161547, - "per_image": 2.4e-7, - "unit": "per_million_tokens" - } - }, - { - "id": "arcee-ai/spotlight", - "object": "model", - "created": 1774530909679, - "owned_by": "arcee", - "providers": 8, - "throughput": 50, - "pricing": { - "input": 0.018174, - "output": 0.018174, - "unit": "per_million_tokens" - } - }, - { - "id": "arcee-ai/maestro-reasoning", - "object": "model", - "created": 1774530909679, - "owned_by": "arcee", - "providers": 8, - "throughput": 50, - "pricing": { - "input": 0.016227, - "output": 0.059498, - "unit": "per_million_tokens" - } - }, - { - "id": "arcee-ai/virtuoso-large", - "object": "model", - "created": 1774530909679, - "owned_by": "arcee", - "providers": 8, - "throughput": 50, - "pricing": { - "input": 0.017475, - "output": 0.02796, - "unit": "per_million_tokens" - } - }, - { - "id": "arcee-ai/coder-large", - "object": "model", - "created": 1774530909679, - "owned_by": "arcee", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.017475, - "output": 0.02796, - "unit": "per_million_tokens" - } - }, - { - "id": "inception/mercury-coder", - "object": "model", - "created": 1774530909679, - "owned_by": "inception", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.009087, - "output": 0.027261, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-4b:free", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "meta-llama/llama-guard-4-12b", - "object": "model", - "created": 1774530909679, - "owned_by": "meta", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.018174, - "output": 0.018174, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-30b-a3b", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.008077, - "output": 0.028271, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-8b", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.004039, - "output": 0.032309, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-14b", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-32b", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.009087, - "output": 0.027261, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen3-235b-a22b", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.015145, - "output": 0.06058, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/o4-mini-high", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.015145, - "output": 0.06058, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/o3", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.15145, - "output": 0.6058, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/o4-mini", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.015145, - "output": 0.06058, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen2.5-coder-7b-instruct", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.009087, - "output": 0.027261, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-4.1", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.036348, - "output": 0.145392, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-4.1-mini", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.015145, - "output": 0.06058, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-4.1-nano", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "eleutherai/llemma_7b", - "object": "model", - "created": 1774530909679, - "owned_by": "eleutherai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.018174, - "output": 0.027261, - "unit": "per_million_tokens" - } - }, - { - "id": "alfredpros/codellama-7b-instruct-solidity", - "object": "model", - "created": 1774530909679, - "owned_by": "alfredpros", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.018174, - "output": 0.027261, - "unit": "per_million_tokens" - } - }, - { - "id": "x-ai/grok-3-mini-beta", - "object": "model", - "created": 1774530909679, - "owned_by": "xai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.013631, - "output": 0.022718, - "unit": "per_million_tokens" - } - }, - { - "id": "x-ai/grok-3-beta", - "object": "model", - "created": 1774530909679, - "owned_by": "xai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.06058, - "output": 0.3029, - "unit": "per_million_tokens" - } - }, - { - "id": "nvidia/llama-3.1-nemotron-ultra-253b-v1", - "object": "model", - "created": 1774530909679, - "owned_by": "nvidia", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.018931, - "output": 0.056794, - "unit": "per_million_tokens" - } - }, - { - "id": "meta-llama/llama-4-maverick", - "object": "model", - "created": 1774530909679, - "owned_by": "meta", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "meta-llama/llama-4-scout", - "object": "model", - "created": 1774530909679, - "owned_by": "meta", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.007652, - "output": 0.028696, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen2.5-vl-32b-instruct", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.009087, - "output": 0.027261, - "unit": "per_million_tokens" - } - }, - { - "id": "deepseek/deepseek-chat-v3-0324", - "object": "model", - "created": 1774530909679, - "owned_by": "deepseek", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.007494, - "output": 0.028854, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/o1-pro", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.18174, - "output": 0.72696, - "unit": "per_million_tokens" - } - }, - { - "id": "mistralai/mistral-small-3.1-24b-instruct:free", - "object": "model", - "created": 1774530909679, - "owned_by": "mistral.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "mistralai/mistral-small-3.1-24b-instruct", - "object": "model", - "created": 1774530909679, - "owned_by": "mistral.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output" - ], - "pricing": { - "input": 0.007789, - "output": 0.028559, - "unit": "per_million_tokens" - } - }, - { - "id": "allenai/olmo-2-0325-32b-instruct", - "object": "model", - "created": 1774530909679, - "owned_by": "allenai", - "providers": 8, - "throughput": 50, - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "google/gemma-3-4b-it:free", - "object": "model", - "created": 1774530909679, - "owned_by": "google", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "google/gemma-3-4b-it", - "object": "model", - "created": 1774530909679, - "owned_by": "google", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "google/gemma-3-12b-it:free", - "object": "model", - "created": 1774530909679, - "owned_by": "google", - "providers": 8, - "throughput": 50, - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "google/gemma-3-12b-it", - "object": "model", - "created": 1774530909679, - "owned_by": "google", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "cohere/command-a", - "object": "model", - "created": 1774530909679, - "owned_by": "cohere", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.036348, - "output": 0.145392, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-4o-mini-search-preview", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-4o-search-preview", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.036348, - "output": 0.145392, - "unit": "per_million_tokens" - } - }, - { - "id": "google/gemma-3-27b-it:free", - "object": "model", - "created": 1774530909679, - "owned_by": "google", - "providers": 8, - "throughput": 50, - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "google/gemma-3-27b-it", - "object": "model", - "created": 1774530909679, - "owned_by": "google", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "thedrummer/skyfall-36b-v2", - "object": "model", - "created": 1774530909679, - "owned_by": "thedrummer", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.018511, - "output": 0.026924, - "unit": "per_million_tokens" - } - }, - { - "id": "perplexity/sonar-reasoning-pro", - "object": "model", - "created": 1774530909679, - "owned_by": "perplexity", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.036348, - "output": 0.145392, - "unit": "per_million_tokens" - } - }, - { - "id": "perplexity/sonar-pro", - "object": "model", - "created": 1774530909679, - "owned_by": "perplexity", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.06058, - "output": 0.3029, - "unit": "per_million_tokens" - } - }, - { - "id": "perplexity/sonar-deep-research", - "object": "model", - "created": 1774530909679, - "owned_by": "perplexity", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.036348, - "output": 0.145392, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwq-32b", - "object": "model", - "created": 1774530909679, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.007469, - "output": 0.028879, - "unit": "per_million_tokens" - } - }, - { - "id": "google/gemini-2.0-flash-lite-001", - "object": "model", - "created": 1774530909679, - "owned_by": "google", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input", - "tool_calling" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "anthropic/claude-3.7-sonnet", - "object": "model", - "created": 1774530909679, - "owned_by": "anthropic", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.06058, - "output": 0.3029, - "unit": "per_million_tokens" - } - }, - { - "id": "anthropic/claude-3.7-sonnet:thinking", - "object": "model", - "created": 1774530909679, - "owned_by": "anthropic", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.06058, - "output": 0.3029, - "unit": "per_million_tokens" - } - }, - { - "id": "mistralai/mistral-saba", - "object": "model", - "created": 1774530909679, - "owned_by": "mistral.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.009087, - "output": 0.027261, - "unit": "per_million_tokens" - } - }, - { - "id": "meta-llama/llama-guard-3-8b", - "object": "model", - "created": 1774530909679, - "owned_by": "meta", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.009087, - "output": 0.027261, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/o3-mini-high", - "object": "model", - "created": 1774530909679, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.015145, - "output": 0.06058, - "unit": "per_million_tokens" - } - }, - { - "id": "google/gemini-2.0-flash-001", - "object": "model", - "created": 1774530909679, - "owned_by": "google", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "audio_input", - "tool_calling" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen-vl-plus", - "object": "model", - "created": 1774530909680, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.009087, - "output": 0.027261, - "unit": "per_million_tokens" - } - }, - { - "id": "aion-labs/aion-1.0", - "object": "model", - "created": 1774530909680, - "owned_by": "aion-labs", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.06058, - "output": 0.12116, - "unit": "per_million_tokens" - } - }, - { - "id": "aion-labs/aion-1.0-mini", - "object": "model", - "created": 1774530909680, - "owned_by": "aion-labs", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.015145, - "output": 0.03029, - "unit": "per_million_tokens" - } - }, - { - "id": "aion-labs/aion-rp-llama-3.1-8b", - "object": "model", - "created": 1774530909680, - "owned_by": "aion-labs", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.025242, - "output": 0.050483, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen-vl-max", - "object": "model", - "created": 1774530909680, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.015145, - "output": 0.06058, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen-turbo", - "object": "model", - "created": 1774530909680, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.007829, - "output": 0.031315, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen2.5-vl-72b-instruct", - "object": "model", - "created": 1774530909680, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.022718, - "output": 0.022718, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen-plus", - "object": "model", - "created": 1774530909680, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.009087, - "output": 0.027261, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen-max", - "object": "model", - "created": 1774530909680, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.015145, - "output": 0.06058, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/o3-mini", - "object": "model", - "created": 1774530909680, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.015145, - "output": 0.06058, - "unit": "per_million_tokens" - } - }, - { - "id": "mistralai/mistral-small-24b-instruct-2501", - "object": "model", - "created": 1774530909680, - "owned_by": "mistral.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.01398, - "output": 0.022368, - "unit": "per_million_tokens" - } - }, - { - "id": "deepseek/deepseek-r1-distill-qwen-32b", - "object": "model", - "created": 1774530909680, - "owned_by": "deepseek", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.018174, - "output": 0.018174, - "unit": "per_million_tokens" - } - }, - { - "id": "perplexity/sonar", - "object": "model", - "created": 1774530909680, - "owned_by": "perplexity", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.022718, - "output": 0.022718, - "unit": "per_million_tokens" - } - }, - { - "id": "deepseek/deepseek-r1-distill-llama-70b", - "object": "model", - "created": 1774530909680, - "owned_by": "deepseek", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.021203, - "output": 0.024232, - "unit": "per_million_tokens" - } - }, - { - "id": "deepseek/deepseek-r1", - "object": "model", - "created": 1774530909680, - "owned_by": "deepseek", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "minimax/minimax-01", - "object": "model", - "created": 1774530909680, - "owned_by": "minimax", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.00699, - "output": 0.038445, - "unit": "per_million_tokens" - } - }, - { - "id": "microsoft/phi-4", - "object": "model", - "created": 1774530909680, - "owned_by": "microsoft", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.011525, - "output": 0.024823, - "unit": "per_million_tokens" - } - }, - { - "id": "sao10k/l3.1-70b-hanami-x1", - "object": "model", - "created": 1774530909680, - "owned_by": "sao10k", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.037863, - "output": 0.037863, - "unit": "per_million_tokens" - } - }, - { - "id": "deepseek/deepseek-chat", - "object": "model", - "created": 1774530909680, - "owned_by": "deepseek", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.008954, - "output": 0.036481, - "unit": "per_million_tokens" - } - }, - { - "id": "sao10k/l3.3-euryale-70b", - "object": "model", - "created": 1774530909680, - "owned_by": "sao10k", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.021095, - "output": 0.02434, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/o1", - "object": "model", - "created": 1774530909680, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.15145, - "output": 0.6058, - "unit": "per_million_tokens" - } - }, - { - "id": "cohere/command-r7b-12-2024", - "object": "model", - "created": 1774530909680, - "owned_by": "cohere", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "meta-llama/llama-3.3-70b-instruct:free", - "object": "model", - "created": 1774530909680, - "owned_by": "meta", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "meta-llama/llama-3.3-70b-instruct", - "object": "model", - "created": 1774530909680, - "owned_by": "meta", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.008654, - "output": 0.027694, - "unit": "per_million_tokens" - } - }, - { - "id": "amazon/nova-lite-v1", - "object": "model", - "created": 1774530909680, - "owned_by": "amazon", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "amazon/nova-micro-v1", - "object": "model", - "created": 1774530909680, - "owned_by": "amazon", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "amazon/nova-pro-v1", - "object": "model", - "created": 1774530909680, - "owned_by": "amazon", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.015145, - "output": 0.06058, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-4o-2024-11-20", - "object": "model", - "created": 1774530909680, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.036348, - "output": 0.145392, - "unit": "per_million_tokens" - } - }, - { - "id": "mistralai/mistral-large-2411", - "object": "model", - "created": 1774530909680, - "owned_by": "mistral.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.045435, - "output": 0.136305, - "unit": "per_million_tokens" - } - }, - { - "id": "mistralai/mistral-large-2407", - "object": "model", - "created": 1774530909680, - "owned_by": "mistral.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.045435, - "output": 0.136305, - "unit": "per_million_tokens" - } - }, - { - "id": "mistralai/pixtral-large-2411", - "object": "model", - "created": 1774530909680, - "owned_by": "mistral.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.045435, - "output": 0.136305, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen-2.5-coder-32b-instruct", - "object": "model", - "created": 1774530909680, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.018065, - "output": 0.02737, - "unit": "per_million_tokens" - } - }, - { - "id": "thedrummer/unslopnemo-12b", - "object": "model", - "created": 1774530909680, - "owned_by": "thedrummer", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.018174, - "output": 0.018174, - "unit": "per_million_tokens" - } - }, - { - "id": "anthropic/claude-3.5-haiku", - "object": "model", - "created": 1774530909680, - "owned_by": "anthropic", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.012621, - "output": 0.063104, - "unit": "per_million_tokens" - } - }, - { - "id": "anthropic/claude-3.5-sonnet", - "object": "model", - "created": 1774530909680, - "owned_by": "anthropic", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.06058, - "output": 0.3029, - "unit": "per_million_tokens" - } - }, - { - "id": "anthracite-org/magnum-v4-72b", - "object": "model", - "created": 1774530909680, - "owned_by": "anthracite-org", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.068153, - "output": 0.113588, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen-2.5-7b-instruct", - "object": "model", - "created": 1774530909680, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.011184, - "output": 0.02796, - "unit": "per_million_tokens" - } - }, - { - "id": "nvidia/llama-3.1-nemotron-70b-instruct", - "object": "model", - "created": 1774530909680, - "owned_by": "nvidia", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.037863, - "output": 0.037863, - "unit": "per_million_tokens" - } - }, - { - "id": "inflection/inflection-3-productivity", - "object": "model", - "created": 1774530909680, - "owned_by": "inflection", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.036348, - "output": 0.145392, - "unit": "per_million_tokens" - } - }, - { - "id": "inflection/inflection-3-pi", - "object": "model", - "created": 1774530909680, - "owned_by": "inflection", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.036348, - "output": 0.145392, - "unit": "per_million_tokens" - } - }, - { - "id": "thedrummer/rocinante-12b", - "object": "model", - "created": 1774530909680, - "owned_by": "thedrummer", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.010299, - "output": 0.026049, - "unit": "per_million_tokens" - } - }, - { - "id": "meta-llama/llama-3.2-3b-instruct:free", - "object": "model", - "created": 1774530909680, - "owned_by": "meta", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "meta-llama/llama-3.2-3b-instruct", - "object": "model", - "created": 1774530909680, - "owned_by": "meta", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.004741, - "output": 0.031607, - "unit": "per_million_tokens" - } - }, - { - "id": "meta-llama/llama-3.2-1b-instruct", - "object": "model", - "created": 1774530909680, - "owned_by": "meta", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.004323, - "output": 0.032025, - "unit": "per_million_tokens" - } - }, - { - "id": "meta-llama/llama-3.2-11b-vision-instruct", - "object": "model", - "created": 1774530909680, - "owned_by": "meta", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input" - ], - "pricing": { - "input": 0.018174, - "output": 0.018174, - "unit": "per_million_tokens" - } - }, - { - "id": "qwen/qwen-2.5-72b-instruct", - "object": "model", - "created": 1774530909680, - "owned_by": "alibaba", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.008552, - "output": 0.027796, - "unit": "per_million_tokens" - } - }, - { - "id": "cohere/command-r-plus-08-2024", - "object": "model", - "created": 1774530909680, - "owned_by": "cohere", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.036348, - "output": 0.145392, - "unit": "per_million_tokens" - } - }, - { - "id": "cohere/command-r-08-2024", - "object": "model", - "created": 1774530909680, - "owned_by": "cohere", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "sao10k/l3.1-euryale-70b", - "object": "model", - "created": 1774530909680, - "owned_by": "sao10k", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.022718, - "output": 0.022718, - "unit": "per_million_tokens" - } - }, - { - "id": "nousresearch/hermes-3-llama-3.1-70b", - "object": "model", - "created": 1774530909680, - "owned_by": "nous", - "providers": 8, - "throughput": 50, - "pricing": { - "input": 0.018174, - "output": 0.018174, - "unit": "per_million_tokens" - } - }, - { - "id": "nousresearch/hermes-3-llama-3.1-405b:free", - "object": "model", - "created": 1774530909680, - "owned_by": "nous", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "nousresearch/hermes-3-llama-3.1-405b", - "object": "model", - "created": 1774530909680, - "owned_by": "nous", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.022718, - "output": 0.022718, - "unit": "per_million_tokens" - } - }, - { - "id": "sao10k/l3-lunaris-8b", - "object": "model", - "created": 1774530909680, - "owned_by": "sao10k", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.016155, - "output": 0.020193, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-4o-2024-08-06", - "object": "model", - "created": 1774530909680, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.036348, - "output": 0.145392, - "unit": "per_million_tokens" - } - }, - { - "id": "meta-llama/llama-3.1-8b-instruct", - "object": "model", - "created": 1774530909680, - "owned_by": "meta", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.010385, - "output": 0.025963, - "unit": "per_million_tokens" - } - }, - { - "id": "meta-llama/llama-3.1-70b-instruct", - "object": "model", - "created": 1774530909680, - "owned_by": "meta", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.018174, - "output": 0.018174, - "unit": "per_million_tokens" - } - }, - { - "id": "mistralai/mistral-nemo", - "object": "model", - "created": 1774530909680, - "owned_by": "mistral.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.012116, - "output": 0.024232, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-4o-mini-2024-07-18", - "object": "model", - "created": 1774530909680, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-4o-mini", - "object": "model", - "created": 1774530909680, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.00727, - "output": 0.029078, - "unit": "per_million_tokens" - } - }, - { - "id": "google/gemma-2-27b-it", - "object": "model", - "created": 1774530909680, - "owned_by": "google", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.022718, - "output": 0.022718, - "unit": "per_million_tokens" - } - }, - { - "id": "google/gemma-2-9b-it", - "object": "model", - "created": 1774530909680, - "owned_by": "google", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.009087, - "output": 0.027261, - "unit": "per_million_tokens" - } - }, - { - "id": "sao10k/l3-euryale-70b", - "object": "model", - "created": 1774530909680, - "owned_by": "sao10k", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.037863, - "output": 0.037863, - "unit": "per_million_tokens" - } - }, - { - "id": "nousresearch/hermes-2-pro-llama-3-8b", - "object": "model", - "created": 1774530909680, - "owned_by": "nous", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.018174, - "output": 0.018174, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-4o-2024-05-13", - "object": "model", - "created": 1774530909680, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.09087, - "output": 0.27261, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-4o", - "object": "model", - "created": 1774530909680, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.036348, - "output": 0.145392, - "image_input": 0.0003826, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-4o:extended", - "object": "model", - "created": 1774530909680, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.09087, - "output": 0.27261, - "unit": "per_million_tokens" - } - }, - { - "id": "meta-llama/llama-3-8b-instruct", - "object": "model", - "created": 1774530909680, - "owned_by": "meta", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.015578, - "output": 0.02077, - "unit": "per_million_tokens" - } - }, - { - "id": "meta-llama/llama-3-70b-instruct", - "object": "model", - "created": 1774530909680, - "owned_by": "meta", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.018537, - "output": 0.026898, - "unit": "per_million_tokens" - } - }, - { - "id": "mistralai/mixtral-8x22b-instruct", - "object": "model", - "created": 1774530909680, - "owned_by": "mistral.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.045435, - "output": 0.136305, - "unit": "per_million_tokens" - } - }, - { - "id": "microsoft/wizardlm-2-8x22b", - "object": "model", - "created": 1774530909680, - "owned_by": "microsoft", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.022718, - "output": 0.022718, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-4-turbo", - "object": "model", - "created": 1774530909680, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.189313, - "output": 0.567938, - "unit": "per_million_tokens" - } - }, - { - "id": "anthropic/claude-3-haiku", - "object": "model", - "created": 1774530909680, - "owned_by": "anthropic", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "tool_calling" - ], - "pricing": { - "input": 0.007573, - "output": 0.037863, - "unit": "per_million_tokens" - } - }, - { - "id": "mistralai/mistral-large", - "object": "model", - "created": 1774530909680, - "owned_by": "mistral.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.045435, - "output": 0.136305, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-4-turbo-preview", - "object": "model", - "created": 1774530909680, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.189313, - "output": 0.567938, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-3.5-turbo-0613", - "object": "model", - "created": 1774530909680, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.018931, - "output": 0.056794, - "unit": "per_million_tokens" - } - }, - { - "id": "mistralai/mixtral-8x7b-instruct", - "object": "model", - "created": 1774530909680, - "owned_by": "mistral.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.022718, - "output": 0.022718, - "unit": "per_million_tokens" - } - }, - { - "id": "alpindale/goliath-120b", - "object": "model", - "created": 1774530909680, - "owned_by": "alpindale", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.06058, - "output": 0.12116, - "unit": "per_million_tokens" - } - }, - { - "id": "openrouter/auto", - "object": "model", - "created": 1774530909680, - "owned_by": "openrouter", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output", - "audio_input", - "tool_calling" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-4-1106-preview", - "object": "model", - "created": 1774530909680, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.189313, - "output": 0.567938, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-3.5-turbo-instruct", - "object": "model", - "created": 1774530909680, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "pricing": { - "input": 0.032454, - "output": 0.043271, - "unit": "per_million_tokens" - } - }, - { - "id": "mistralai/mistral-7b-instruct-v0.1", - "object": "model", - "created": 1774530909680, - "owned_by": "mistral.ai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.013328, - "output": 0.02302, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-3.5-turbo-16k", - "object": "model", - "created": 1774530909680, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.077889, - "output": 0.103851, - "unit": "per_million_tokens" - } - }, - { - "id": "mancer/weaver", - "object": "model", - "created": 1774530909680, - "owned_by": "mancer", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.019472, - "output": 0.025963, - "unit": "per_million_tokens" - } - }, - { - "id": "undi95/remm-slerp-l2-13b", - "object": "model", - "created": 1774530909680, - "owned_by": "undi95", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.018587, - "output": 0.026848, - "unit": "per_million_tokens" - } - }, - { - "id": "gryphe/mythomax-l2-13b", - "object": "model", - "created": 1774530909680, - "owned_by": "gryphe", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.018174, - "output": 0.018174, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-4-0314", - "object": "model", - "created": 1774530909680, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text" - ], - "pricing": { - "input": 0.252417, - "output": 0.504833, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-4", - "object": "model", - "created": 1774530909680, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.252417, - "output": 0.504833, - "unit": "per_million_tokens" - } - }, - { - "id": "openai/gpt-3.5-turbo", - "object": "model", - "created": 1774530909680, - "owned_by": "openai", - "providers": 8, - "throughput": 50, - "capabilities": [ - "text", - "tool_calling" - ], - "pricing": { - "input": 0.018931, - "output": 0.056794, - "unit": "per_million_tokens" - } - }, - { - "id": "grok-3", - "object": "model", - "created": 1774530909680, - "owned_by": "xai", - "providers": 1, - "throughput": 50, - "capabilities": [ - "text", - "image_output", - "audio_input", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "grok-3-mini", - "object": "model", - "created": 1774530909680, - "owned_by": "xai", - "providers": 1, - "throughput": 5, - "capabilities": [ - "text", - "image_output", - "audio_input", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "grok-4-0709", - "object": "model", - "created": 1774530909680, - "owned_by": "xai", - "providers": 1, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output", - "audio_input", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "grok-4-1-fast-non-reasoning", - "object": "model", - "created": 1774530909680, - "owned_by": "xai", - "providers": 1, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output", - "audio_input", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "grok-4-1-fast-reasoning", - "object": "model", - "created": 1774530909680, - "owned_by": "xai", - "providers": 1, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output", - "audio_input", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "grok-4-fast-non-reasoning", - "object": "model", - "created": 1774530909680, - "owned_by": "xai", - "providers": 1, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output", - "audio_input", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "grok-4-fast-reasoning", - "object": "model", - "created": 1774530909680, - "owned_by": "xai", - "providers": 1, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output", - "audio_input", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "grok-4.20-0309-non-reasoning", - "object": "model", - "created": 1774530909680, - "owned_by": "xai", - "providers": 1, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "grok-4.20-0309-reasoning", - "object": "model", - "created": 1774530909680, - "owned_by": "xai", - "providers": 1, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "grok-4.20-multi-agent-0309", - "object": "model", - "created": 1774530909680, - "owned_by": "xai", - "providers": 1, - "throughput": 50, - "capabilities": [ - "text", - "image_input", - "image_output", - "audio_output" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "grok-code-fast-1", - "object": "model", - "created": 1774530909680, - "owned_by": "xai", - "providers": 1, - "throughput": 50, - "capabilities": [ - "text", - "image_output", - "audio_input", - "audio_output", - "tool_calling" - ], - "pricing": { - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "grok-imagine-image", - "object": "model", - "created": 1774530909680, - "owned_by": "xai", - "providers": 1, - "throughput": 50, - "capabilities": [ - "text", - "image_output" - ], - "pricing": { - "per_image": 0.014, - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "grok-imagine-image-pro", - "object": "model", - "created": 1774530909680, - "owned_by": "xai", - "providers": 1, - "throughput": 50, - "capabilities": [ - "text", - "image_output" - ], - "pricing": { - "per_image": 0.028, - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - }, - { - "id": "grok-imagine-video", - "object": "model", - "created": 1774530909680, - "owned_by": "xai", - "providers": 1, - "throughput": 50, - "capabilities": [ - "text", - "image_output" - ], - "pricing": { - "per_request": 0.1, - "input": 0, - "output": 0, - "unit": "per_million_tokens" - } - } - ] -} \ No newline at end of file diff --git a/apps/api/dev/runTestServer.sh b/apps/api/dev/runTestServer.sh index 96f6ff5..e124786 100644 --- a/apps/api/dev/runTestServer.sh +++ b/apps/api/dev/runTestServer.sh @@ -10,9 +10,10 @@ trap cleanup EXIT INT TERM export NODE_ENV="${NODE_ENV:-test}" export DATA_SOURCE_PREFERENCE="${DATA_SOURCE_PREFERENCE:-filesystem}" export DATA_CACHE_TTL_MS="${DATA_CACHE_TTL_MS:-600000}" -export API_PROVIDERS_FILE="${API_PROVIDERS_FILE:-.test-data/providers.json}" -export API_KEYS_FILE="${API_KEYS_FILE:-.test-data/keys.json}" -export API_MODELS_FILE="${API_MODELS_FILE:-.test-data/models.json}" +export ALLOW_MOCK_PROVIDERS="${ALLOW_MOCK_PROVIDERS:-1}" +export API_PROVIDERS_FILE="${API_PROVIDERS_FILE:-.tmp/test-data/providers.json}" +export API_KEYS_FILE="${API_KEYS_FILE:-.tmp/test-data/keys.json}" +export API_MODELS_FILE="${API_MODELS_FILE:-.tmp/test-data/models.json}" export API_TIERS_FILE="${API_TIERS_FILE:-.test-data/tiers.json}" export PORT="${TEST_DEV_PORT:-3100}" export CLUSTER_WORKERS="${TEST_CLUSTER_WORKERS:-0}" diff --git a/apps/api/dev/testApi.ts b/apps/api/dev/testApi.ts index 12fea8c..ef8f1ce 100644 --- a/apps/api/dev/testApi.ts +++ b/apps/api/dev/testApi.ts @@ -5,7 +5,7 @@ import path from 'path'; const envFile = process.env.NODE_ENV === 'test' ? '.env.test' : '.env'; dotenv.config({ path: path.resolve(process.cwd(), envFile) }); -import { setupMockProviderConfig, restoreProviderConfig } from './testSetup.js'; +import { DEFAULT_TEST_API_KEY, setupMockProviderConfig, restoreProviderConfig } from './testSetup.js'; // Load environment variables (optional, server should have them) @@ -161,8 +161,8 @@ async function testApiWithMockProvider() { const apiUrl = `${baseUrl}/v1/chat/completions`; const modelId = 'gpt-3.5-turbo'; const testPrompt = 'Write a short haiku about APIs.'; - // Use an existing admin API key that we know is valid - const apiKey = process.env.TEST_API_KEY || 'test-key-for-mock-provider'; + // Use the standard automated test API key unless the environment overrides it. + const apiKey = process.env.TEST_API_KEY || DEFAULT_TEST_API_KEY; console.log(`[TEST] Testing API endpoint: ${apiUrl}`); console.log(`[TEST] Using model: ${modelId}`); diff --git a/apps/api/dev/testOpenAISdk.ts b/apps/api/dev/testOpenAISdk.ts index 0c9c440..1d63994 100644 --- a/apps/api/dev/testOpenAISdk.ts +++ b/apps/api/dev/testOpenAISdk.ts @@ -3,15 +3,20 @@ import dotenv from 'dotenv'; import path from 'path'; import OpenAI from 'openai'; -import { setupMockProviderConfig, restoreProviderConfig } from './testSetup.js'; +import { DEFAULT_TEST_API_KEY, setupMockProviderConfig, restoreProviderConfig } from './testSetup.js'; const envFile = process.env.NODE_ENV === 'test' ? '.env.test' : '.env'; dotenv.config({ path: path.resolve(process.cwd(), envFile) }); -function resolveClientBaseUrl(): string { +function resolveRawApiBaseUrl(): string { const port = process.env.PORT || '3000'; const rawBaseUrl = process.env.TEST_API_BASE_URL || `http://localhost:${port}`; const normalized = rawBaseUrl.replace(/\/$/, ''); + return normalized.endsWith('/v1') ? normalized.slice(0, -3) : normalized; +} + +function resolveClientBaseUrl(): string { + const normalized = resolveRawApiBaseUrl(); return normalized.endsWith('/v1') ? normalized : `${normalized}/v1`; } @@ -38,17 +43,24 @@ function getReasoningSummaryText(item: any): string | undefined { return typeof firstSummaryText?.text === 'string' ? firstSummaryText.text : undefined; } +function assertNativeProxyResponseId(responseId: string): void { + assert.match(responseId, /^resp_[a-f0-9]{32}$/); +} + async function runSdkCompatibilityTest() { const manageSetup = process.env.TEST_SETUP_MODE !== 'external'; if (manageSetup) { setupMockProviderConfig(); } - const apiKey = process.env.TEST_API_KEY || 'test-key-for-mock-provider'; + const apiKey = process.env.TEST_API_KEY || DEFAULT_TEST_API_KEY; const model = 'gpt-3.5-turbo'; const responsesModel = 'gpt-5.4'; + const rawBaseUrl = resolveRawApiBaseUrl(); const baseURL = resolveClientBaseUrl(); + const nativeResponsesBaseUrl = `${rawBaseUrl}/native/auto/v1`; const client = new OpenAI({ apiKey, baseURL }); + const nativeResponsesClient = new OpenAI({ apiKey, baseURL: nativeResponsesBaseUrl }); console.log(`[SDK-TEST] Testing OpenAI Node SDK compatibility against ${baseURL}`); @@ -307,6 +319,78 @@ async function runSdkCompatibilityTest() { assert.ok(Array.isArray(completedToolAssistantMessages[0]?.content)); assert.ok(completedToolAssistantMessages[0].content.some((part: any) => part?.type === 'tool_calls')); console.log('[SDK-TEST] ✅ Streaming responses.create tool calls include an assistant message in the completed response.'); + + const nativeResponseObject = await nativeResponsesClient.responses.create({ + model: responsesModel, + input: 'Say hello from the responses API.', + }); + + assert.equal(nativeResponseObject.object, 'response'); + assert.equal(nativeResponseObject.status, 'completed'); + assert.equal(nativeResponseObject.output_text, expectedHelloResponseText); + assert.ok(typeof nativeResponseObject.id === 'string'); + assertNativeProxyResponseId(nativeResponseObject.id); + console.log('[SDK-TEST] ✅ Native responses.create rewrites response ids to proxy-owned ids.'); + + const nativeFollowupInput = 'Say hello again from native responses.'; + const expectedNativeFollowupText = getExpectedMockResponsesText(nativeFollowupInput); + const nativeFollowup = await nativeResponsesClient.responses.create({ + model: responsesModel, + input: nativeFollowupInput, + previous_response_id: nativeResponseObject.id, + }); + + assert.equal(nativeFollowup.object, 'response'); + assert.equal(nativeFollowup.status, 'completed'); + assert.ok(typeof nativeFollowup.id === 'string' && nativeFollowup.id !== nativeResponseObject.id); + assertNativeProxyResponseId(nativeFollowup.id); + assert.equal(nativeFollowup.output_text, expectedNativeFollowupText); + console.log('[SDK-TEST] ✅ Native responses.create resolves previous_response_id locally.'); + + const nativeResponseStream = await nativeResponsesClient.responses.create({ + model: responsesModel, + input: streamingResponsesInput, + stream: true, + }); + + const nativeStreamResponseIds = new Set(); + let completedNativeStreamResponse: any; + for await (const event of nativeResponseStream as AsyncIterable) { + if (typeof event?.response_id === 'string') { + nativeStreamResponseIds.add(event.response_id); + } + if (typeof event?.response?.id === 'string') { + nativeStreamResponseIds.add(event.response.id); + } + if (event?.type === 'response.completed') { + completedNativeStreamResponse = event.response; + } + } + + assert.ok(completedNativeStreamResponse); + assert.equal(nativeStreamResponseIds.size, 1); + const [nativeStreamResponseId] = Array.from(nativeStreamResponseIds); + assert.ok(typeof nativeStreamResponseId === 'string'); + assertNativeProxyResponseId(nativeStreamResponseId); + assert.equal(completedNativeStreamResponse?.id, nativeStreamResponseId); + console.log('[SDK-TEST] ✅ Native streaming responses keep a consistent proxy-owned response id.'); + + const missingNativeResponse = await fetch(`${rawBaseUrl}/native/auto/v1/responses`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify({ + model: responsesModel, + input: 'This request should fail.', + previous_response_id: 'resp_missing_history_id', + }), + }); + const missingNativeResponseBody = await missingNativeResponse.json(); + assert.equal(missingNativeResponse.status, 400); + assert.ok(String(missingNativeResponseBody?.error || '').includes("Previous response with id 'resp_missing_history_id' not found.")); + console.log('[SDK-TEST] ✅ Native responses reject unresolved previous_response_id values.'); } finally { if (manageSetup) { restoreProviderConfig(); diff --git a/apps/api/dev/testSetup.ts b/apps/api/dev/testSetup.ts index 28a91b9..c56599d 100644 --- a/apps/api/dev/testSetup.ts +++ b/apps/api/dev/testSetup.ts @@ -5,6 +5,11 @@ import { fileURLToPath } from 'url'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const apiRoot = path.resolve(__dirname, '..'); +const testDataRoot = path.join(apiRoot, '.tmp', 'test-data'); +const DEFAULT_TEST_MODELS_CREATED_AT = 1_700_000_000; + +export const DEFAULT_TEST_API_KEY = 'test-key-for-automated-testing-0123456789abcdef'; +const LEGACY_TEST_API_KEY = 'test-key-for-mock-provider'; function resolveTestDataPath(envVarName: string, defaultPath: string): string { const override = process.env[envVarName]; @@ -15,62 +20,58 @@ function resolveTestDataPath(envVarName: string, defaultPath: string): string { } // Create a test configuration that sets up providers.json to use the mock provider -export function setupMockProviderConfig() { +export function setupMockProviderConfig(mode: 'openai' | 'anthropic' = 'openai') { const mockPort = Number(process.env.MOCK_PROVIDER_PORT || '3001'); - const testApiKey = process.env.TEST_API_KEY || 'test-key-for-mock-provider'; + const testApiKey = process.env.TEST_API_KEY || DEFAULT_TEST_API_KEY; const hashSecret = process.env.API_KEY_HASH_SECRET || 'anygpt-api'; const hashIterations = Number(process.env.API_KEY_HASH_ITERATIONS || '20000'); const hashKeylen = Number(process.env.API_KEY_HASH_KEYLEN || '32'); const hashDigest = 'sha256'; - const providersFilePath = resolveTestDataPath('API_PROVIDERS_FILE', path.join(apiRoot, 'providers.json')); - const keysFilePath = resolveTestDataPath('API_KEYS_FILE', path.join(apiRoot, 'keys.json')); - const modelsFilePath = resolveTestDataPath('API_MODELS_FILE', path.join(apiRoot, 'models.json')); + const providersFilePath = resolveTestDataPath('API_PROVIDERS_FILE', path.join(testDataRoot, 'providers.json')); + const keysFilePath = resolveTestDataPath('API_KEYS_FILE', path.join(testDataRoot, 'keys.json')); + const modelsFilePath = resolveTestDataPath('API_MODELS_FILE', path.join(testDataRoot, 'models.json')); const backupProvidersPath = `${providersFilePath}.backup`; const backupKeysPath = `${keysFilePath}.backup`; const backupModelsPath = `${modelsFilePath}.backup`; - - // Try to preserve existing response times and stats - let existingProvider = null; - if (fs.existsSync(providersFilePath)) { - try { - const existingProviders = JSON.parse(fs.readFileSync(providersFilePath, 'utf8')); - existingProvider = existingProviders.find((p: any) => p.id === 'openai-mock'); - } catch (error) { - console.log('[TEST-SETUP] Could not parse existing providers.json, starting fresh'); - } - } + + const providerId = mode === 'anthropic' ? 'claude-mock' : 'openai-mock'; + const primaryModelId = mode === 'anthropic' ? 'claude-3-5-sonnet' : 'gpt-3.5-turbo'; + const secondaryModelId = mode === 'anthropic' ? 'claude-3-7-sonnet' : 'gpt-5.4'; + const providerUrl = mode === 'anthropic' + ? `http://localhost:${mockPort}/v1/messages` + : `http://localhost:${mockPort}/v1/chat/completions`; const mockProvider = { - id: 'openai-mock', // Use existing provider ID to override it + id: providerId, apiKey: 'mock-api-key-for-testing', - provider_url: `http://localhost:${mockPort}/v1/chat/completions`, // Point to our mock - streamingCompatible: true, // Mock provider supports streaming + provider_url: providerUrl, + streamingCompatible: true, models: { - 'gpt-3.5-turbo': { - id: 'gpt-3.5-turbo', - token_generation_speed: existingProvider?.models?.['gpt-3.5-turbo']?.token_generation_speed || 50, - response_times: existingProvider?.models?.['gpt-3.5-turbo']?.response_times || [], - errors: existingProvider?.models?.['gpt-3.5-turbo']?.errors || 0, - consecutive_errors: existingProvider?.models?.['gpt-3.5-turbo']?.consecutive_errors || 0, - avg_response_time: existingProvider?.models?.['gpt-3.5-turbo']?.avg_response_time || null, - avg_provider_latency: existingProvider?.models?.['gpt-3.5-turbo']?.avg_provider_latency || null, - avg_token_speed: existingProvider?.models?.['gpt-3.5-turbo']?.avg_token_speed || null + [primaryModelId]: { + id: primaryModelId, + token_generation_speed: 50, + response_times: [], + errors: 0, + consecutive_errors: 0, + avg_response_time: null, + avg_provider_latency: null, + avg_token_speed: null }, - 'gpt-5.4': { - id: 'gpt-5.4', - token_generation_speed: existingProvider?.models?.['gpt-5.4']?.token_generation_speed || 50, - response_times: existingProvider?.models?.['gpt-5.4']?.response_times || [], - errors: existingProvider?.models?.['gpt-5.4']?.errors || 0, - consecutive_errors: existingProvider?.models?.['gpt-5.4']?.consecutive_errors || 0, - avg_response_time: existingProvider?.models?.['gpt-5.4']?.avg_response_time || null, - avg_provider_latency: existingProvider?.models?.['gpt-5.4']?.avg_provider_latency || null, - avg_token_speed: existingProvider?.models?.['gpt-5.4']?.avg_token_speed || null + [secondaryModelId]: { + id: secondaryModelId, + token_generation_speed: 50, + response_times: [], + errors: 0, + consecutive_errors: 0, + avg_response_time: null, + avg_provider_latency: null, + avg_token_speed: null } }, - avg_response_time: existingProvider?.avg_response_time || null, - avg_provider_latency: existingProvider?.avg_provider_latency || null, - errors: existingProvider?.errors || 0, - provider_score: existingProvider?.provider_score || null, + avg_response_time: null, + avg_provider_latency: null, + errors: 0, + provider_score: mode === 'anthropic' ? 95 : 100, disabled: false }; @@ -82,24 +83,24 @@ export function setupMockProviderConfig() { tier: 'enterprise' }; - const created = Math.floor(Date.now() / 1000); + const created = DEFAULT_TEST_MODELS_CREATED_AT; const testModels = { object: 'list', data: [ { - id: 'gpt-3.5-turbo', + id: primaryModelId, object: 'model', created, - owned_by: 'openai-mock', + owned_by: providerId, providers: 1, throughput: 50, capabilities: ['text', 'tool_calling'], }, { - id: 'gpt-5.4', + id: secondaryModelId, object: 'model', created, - owned_by: 'openai-mock', + owned_by: providerId, providers: 1, throughput: 50, capabilities: ['text', 'tool_calling'], @@ -135,15 +136,6 @@ export function setupMockProviderConfig() { console.log('[TEST-SETUP] Created isolated test models.json'); // Add test API key to keys.json - let existingKeys: Record = {}; - if (fs.existsSync(keysFilePath)) { - try { - existingKeys = JSON.parse(fs.readFileSync(keysFilePath, 'utf8')); - } catch (error) { - console.log('[TEST-SETUP] Could not parse existing keys.json, starting fresh'); - } - } - const deriveKeyHash = (value: string) => crypto.pbkdf2Sync( value, hashSecret, @@ -152,9 +144,7 @@ export function setupMockProviderConfig() { hashDigest ).toString('hex'); - const updatedKeys: Record = { - ...existingKeys, - }; + const updatedKeys: Record = {}; if (!updatedKeys[testApiKey]) { updatedKeys[testApiKey] = testUserKey; @@ -165,11 +155,11 @@ export function setupMockProviderConfig() { updatedKeys[hashedApiKey] = testUserKey; } - if (!updatedKeys['test-key-for-mock-provider']) { - updatedKeys['test-key-for-mock-provider'] = testUserKey; + if (!updatedKeys[LEGACY_TEST_API_KEY]) { + updatedKeys[LEGACY_TEST_API_KEY] = testUserKey; } - const hashedFallbackKey = deriveKeyHash('test-key-for-mock-provider'); + const hashedFallbackKey = deriveKeyHash(LEGACY_TEST_API_KEY); if (!updatedKeys[hashedFallbackKey]) { updatedKeys[hashedFallbackKey] = testUserKey; } @@ -178,81 +168,36 @@ export function setupMockProviderConfig() { console.log('[TEST-SETUP] Added test API key to keys.json'); } -export function restoreProviderConfig() { - const providersFilePath = resolveTestDataPath('API_PROVIDERS_FILE', path.join(apiRoot, 'providers.json')); - const keysFilePath = resolveTestDataPath('API_KEYS_FILE', path.join(apiRoot, 'keys.json')); - const modelsFilePath = resolveTestDataPath('API_MODELS_FILE', path.join(apiRoot, 'models.json')); - const backupProvidersPath = `${providersFilePath}.backup`; - const backupKeysPath = `${keysFilePath}.backup`; - const backupModelsPath = `${modelsFilePath}.backup`; - - // Preserve response times and stats from the test run - let updatedProviderData = null; - if (fs.existsSync(providersFilePath)) { - try { - const currentProviders = JSON.parse(fs.readFileSync(providersFilePath, 'utf8')); - updatedProviderData = currentProviders.find((p: any) => p.id === 'openai-mock'); - } catch (error) { - console.log('[TEST-CLEANUP] Could not parse current providers.json'); - } +function restoreOrRemoveGeneratedFile(filePath: string, backupPath: string, label: string): void { + if (fs.existsSync(backupPath)) { + fs.copyFileSync(backupPath, filePath); + fs.unlinkSync(backupPath); + console.log(`[TEST-CLEANUP] Restored original ${label}`); + return; } - if (fs.existsSync(backupProvidersPath)) { - // Read the backup - const backupProviders = JSON.parse(fs.readFileSync(backupProvidersPath, 'utf8')); - - // Find the existing provider in backup and merge the new response times - if (updatedProviderData) { - const existingProviderIndex = backupProviders.findIndex((p: any) => p.id === 'openai-mock'); - if (existingProviderIndex >= 0) { - // Merge response times and updated stats - const existingProvider = backupProviders[existingProviderIndex]; - if (existingProvider.models && existingProvider.models['gpt-3.5-turbo'] && - updatedProviderData.models && updatedProviderData.models['gpt-3.5-turbo']) { - - // Keep all the new response times, errors, and computed stats - existingProvider.models['gpt-3.5-turbo'].response_times = - updatedProviderData.models['gpt-3.5-turbo'].response_times || existingProvider.models['gpt-3.5-turbo'].response_times; - existingProvider.models['gpt-3.5-turbo'].errors = - updatedProviderData.models['gpt-3.5-turbo'].errors; - existingProvider.models['gpt-3.5-turbo'].consecutive_errors = - updatedProviderData.models['gpt-3.5-turbo'].consecutive_errors; - existingProvider.models['gpt-3.5-turbo'].avg_response_time = - updatedProviderData.models['gpt-3.5-turbo'].avg_response_time; - existingProvider.models['gpt-3.5-turbo'].avg_provider_latency = - updatedProviderData.models['gpt-3.5-turbo'].avg_provider_latency; - existingProvider.models['gpt-3.5-turbo'].avg_token_speed = - updatedProviderData.models['gpt-3.5-turbo'].avg_token_speed; - - // Update provider-level stats too - existingProvider.avg_response_time = updatedProviderData.avg_response_time; - existingProvider.avg_provider_latency = updatedProviderData.avg_provider_latency; - existingProvider.errors = updatedProviderData.errors; - existingProvider.provider_score = updatedProviderData.provider_score; - - console.log('[TEST-CLEANUP] Merged new response times and stats into original provider data'); - } - } - } - - // Write the merged data back - fs.writeFileSync(providersFilePath, JSON.stringify(backupProviders, null, 2)); - fs.unlinkSync(backupProvidersPath); - console.log('[TEST-CLEANUP] Restored providers.json with updated response times'); - } else { - // If no backup exists, keep the current file (which should have the new response times) - console.log('[TEST-CLEANUP] No backup found, keeping current providers.json with new response times'); + if (fs.existsSync(filePath)) { + fs.unlinkSync(filePath); + console.log(`[TEST-CLEANUP] Removed generated ${label}`); } +} - if (fs.existsSync(backupKeysPath)) { - fs.copyFileSync(backupKeysPath, keysFilePath); - fs.unlinkSync(backupKeysPath); - console.log('[TEST-CLEANUP] Restored original keys.json'); - } +function removeDirectoryIfEmpty(dirPath: string): void { + if (!fs.existsSync(dirPath)) return; + if (fs.readdirSync(dirPath).length > 0) return; + fs.rmdirSync(dirPath); +} - if (fs.existsSync(backupModelsPath)) { - fs.copyFileSync(backupModelsPath, modelsFilePath); - fs.unlinkSync(backupModelsPath); - console.log('[TEST-CLEANUP] Restored original models.json'); - } +export function restoreProviderConfig() { + const providersFilePath = resolveTestDataPath('API_PROVIDERS_FILE', path.join(testDataRoot, 'providers.json')); + const keysFilePath = resolveTestDataPath('API_KEYS_FILE', path.join(testDataRoot, 'keys.json')); + const modelsFilePath = resolveTestDataPath('API_MODELS_FILE', path.join(testDataRoot, 'models.json')); + const backupProvidersPath = `${providersFilePath}.backup`; + const backupKeysPath = `${keysFilePath}.backup`; + const backupModelsPath = `${modelsFilePath}.backup`; + + restoreOrRemoveGeneratedFile(providersFilePath, backupProvidersPath, 'providers.json'); + restoreOrRemoveGeneratedFile(keysFilePath, backupKeysPath, 'keys.json'); + restoreOrRemoveGeneratedFile(modelsFilePath, backupModelsPath, 'models.json'); + removeDirectoryIfEmpty(testDataRoot); } diff --git a/apps/api/dev/testWs.ts b/apps/api/dev/testWs.ts index d515c2d..430c152 100644 --- a/apps/api/dev/testWs.ts +++ b/apps/api/dev/testWs.ts @@ -7,7 +7,9 @@ dotenv.config({ path: path.resolve(process.cwd(), envFile) }); import WebSocket, { RawData } from 'ws'; -const API_KEY = process.env.TEST_API_KEY || 'test-key-for-mock-provider'; +import { DEFAULT_TEST_API_KEY } from './testSetup.js'; + +const API_KEY = process.env.TEST_API_KEY || DEFAULT_TEST_API_KEY; const url = process.env.WS_URL || 'ws://localhost:3000/ws'; const ws = new WebSocket(url); diff --git a/apps/api/modules/providerIdentity.ts b/apps/api/modules/providerIdentity.ts new file mode 100644 index 0000000..039a57b --- /dev/null +++ b/apps/api/modules/providerIdentity.ts @@ -0,0 +1,138 @@ +import { urlHasExpectedHostname } from './urlGuards.js'; + +export type ProviderFamily = + | 'openai' + | 'openrouter' + | 'deepseek' + | 'gemini' + | 'anthropic' + | 'xai' + | 'mock' + | 'unknown'; + +export type ProviderIdentityInput = + | string + | { + id?: string | null; + provider?: string | null; + type?: string | null; + provider_url?: string | null; + url?: string | null; + }; + +type ProviderFamilyDetector = { + family: Exclude; + tokens: string[]; + hostnames?: string[]; +}; + +const PROVIDER_FAMILY_DETECTORS: ProviderFamilyDetector[] = [ + { family: 'mock', tokens: ['mock'] }, + { + family: 'openrouter', + tokens: ['openrouter'], + hostnames: ['openrouter.ai'], + }, + { + family: 'deepseek', + tokens: ['deepseek'], + hostnames: ['api.deepseek.com'], + }, + { + family: 'xai', + tokens: ['xai', 'x-ai'], + hostnames: ['api.x.ai', 'x.ai'], + }, + { + family: 'anthropic', + tokens: ['anthropic', 'claude'], + hostnames: ['api.anthropic.com'], + }, + { + family: 'gemini', + tokens: ['gemini', 'google', 'imagen', 'nano-banana'], + hostnames: ['generativelanguage.googleapis.com'], + }, + { + family: 'openai', + tokens: ['openai'], + hostnames: ['api.openai.com', 'openai.com'], + }, +]; + +function normalizeValue(value: unknown): string { + return String(value ?? '').trim().toLowerCase(); +} + +function getProviderIdentityFields(input: ProviderIdentityInput): { + id: string; + provider: string; + type: string; + providerUrl: string; +} { + if (typeof input === 'string') { + return { + id: normalizeValue(input), + provider: '', + type: '', + providerUrl: '', + }; + } + + return { + id: normalizeValue(input?.id), + provider: normalizeValue(input?.provider), + type: normalizeValue(input?.type), + providerUrl: normalizeValue(input?.provider_url ?? input?.url), + }; +} + +function matchesFamilyToken(values: string[], token: string): boolean { + return values.some((value) => value === token || value.includes(token)); +} + +export function resolveProviderFamily(input: ProviderIdentityInput): ProviderFamily { + const { id, provider, type, providerUrl } = getProviderIdentityFields(input); + const values = [id, provider, type].filter(Boolean); + + for (const detector of PROVIDER_FAMILY_DETECTORS) { + if (detector.tokens.some((token) => matchesFamilyToken(values, token))) { + return detector.family; + } + if ( + providerUrl && + detector.hostnames?.some((hostname) => + urlHasExpectedHostname(providerUrl, hostname), + ) + ) { + return detector.family; + } + } + + return 'unknown'; +} + +export function isGoogleFamilyProvider(input: ProviderIdentityInput): boolean { + return resolveProviderFamily(input) === 'gemini'; +} + +export function isOpenAiNativeProvider(input: ProviderIdentityInput): boolean { + return resolveProviderFamily(input) === 'openai'; +} + +export function isOpenRouterProvider(input: ProviderIdentityInput): boolean { + return resolveProviderFamily(input) === 'openrouter'; +} + +export function isXaiProvider(input: ProviderIdentityInput): boolean { + return resolveProviderFamily(input) === 'xai'; +} + +export function normalizeProviderFamilyKey(input: ProviderIdentityInput): string { + const family = resolveProviderFamily(input); + if (family !== 'unknown') return family; + + const { id } = getProviderIdentityFields(input); + const dashIndex = id.indexOf('-'); + return dashIndex > 0 ? id.slice(0, dashIndex) : id; +} diff --git a/apps/api/modules/responsesHistory.ts b/apps/api/modules/responsesHistory.ts index ad20380..f170d87 100644 --- a/apps/api/modules/responsesHistory.ts +++ b/apps/api/modules/responsesHistory.ts @@ -13,6 +13,10 @@ export type StoredResponsesHistoryEntry = { previous_response_id?: string; replay_depth?: number; compacted?: boolean; + owner_scope?: string; + provider_family?: string; + provider_id?: string; + upstream_response_id?: string; }; export type ResponsesHistoryMergeResult = { @@ -107,6 +111,22 @@ function normalizeStoredResponsesHistoryEntry(raw: StoredResponsesHistoryEntry): : undefined, replay_depth: replayDepth, compacted: raw?.compacted === true, + owner_scope: + typeof raw?.owner_scope === 'string' && raw.owner_scope.trim() + ? raw.owner_scope.trim() + : undefined, + provider_family: + typeof raw?.provider_family === 'string' && raw.provider_family.trim() + ? raw.provider_family.trim() + : undefined, + provider_id: + typeof raw?.provider_id === 'string' && raw.provider_id.trim() + ? raw.provider_id.trim() + : undefined, + upstream_response_id: + typeof raw?.upstream_response_id === 'string' && raw.upstream_response_id.trim() + ? raw.upstream_response_id.trim() + : undefined, }; } diff --git a/apps/api/package.json b/apps/api/package.json index 4aa8ffa..1e0fd63 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -29,8 +29,8 @@ "test": "bash ./dev/cleanupTestPorts.sh && concurrently --kill-others --success first --names \"MOCK,API,TEST\" -c \"bgYellow.bold,bgBlue.bold,bgMagenta.bold\" \"bash ../../bun.sh run test:mock\" \"bash ../../bun.sh run test:dev\" \"wait-on tcp:localhost:3101 tcp:localhost:3100 && bash ../../bun.sh run test:run && bash ../../bun.sh run test:sdk\"", "test:mock": "bash -lc 'NODE_ENV=${NODE_ENV:-test} MOCK_PROVIDER_PORT=${MOCK_PROVIDER_PORT:-3101} bash ../../bun.sh run ./dev/mockProvider.ts'", "test:dev": "bash ./dev/runTestServer.sh", - "test:run": "bash -lc 'NODE_ENV=${NODE_ENV:-test} TEST_SETUP_MODE=${TEST_SETUP_MODE:-external} TEST_API_BASE_URL=${TEST_API_BASE_URL:-http://localhost:3100} API_PROVIDERS_FILE=${API_PROVIDERS_FILE:-.test-data/providers.json} API_KEYS_FILE=${API_KEYS_FILE:-.test-data/keys.json} API_MODELS_FILE=${API_MODELS_FILE:-.test-data/models.json} API_TIERS_FILE=${API_TIERS_FILE:-.test-data/tiers.json} bash ../../bun.sh run ./dev/testApi.ts'", - "test:sdk": "bash -lc 'NODE_ENV=${NODE_ENV:-test} TEST_SETUP_MODE=${TEST_SETUP_MODE:-external} TEST_API_BASE_URL=${TEST_API_BASE_URL:-http://localhost:3100} API_PROVIDERS_FILE=${API_PROVIDERS_FILE:-.test-data/providers.json} API_KEYS_FILE=${API_KEYS_FILE:-.test-data/keys.json} API_MODELS_FILE=${API_MODELS_FILE:-.test-data/models.json} API_TIERS_FILE=${API_TIERS_FILE:-.test-data/tiers.json} bash ../../bun.sh run ./dev/testOpenAISdk.ts'", + "test:run": "bash -lc 'NODE_ENV=${NODE_ENV:-test} TEST_SETUP_MODE=${TEST_SETUP_MODE:-external} TEST_API_BASE_URL=${TEST_API_BASE_URL:-http://localhost:3100} API_PROVIDERS_FILE=${API_PROVIDERS_FILE:-.tmp/test-data/providers.json} API_KEYS_FILE=${API_KEYS_FILE:-.tmp/test-data/keys.json} API_MODELS_FILE=${API_MODELS_FILE:-.tmp/test-data/models.json} API_TIERS_FILE=${API_TIERS_FILE:-.test-data/tiers.json} bash ../../bun.sh run ./dev/testApi.ts'", + "test:sdk": "bash -lc 'NODE_ENV=${NODE_ENV:-test} TEST_SETUP_MODE=${TEST_SETUP_MODE:-external} TEST_API_BASE_URL=${TEST_API_BASE_URL:-http://localhost:3100} API_PROVIDERS_FILE=${API_PROVIDERS_FILE:-.tmp/test-data/providers.json} API_KEYS_FILE=${API_KEYS_FILE:-.tmp/test-data/keys.json} API_MODELS_FILE=${API_MODELS_FILE:-.tmp/test-data/models.json} API_TIERS_FILE=${API_TIERS_FILE:-.test-data/tiers.json} bash ../../bun.sh run ./dev/testOpenAISdk.ts'", "test:ws": "cross-env NODE_ENV=test bash ../../bun.sh run ./dev/testWs.ts" }, "dependencies": { diff --git a/apps/api/routes/nativeProviders.ts b/apps/api/routes/nativeProviders.ts new file mode 100644 index 0000000..12cb3fd --- /dev/null +++ b/apps/api/routes/nativeProviders.ts @@ -0,0 +1,1784 @@ +import HyperExpress, { Request, Response } from '../lib/uws-compat.js'; +import crypto from 'node:crypto'; +import dotenv from 'dotenv'; +import { + runAuthMiddleware, + runRateLimitMiddleware, + normalizeApiKey +} from '../modules/middlewareFactory.js'; +import type { RequestTimestampStore } from '../modules/rateLimit.js'; +import { logError } from '../modules/errorLogger.js'; +import { + updateUserTokenUsage, + type TierData +} from '../modules/userData.js'; +import { + buildModelAccessError, + isModelAllowedForTier +} from '../modules/planAccess.js'; +import { + dataManager, + type LoadedProviderData, + type LoadedProviders +} from '../modules/dataManager.js'; +import { fetchWithTimeout } from '../modules/http.js'; +import { + createResponsesItemId +} from '../modules/openaiResponsesFormat.js'; +import { + createSseDataParser, + extractUsageTokens, + getHeaderValue +} from '../modules/openaiRequestSupport.js'; +import { + getBackpressureRetryAfterSeconds, + withBufferedRequestBody +} from '../modules/requestIntake.js'; +import { resolveProviderFamily } from '../modules/providerIdentity.js'; +import { redactToken, buildSafeUpstreamErrorMessage } from '../modules/redaction.js'; +import { getRequestQueueForLane } from '../modules/requestQueue.js'; +import { applyResponseHeaders, buildModelsPayload } from './models.js'; +import { + type StoredResponsesHistoryEntry, + buildResponsesHistoryStoragePlan, + buildStoredResponsesHistoryOutput, + cloneResponsesHistoryValue, + loadResponsesHistoryEntry, + mergeResponsesHistoryInput, + saveResponsesHistoryEntry +} from '../modules/responsesHistory.js'; + +dotenv.config(); + +const router = new HyperExpress.Router(); +const requestTimestamps: RequestTimestampStore = {}; +const HOP_BY_HOP_RESPONSE_HEADERS = new Set([ + 'connection', + 'content-length', + 'keep-alive', + 'proxy-authenticate', + 'proxy-authorization', + 'te', + 'trailer', + 'transfer-encoding', + 'upgrade' +]); +const SAFE_NATIVE_RESPONSE_HEADERS = new Set([ + 'cache-control', + 'content-disposition', + 'content-type', + 'retry-after' +]); +const nativeProviderCooldowns = new Map(); +const DEFAULT_NATIVE_PROVIDER_COOLDOWN_MS = (() => { + const raw = Number(process.env.NATIVE_PROVIDER_COOLDOWN_MS ?? 60_000); + return Number.isFinite(raw) && raw > 0 ? Math.max(1_000, Math.ceil(raw)) : 60_000; +})(); + +type NativeFamily = + | 'openai' + | 'anthropic' + | 'gemini' + | 'openrouter' + | 'deepseek' + | 'xai'; + +type NativeFamilyAlias = NativeFamily | 'claude' | 'google' | 'x-ai'; + +type NativeResponsesHistoryContext = { + proxyResponseId: string; + ownerScope?: string; + inputDelta: any[]; + mergedInput: any[]; + previousEntry: StoredResponsesHistoryEntry | null; +}; + +function canonicalizeNativeFamily(raw: string): NativeFamily | null { + const normalized = String(raw || '').trim().toLowerCase(); + if (!normalized) return null; + if (normalized === 'openai') return 'openai'; + if (normalized === 'anthropic' || normalized === 'claude') return 'anthropic'; + if (normalized === 'gemini' || normalized === 'google') return 'gemini'; + if (normalized === 'openrouter') return 'openrouter'; + if (normalized === 'deepseek') return 'deepseek'; + if (normalized === 'xai' || normalized === 'x-ai') return 'xai'; + return null; +} + +function isAutoNativeFamilySelector(raw: string): boolean { + const normalized = String(raw || '').trim().toLowerCase(); + return normalized === 'auto' || normalized === 'mutual'; +} + +function providerFamilyToNativeFamily(rawFamily: string): NativeFamily | null { + const normalized = String(rawFamily || '').trim().toLowerCase(); + if (!normalized) return null; + if (normalized === 'openai') return 'openai'; + if (normalized === 'anthropic') return 'anthropic'; + if (normalized === 'gemini') return 'gemini'; + if (normalized === 'openrouter') return 'openrouter'; + if (normalized === 'deepseek') return 'deepseek'; + if (normalized === 'xai') return 'xai'; + if (normalized === 'mock') return 'openai'; + return null; +} + +function inferNativeFamilyFromModelHeuristics(modelId: string): NativeFamily | null { + const normalized = String(modelId || '').trim().toLowerCase(); + if (!normalized) return null; + const noNamespace = normalized.includes('/') + ? normalized.split('/').pop() || normalized + : normalized; + + if (normalized.startsWith('openai/') || /^gpt([\-._]|$)/.test(noNamespace)) { + return 'openai'; + } + if ( + normalized.startsWith('anthropic/') || + normalized.startsWith('claude/') || + /^claude([\-._]|$)/.test(noNamespace) + ) { + return 'anthropic'; + } + if ( + normalized.startsWith('gemini/') || + normalized.startsWith('google/') || + normalized.startsWith('imagen/') || + /^gemini([\-._]|$)/.test(noNamespace) || + /^imagen([\-._]|$)/.test(noNamespace) || + /^nano-banana([\-._]|$)/.test(noNamespace) + ) { + return 'gemini'; + } + if (normalized.startsWith('openrouter/')) return 'openrouter'; + if ( + normalized.startsWith('deepseek/') || + /^deepseek([\-._]|$)/.test(noNamespace) + ) { + return 'deepseek'; + } + if ( + normalized.startsWith('xai/') || + normalized.startsWith('x-ai/') || + /^grok([\-._]|$)/.test(noNamespace) + ) { + return 'xai'; + } + + return null; +} + +function inferNativeFamilyFromSubpathHeuristics(subpath: string): NativeFamily | null { + const normalized = String(subpath || '').trim().toLowerCase(); + if (!normalized) return null; + + if ( + normalized === '/responses' || + normalized === '/v1/responses' || + normalized === '/chat/completions' || + normalized === '/v1/chat/completions' + ) { + return 'openai'; + } + + if (normalized === '/messages' || normalized === '/v1/messages') { + return 'anthropic'; + } + + if (normalized.startsWith('/v1beta/models/') || normalized.startsWith('/models/')) { + return 'gemini'; + } + + return null; +} + +function extractRoutingModelId(parsedBody: any, subpath: string): string | null { + if (typeof parsedBody?.model === 'string' && parsedBody.model.trim()) { + return parsedBody.model.trim(); + } + + const directMatch = subpath.match(/\/models\/([^/:?]+)(?::[A-Za-z][A-Za-z0-9]*)?/i); + if (directMatch?.[1]) { + try { + return decodeURIComponent(directMatch[1]); + } catch { + return directMatch[1]; + } + } + + return null; +} + +function inferNativeFamilyFromModelAndProviders( + modelId: string, + providers: LoadedProviders +): NativeFamily | null { + const normalizedModelId = String(modelId || '').trim(); + if (!normalizedModelId) return null; + + const candidates = providers + .filter(provider => !provider.disabled) + .filter(provider => typeof provider.apiKey === 'string' && provider.apiKey.trim().length > 0) + .filter(provider => typeof provider.provider_url === 'string' && provider.provider_url.trim().length > 0) + .filter(provider => providerSupportsModel(provider, normalizedModelId)) + .map(provider => { + const family = providerFamilyToNativeFamily( + resolveProviderFamily({ + id: provider.id, + provider: (provider as any)?.provider, + type: (provider as any)?.type, + provider_url: provider.provider_url + }) + ); + return { provider, family }; + }) + .filter((entry): entry is { provider: LoadedProviderData; family: NativeFamily } => + Boolean(entry.family) + ) + .sort((left, right) => { + const leftScore = + typeof left.provider.provider_score === 'number' && Number.isFinite(left.provider.provider_score) + ? left.provider.provider_score + : Number.NEGATIVE_INFINITY; + const rightScore = + typeof right.provider.provider_score === 'number' && Number.isFinite(right.provider.provider_score) + ? right.provider.provider_score + : Number.NEGATIVE_INFINITY; + if (leftScore !== rightScore) return rightScore - leftScore; + + const leftLatency = + typeof left.provider.avg_response_time === 'number' && Number.isFinite(left.provider.avg_response_time) + ? left.provider.avg_response_time + : Number.POSITIVE_INFINITY; + const rightLatency = + typeof right.provider.avg_response_time === 'number' && Number.isFinite(right.provider.avg_response_time) + ? right.provider.avg_response_time + : Number.POSITIVE_INFINITY; + if (leftLatency !== rightLatency) return leftLatency - rightLatency; + + return left.provider.id.localeCompare(right.provider.id); + }); + + if (candidates.length > 0) { + return candidates[0].family; + } + + return inferNativeFamilyFromModelHeuristics(normalizedModelId); +} + +function isOpenAiResponsesSubpath(subpath: string): boolean { + return /\/responses(?:\/|$)/i.test(String(subpath || '')); +} + +const NATIVE_RESPONSES_HISTORY_FAMILIES = new Set([ + 'openai', + 'openrouter', + 'deepseek', + 'xai' +]); + +function supportsNativeResponsesHistory( + family: NativeFamily | null | undefined +): family is NativeFamily { + return Boolean(family && NATIVE_RESPONSES_HISTORY_FAMILIES.has(family)); +} + +function buildNativeResponsesOwnerScope( + request: Request +): string | undefined { + const normalizedUserId = + typeof (request as any)?.userId === 'string' && (request as any).userId.trim() + ? (request as any).userId.trim() + : ''; + if (normalizedUserId) return `user:${normalizedUserId}`; + const normalizedApiKey = + typeof request.apiKey === 'string' && request.apiKey.trim() + ? request.apiKey.trim() + : ''; + if (!normalizedApiKey) return undefined; + try { + return `key:${crypto + .createHash('sha256') + .update(normalizedApiKey) + .digest('hex') + .slice(0, 24)}`; + } catch { + return `key:${normalizedApiKey.slice(0, 8)}`; + } +} + +function isNativeResponsesHistoryEntryUsable( + entry: StoredResponsesHistoryEntry | null, + ownerScope?: string +): entry is StoredResponsesHistoryEntry { + if (!entry) return false; + const entryOwnerScope = + typeof (entry as any)?.owner_scope === 'string' && + (entry as any).owner_scope.trim() + ? (entry as any).owner_scope.trim() + : ''; + if (ownerScope && entryOwnerScope && entryOwnerScope !== ownerScope) { + return false; + } + const providerFamily = canonicalizeNativeFamily( + typeof (entry as any)?.provider_family === 'string' + ? (entry as any).provider_family + : '' + ); + if (providerFamily && !supportsNativeResponsesHistory(providerFamily)) { + return false; + } + return true; +} + +function rewriteNativeResponsesResponseObject( + responsePayload: any, + proxyResponseId: string +): any { + if (!responsePayload || typeof responsePayload !== 'object' || Array.isArray(responsePayload)) { + return responsePayload; + } + const rewritten = cloneResponsesHistoryValue(responsePayload); + rewritten.id = proxyResponseId; + if (typeof rewritten.response_id === 'string' && rewritten.response_id.trim()) { + rewritten.response_id = proxyResponseId; + } + return rewritten; +} + +function rewriteNativeResponsesEventPayload( + eventPayload: any, + proxyResponseId: string +): any { + if (!eventPayload || typeof eventPayload !== 'object' || Array.isArray(eventPayload)) { + return eventPayload; + } + const rewritten = cloneResponsesHistoryValue(eventPayload); + if (typeof rewritten.response_id === 'string' && rewritten.response_id.trim()) { + rewritten.response_id = proxyResponseId; + } + if (rewritten.response && typeof rewritten.response === 'object') { + rewritten.response = rewriteNativeResponsesResponseObject( + rewritten.response, + proxyResponseId + ); + } + return rewritten; +} + +function serializeSseEvent(eventName: string | undefined, data: string): string { + let serialized = ''; + if (typeof eventName === 'string' && eventName.trim()) { + serialized += `event: ${eventName}\n`; + } + for (const line of String(data ?? '').split('\n')) { + serialized += `data: ${line}\n`; + } + return `${serialized}\n`; +} + +function getNativeResponsesToolCallKey(rawCall: any): string | null { + if (!rawCall || typeof rawCall !== 'object') return null; + const callId = + typeof rawCall?.call_id === 'string' && rawCall.call_id.trim() + ? rawCall.call_id.trim() + : typeof rawCall?.tool_call_id === 'string' && rawCall.tool_call_id.trim() + ? rawCall.tool_call_id.trim() + : undefined; + if (callId) return `call:${callId}`; + const id = + typeof rawCall?.id === 'string' && rawCall.id.trim() + ? rawCall.id.trim() + : undefined; + if (id) return `id:${id}`; + const name = + typeof rawCall?.name === 'string' && rawCall.name.trim() + ? rawCall.name.trim() + : rawCall?.function && + typeof rawCall.function === 'object' && + typeof rawCall.function.name === 'string' && + rawCall.function.name.trim() + ? rawCall.function.name.trim() + : undefined; + return name ? `name:${name}` : null; +} + +function normalizeNativeResponsesToolCall(rawCall: any): Record | null { + if (!rawCall || typeof rawCall !== 'object') return null; + const functionPayload = + rawCall?.function && typeof rawCall.function === 'object' + ? rawCall.function + : rawCall; + const name = + typeof functionPayload?.name === 'string' && functionPayload.name.trim() + ? functionPayload.name.trim() + : typeof rawCall?.name === 'string' && rawCall.name.trim() + ? rawCall.name.trim() + : ''; + if (!name) return null; + const normalized: Record = { + name, + arguments: + typeof functionPayload?.arguments !== 'undefined' + ? cloneResponsesHistoryValue(functionPayload.arguments) + : typeof rawCall?.arguments !== 'undefined' + ? cloneResponsesHistoryValue(rawCall.arguments) + : '{}', + status: + typeof rawCall?.status === 'string' && rawCall.status.trim() + ? rawCall.status.trim() + : 'completed' + }; + if (typeof rawCall?.id === 'string' && rawCall.id.trim()) { + normalized.id = rawCall.id.trim(); + } + if (typeof rawCall?.call_id === 'string' && rawCall.call_id.trim()) { + normalized.call_id = rawCall.call_id.trim(); + } else if ( + typeof rawCall?.tool_call_id === 'string' && + rawCall.tool_call_id.trim() + ) { + normalized.call_id = rawCall.tool_call_id.trim(); + } + return normalized; +} + +function upsertNativeResponsesToolCall( + toolCalls: Record[], + rawCall: any +): void { + const normalized = normalizeNativeResponsesToolCall(rawCall); + if (!normalized) return; + const key = getNativeResponsesToolCallKey(normalized); + const index = key + ? toolCalls.findIndex(entry => getNativeResponsesToolCallKey(entry) === key) + : -1; + if (index >= 0) { + toolCalls[index] = { + ...toolCalls[index], + ...normalized + }; + return; + } + toolCalls.push(normalized); +} + +function collectNativeResponsesToolCallsFromItem( + toolCalls: Record[], + item: any +): void { + if (!item || typeof item !== 'object') return; + if (item.type === 'function_call') { + upsertNativeResponsesToolCall(toolCalls, item); + return; + } + if (!Array.isArray(item.content)) return; + for (const part of item.content) { + if ( + part && + typeof part === 'object' && + part.type === 'tool_calls' && + Array.isArray(part.tool_calls) + ) { + for (const toolCall of part.tool_calls) { + upsertNativeResponsesToolCall(toolCalls, toolCall); + } + } + } +} + +function normalizeNativeResponsesInput(rawInput: any): any[] { + if (Array.isArray(rawInput)) return rawInput; + if (typeof rawInput === 'string') { + return [{ type: 'input_text', text: rawInput }]; + } + if (rawInput && typeof rawInput === 'object') { + return [rawInput]; + } + throw new Error('input must be a string, array, or object.'); +} + +async function persistNativeResponsesHistoryEntry(params: { + context: NativeResponsesHistoryContext; + responsePayload: any; + modelId: string | null; + request: Request; + routedFamily: NativeFamily; + providerId: string; + upstreamResponseId?: string | null; +}): Promise { + const responseId = + typeof params.context?.proxyResponseId === 'string' && + params.context.proxyResponseId.trim() + ? params.context.proxyResponseId.trim() + : typeof params.responsePayload?.id === 'string' && + params.responsePayload.id.trim() + ? params.responsePayload.id.trim() + : ''; + if (!responseId) return; + + const upstreamResponseId = + typeof params.upstreamResponseId === 'string' && params.upstreamResponseId.trim() + ? params.upstreamResponseId.trim() + : typeof params.responsePayload?.id === 'string' && + params.responsePayload.id.trim() && + params.responsePayload.id.trim() !== responseId + ? params.responsePayload.id.trim() + : undefined; + const model = + typeof params.responsePayload?.model === 'string' && + params.responsePayload.model.trim() + ? params.responsePayload.model.trim() + : typeof params.modelId === 'string' && params.modelId.trim() + ? params.modelId.trim() + : 'unknown'; + const created = + typeof params.responsePayload?.created === 'number' && + Number.isFinite(params.responsePayload.created) + ? Math.floor(params.responsePayload.created) + : Math.floor(Date.now() / 1000); + const outputText = + typeof params.responsePayload?.output_text === 'string' + ? params.responsePayload.output_text + : ''; + const toolCalls = Array.isArray(params.responsePayload?.tool_calls) + ? cloneResponsesHistoryValue(params.responsePayload.tool_calls) + : undefined; + const output = Array.isArray(params.responsePayload?.output) + ? cloneResponsesHistoryValue(params.responsePayload.output) + : buildStoredResponsesHistoryOutput(outputText, toolCalls); + + const storagePlan = buildResponsesHistoryStoragePlan({ + previousEntry: params.context.previousEntry, + inputDelta: params.context.inputDelta, + fullInput: params.context.mergedInput + }); + + try { + await saveResponsesHistoryEntry({ + id: responseId, + model, + output, + output_text: outputText, + created, + owner_scope: params.context.ownerScope, + provider_family: params.routedFamily, + provider_id: params.providerId, + upstream_response_id: upstreamResponseId, + ...storagePlan + }); + } catch (historyError: any) { + await logError( + { + message: 'Failed to persist native responses history entry.', + errorMessage: historyError?.message || String(historyError), + errorStack: historyError?.stack, + responseId, + upstreamResponseId, + providerId: params.providerId, + model + }, + params.request + ); + } +} + +function estimateTokens(content: unknown): number { + if (typeof content === 'string') return Math.ceil(content.length / 4); + try { + return Math.ceil(JSON.stringify(content ?? '').length / 4); + } catch { + return Math.ceil(String(content ?? '').length / 4); + } +} + +function normalizeJsonBody(rawBody: Buffer, contentType: string): any | null { + if (!rawBody || rawBody.length === 0) return null; + if (!contentType.toLowerCase().includes('application/json')) return null; + try { + return JSON.parse(rawBody.toString('utf8')); + } catch { + return null; + } +} + +function sanitizeNativeToolSchema(schema: any): any { + const visit = (value: any): any => { + if (Array.isArray(value)) return value.map(entry => visit(entry)); + if (!value || typeof value !== 'object') return value; + + const normalized: Record = {}; + for (const [key, entry] of Object.entries(value)) { + normalized[key] = visit(entry); + } + + const rawType = normalized.type; + const typeList = Array.isArray(rawType) + ? rawType + .filter(entry => typeof entry === 'string') + .map(entry => String(entry).toLowerCase()) + : typeof rawType === 'string' + ? [rawType.toLowerCase()] + : []; + if ( + typeList.includes('array') && + typeList.length > 0 && + typeof normalized.items === 'undefined' && + typeof normalized.prefixItems === 'undefined' + ) { + normalized.items = { type: 'string' }; + } + + return normalized; + }; + + return visit(schema); +} + +function sanitizeOpenAIResponseToolSchemas(payload: any): any { + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) return payload; + if (!Array.isArray(payload.tools) || payload.tools.length === 0) return payload; + + const sanitizedTools = payload.tools.map((tool: any) => { + if (!tool || typeof tool !== 'object' || Array.isArray(tool)) return tool; + + const normalizedTool: Record = { ...tool }; + if ( + typeof (tool as any).parameters === 'object' && + (tool as any).parameters && + !Array.isArray((tool as any).parameters) + ) { + normalizedTool.parameters = sanitizeNativeToolSchema((tool as any).parameters); + } + + const fn = (tool as any).function; + if (fn && typeof fn === 'object' && !Array.isArray(fn)) { + const normalizedFn: Record = { ...fn }; + if ( + typeof fn.parameters === 'object' && + fn.parameters && + !Array.isArray(fn.parameters) + ) { + normalizedFn.parameters = sanitizeNativeToolSchema(fn.parameters); + } + normalizedTool.function = normalizedFn; + } + + return normalizedTool; + }); + + return { + ...payload, + tools: sanitizedTools + }; +} + +function extractNativeModelId( + family: NativeFamily, + subpath: string, + parsedBody: any +): string | null { + if (typeof parsedBody?.model === 'string' && parsedBody.model.trim()) { + return parsedBody.model.trim(); + } + + if (family === 'gemini') { + const match = subpath.match(/\/models\/([^/:?]+)(?::[A-Za-z][A-Za-z0-9]*)?/i); + if (match?.[1]) { + try { + return decodeURIComponent(match[1]); + } catch { + return match[1]; + } + } + } + + return null; +} + +function providerSupportsModel( + provider: LoadedProviderData, + modelId: string | null +): boolean { + if (!modelId) return true; + const models = provider.models || {}; + if (modelId in models) return true; + const tail = modelId.includes('/') ? modelId.split('/').pop() || modelId : modelId; + if (tail in models) return true; + return false; +} + +function isProviderFamilyMatch( + provider: LoadedProviderData, + family: NativeFamily +): boolean { + const resolved = resolveProviderFamily({ + id: provider.id, + provider: (provider as any)?.provider, + type: (provider as any)?.type, + provider_url: provider.provider_url + }); + const mapped = providerFamilyToNativeFamily(resolved); + return mapped === family; +} + +function selectBestNativeProvider( + providers: LoadedProviders, + family: NativeFamily, + modelId: string | null +): LoadedProviderData | null { + const candidates = providers + .filter(provider => !isNativeProviderCoolingDown(provider.id, modelId)) + .filter(provider => !provider.disabled) + .filter(provider => typeof provider.apiKey === 'string' && provider.apiKey.trim().length > 0) + .filter(provider => typeof provider.provider_url === 'string' && provider.provider_url.trim().length > 0) + .filter(provider => isProviderFamilyMatch(provider, family)) + .filter(provider => providerSupportsModel(provider, modelId)) + .sort((left, right) => { + const leftScore = + typeof left.provider_score === 'number' && Number.isFinite(left.provider_score) + ? left.provider_score + : Number.NEGATIVE_INFINITY; + const rightScore = + typeof right.provider_score === 'number' && Number.isFinite(right.provider_score) + ? right.provider_score + : Number.NEGATIVE_INFINITY; + if (leftScore !== rightScore) return rightScore - leftScore; + + const leftLatency = + typeof left.avg_response_time === 'number' && Number.isFinite(left.avg_response_time) + ? left.avg_response_time + : Number.POSITIVE_INFINITY; + const rightLatency = + typeof right.avg_response_time === 'number' && Number.isFinite(right.avg_response_time) + ? right.avg_response_time + : Number.POSITIVE_INFINITY; + if (leftLatency !== rightLatency) return leftLatency - rightLatency; + + const leftErrors = + typeof left.errors === 'number' && Number.isFinite(left.errors) + ? left.errors + : Number.POSITIVE_INFINITY; + const rightErrors = + typeof right.errors === 'number' && Number.isFinite(right.errors) + ? right.errors + : Number.POSITIVE_INFINITY; + return leftErrors - rightErrors; + }); + + return candidates[0] || null; +} + +function listNativeProviderCandidates( + providers: LoadedProviders, + family: NativeFamily, + modelId: string | null +): LoadedProviderData[] { + return providers + .filter(provider => !provider.disabled) + .filter(provider => typeof provider.apiKey === 'string' && provider.apiKey.trim().length > 0) + .filter(provider => typeof provider.provider_url === 'string' && provider.provider_url.trim().length > 0) + .filter(provider => isProviderFamilyMatch(provider, family)) + .filter(provider => providerSupportsModel(provider, modelId)) + .sort((left, right) => { + const leftScore = + typeof left.provider_score === 'number' && Number.isFinite(left.provider_score) + ? left.provider_score + : Number.NEGATIVE_INFINITY; + const rightScore = + typeof right.provider_score === 'number' && Number.isFinite(right.provider_score) + ? right.provider_score + : Number.NEGATIVE_INFINITY; + if (leftScore !== rightScore) return rightScore - leftScore; + + const leftLatency = + typeof left.avg_response_time === 'number' && Number.isFinite(left.avg_response_time) + ? left.avg_response_time + : Number.POSITIVE_INFINITY; + const rightLatency = + typeof right.avg_response_time === 'number' && Number.isFinite(right.avg_response_time) + ? right.avg_response_time + : Number.POSITIVE_INFINITY; + if (leftLatency !== rightLatency) return leftLatency - rightLatency; + + const leftErrors = + typeof left.errors === 'number' && Number.isFinite(left.errors) + ? left.errors + : Number.POSITIVE_INFINITY; + const rightErrors = + typeof right.errors === 'number' && Number.isFinite(right.errors) + ? right.errors + : Number.POSITIVE_INFINITY; + return leftErrors - rightErrors; + }); +} + +function getNativeProviderCooldownKey(providerId: string, modelId: string | null): string { + return `${providerId}::${modelId || '*'}`; +} + +function getNativeProviderCooldownRemainingMs( + providerId: string, + modelId: string | null +): number { + const now = Date.now(); + for (const key of [ + getNativeProviderCooldownKey(providerId, modelId), + getNativeProviderCooldownKey(providerId, null) + ]) { + const expiresAt = nativeProviderCooldowns.get(key); + if (!expiresAt) continue; + const remaining = expiresAt - now; + if (remaining > 0) return remaining; + nativeProviderCooldowns.delete(key); + } + return 0; +} + +function isNativeProviderCoolingDown( + providerId: string, + modelId: string | null +): boolean { + return getNativeProviderCooldownRemainingMs(providerId, modelId) > 0; +} + +function markNativeProviderCooldown( + providerId: string, + modelId: string | null, + cooldownMs?: number | null +): void { + const duration = + Number.isFinite(cooldownMs as number) && (cooldownMs as number) > 0 + ? Math.max(1_000, Math.ceil(cooldownMs as number)) + : DEFAULT_NATIVE_PROVIDER_COOLDOWN_MS; + nativeProviderCooldowns.set( + getNativeProviderCooldownKey(providerId, modelId), + Date.now() + duration + ); +} + +function parseRetryAfterMs(value: string | null): number | null { + if (!value) return null; + const trimmed = value.trim(); + if (!trimmed) return null; + const seconds = Number(trimmed); + if (Number.isFinite(seconds) && seconds > 0) { + return Math.max(1_000, Math.ceil(seconds * 1000)); + } + const retryAt = Date.parse(trimmed); + if (Number.isFinite(retryAt)) { + const delta = retryAt - Date.now(); + return delta > 0 ? Math.max(1_000, delta) : null; + } + return null; +} + +function isQuotaOrRateLimitMessage(message: string): boolean { + const normalized = message.toLowerCase(); + return ( + normalized.includes('quota exceeded') || + normalized.includes('resource_exhausted') || + normalized.includes('rate limit') || + normalized.includes('too many requests') || + normalized.includes('retry in') || + normalized.includes('insufficient_quota') || + normalized.includes('billing_hard_limit_reached') || + normalized.includes('billing limit') + ); +} + +function shouldTryNextNativeProvider( + family: NativeFamily, + statusCode: number, + errorText: string +): boolean { + if ([401, 402, 403, 408, 409, 425, 429].includes(statusCode)) return true; + if (statusCode >= 500 && statusCode <= 599) return true; + if (isQuotaOrRateLimitMessage(errorText)) return true; + if (family === 'gemini') { + const normalized = errorText.toLowerCase(); + if ( + normalized.includes('resource_exhausted') || + normalized.includes('service unavailable') + ) { + return true; + } + } + return false; +} + +function shouldRetryNativeTransportError(error: unknown): boolean { + const normalized = String((error as any)?.message || error || '').toLowerCase(); + return ( + normalized.includes('timed out') || + normalized.includes('timeout') || + normalized.includes('abort') || + normalized.includes('econnreset') || + normalized.includes('connect') || + normalized.includes('socket') || + normalized.includes('fetch failed') + ); +} + +function splitProviderBase( + family: NativeFamily, + providerUrl: string +): { origin: string; prefixPath: string; versionPath: string } { + const parsed = new URL(providerUrl); + const trimmedPath = parsed.pathname.replace(/\/+$/, ''); + const match = trimmedPath.match(/^(.*?)(\/v\d+(?:beta)?)(?:\/.*)?$/i); + const prefixPath = + match?.[1] && match[1] !== '/' ? match[1] : ''; + const versionPath = + match?.[2] || + (family === 'gemini' ? '/v1beta' : '/v1'); + return { origin: parsed.origin, prefixPath, versionPath }; +} + +function joinUrlPath(...segments: string[]): string { + const cleaned = segments + .filter(Boolean) + .map((segment, index) => { + if (index === 0) return segment.replace(/\/+$/, ''); + return segment.replace(/^\/+/, '').replace(/\/+$/, ''); + }) + .filter(segment => segment.length > 0); + return cleaned.length > 0 ? cleaned.join('/') : ''; +} + +function buildNativeUpstreamUrl( + family: NativeFamily, + providerUrl: string, + subpath: string, + queryString: string +): string { + const { origin, prefixPath, versionPath } = splitProviderBase( + family, + providerUrl + ); + const normalizedSubpath = + typeof subpath === 'string' && subpath.trim() + ? (subpath.startsWith('/') ? subpath : `/${subpath}`) + : '/'; + + let path = normalizedSubpath; + if (!/^\/v\d/i.test(normalizedSubpath)) { + path = `/${joinUrlPath(prefixPath, versionPath, normalizedSubpath)}`; + } else if (prefixPath) { + path = `/${joinUrlPath(prefixPath, normalizedSubpath)}`; + } + + const upstream = new URL(path, origin); + if (queryString) upstream.search = queryString.startsWith('?') ? queryString : `?${queryString}`; + return upstream.toString(); +} + +function buildUpstreamHeaders( + request: Request, + family: NativeFamily, + provider: LoadedProviderData, + stream: boolean +): Record { + const headers: Record = {}; + for (const [name, value] of Object.entries(request.headers)) { + const normalizedName = name.toLowerCase(); + if ( + HOP_BY_HOP_RESPONSE_HEADERS.has(normalizedName) || + normalizedName === 'host' || + normalizedName === 'authorization' || + normalizedName === 'x-api-key' || + normalizedName === 'x-goog-api-key' || + normalizedName === 'content-length' + ) { + continue; + } + headers[name] = value; + } + + if (stream && !headers.accept) { + headers.accept = 'text/event-stream'; + } + + const providerKey = String(provider.apiKey || '').trim(); + if (family === 'anthropic') { + headers['x-api-key'] = providerKey; + headers['anthropic-version'] = + getHeaderValue(request.headers, 'anthropic-version') || + process.env.ANTHROPIC_VERSION || + '2023-06-01'; + const anthropicBeta = getHeaderValue(request.headers, 'anthropic-beta'); + if (anthropicBeta) headers['anthropic-beta'] = anthropicBeta; + } else if ( + family === 'openai' || + family === 'openrouter' || + family === 'deepseek' || + family === 'xai' + ) { + headers.Authorization = `Bearer ${providerKey}`; + } else { + delete headers.Authorization; + delete headers['x-api-key']; + delete headers['x-goog-api-key']; + } + + return headers; +} + +function copyUpstreamHeaders(upstreamHeaders: Headers, response: Response): void { + for (const [name, value] of upstreamHeaders.entries()) { + const normalized = name.toLowerCase(); + if ( + HOP_BY_HOP_RESPONSE_HEADERS.has(normalized) || + !SAFE_NATIVE_RESPONSE_HEADERS.has(normalized) + ) { + continue; + } + response.setHeader(name, value); + } +} + +function buildNativeUpstreamErrorBody( + family: NativeFamily, + statusCode: number, + timestamp: string +): { error: string; timestamp: string } { + return { + error: buildSafeUpstreamErrorMessage(statusCode, { + label: `${family} native request`, + rateLimitMessage: + 'Rate limit or quota exceeded at the upstream provider. Please retry later.' + }), + timestamp + }; +} + +function extractNativeUsage( + family: NativeFamily, + payload: any +): { promptTokens?: number; completionTokens?: number; totalTokens?: number } { + if (family === 'gemini') { + return { + promptTokens: + typeof payload?.usageMetadata?.promptTokenCount === 'number' + ? payload.usageMetadata.promptTokenCount + : undefined, + completionTokens: + typeof payload?.usageMetadata?.candidatesTokenCount === 'number' + ? payload.usageMetadata.candidatesTokenCount + : undefined, + totalTokens: + typeof payload?.usageMetadata?.totalTokenCount === 'number' + ? payload.usageMetadata.totalTokenCount + : undefined + }; + } + return extractUsageTokens(payload?.usage || payload?.message?.usage || payload); +} + +function extractNativeSubpath(request: Request, familySegment: string): { + subpath: string; + queryString: string; +} { + const [pathOnly, queryString = ''] = String(request.url || request.path || '').split('?'); + const prefix = `/native/${familySegment}`; + let subpath = pathOnly.startsWith(prefix) ? pathOnly.slice(prefix.length) : '/'; + if (!subpath) subpath = '/'; + return { subpath, queryString }; +} + +async function authAndUsageMiddleware( + request: Request, + response: Response, + next: () => void +) { + const timestamp = new Date().toISOString(); + return runAuthMiddleware(request, response, next, { + extractApiKey: req => { + const authorization = req.headers.authorization; + if (typeof authorization === 'string' && authorization.startsWith('Bearer ')) { + return normalizeApiKey(authorization.slice(7)); + } + return normalizeApiKey( + (typeof req.headers['x-api-key'] === 'string' + ? req.headers['x-api-key'] + : typeof req.headers['x-goog-api-key'] === 'string' + ? req.headers['x-goog-api-key'] + : null) + ); + }, + onMissingApiKey: async req => { + const errDetail = { message: 'Missing API key.' }; + await logError(errDetail, req); + return { + status: 401, + body: { error: 'Authentication or configuration failed', timestamp } + }; + }, + onInvalidApiKey: async (req, details) => { + const clientMessage = + details.statusCode === 429 + ? 'Rate limit or quota exceeded. Please retry later.' + : 'Unauthorized: Invalid API key.'; + const logMsg = `Invalid API key. ${details.error || ''}`.trim(); + await logError( + { + message: logMsg, + details: details.error, + apiKey: details.apiKey ? redactToken(details.apiKey) : undefined + }, + req + ); + return { + status: details.statusCode, + body: { error: clientMessage, timestamp } + }; + }, + onInternalError: async (req, error) => { + await logError(error, req); + return { + status: 500, + body: { + error: 'Internal Server Error', + reference: 'Error during authentication processing.', + timestamp + } + }; + } + }); +} + +function rateLimitMiddleware( + request: Request, + response: Response, + next: () => void +) { + const timestamp = new Date().toISOString(); + return runRateLimitMiddleware(request, response, next, requestTimestamps, { + onMissingContext: req => ({ + status: 500, + body: { + error: 'Internal Server Error', + reference: 'Configuration error for rate limiting.', + timestamp + } + }), + onDenied: (_req, details) => ({ + status: 429, + body: { + error: `Rate limit exceeded: Max ${details.limit} ${details.window.toUpperCase()}.`, + timestamp + } + }) + }); +} + +async function handleNativeProviderRequest( + request: Request, + response: Response +): Promise { + const timestamp = new Date().toISOString(); + try { + const requestedFamilySegment = String(request.params.family || '').trim(); + const autoFamilyRouting = isAutoNativeFamilySelector(requestedFamilySegment); + + const contentType = String(getHeaderValue(request.headers, 'content-type') || ''); + const intakeFamilyLabel = requestedFamilySegment || 'unknown'; + const bodyBuffer = + request.method === 'GET' || request.method === 'HEAD' + ? Buffer.alloc(0) + : await withBufferedRequestBody( + request, + { + label: `native:${intakeFamilyLabel}:body-read`, + extra: { + route: request.path, + requestId: request.requestId, + family: intakeFamilyLabel + } + }, + rawBody => Buffer.from(rawBody) + ); + const parsedBody = normalizeJsonBody(bodyBuffer, contentType); + const { subpath, queryString } = extractNativeSubpath(request, requestedFamilySegment); + const isModelsListRoute = + request.method === 'GET' && (subpath === '/models' || subpath === '/v1/models'); + let outboundBodyBuffer = bodyBuffer; + let nativeResponsesHistoryContext: NativeResponsesHistoryContext | null = null; + + let family = canonicalizeNativeFamily(requestedFamilySegment); + let modelId: string | null = null; + + if (autoFamilyRouting) { + if (isModelsListRoute) { + family = 'openai'; + response.setHeader('X-AnyGPT-Routed-Family', family); + } else { + modelId = extractRoutingModelId(parsedBody, subpath); + if (modelId) { + const providersForFamilyResolution = + await dataManager.load('providers'); + family = inferNativeFamilyFromModelAndProviders( + modelId, + providersForFamilyResolution + ); + if (!family) { + response.status(404).json({ + error: `Auto native routing could not resolve provider family for model '${modelId}'.`, + timestamp + }); + return; + } + } else { + family = inferNativeFamilyFromSubpathHeuristics(subpath); + if (!family) { + response.status(400).json({ + error: + "Auto native routing requires a model id in request body.model or path '/models/{id}'.", + timestamp + }); + return; + } + } + response.setHeader('X-AnyGPT-Routed-Family', family); + } + } else { + if (!family) { + response.status(404).json({ + error: `Unsupported native provider family '${requestedFamilySegment}'.`, + timestamp + }); + return; + } + modelId = extractNativeModelId(family, subpath, parsedBody); + } + + if ( + family === 'openai' && + request.method === 'GET' && + (subpath === '/models' || subpath === '/v1/models') + ) { + const payload = await buildModelsPayload(request); + applyResponseHeaders(response, payload.headers); + if (!payload.ok) { + response.status(payload.statusCode).json(payload.body); + return; + } + response.json(payload.body); + return; + } + + const isNativeResponsesRoute = + supportsNativeResponsesHistory(family) && isOpenAiResponsesSubpath(subpath); + + if ( + isNativeResponsesRoute && + parsedBody && + typeof parsedBody === 'object' && + !Array.isArray(parsedBody) + ) { + const sanitizedPayload = sanitizeOpenAIResponseToolSchemas(parsedBody); + const outboundPayload: Record = { ...sanitizedPayload }; + const ownerScope = buildNativeResponsesOwnerScope(request); + const hasInput = Object.prototype.hasOwnProperty.call(outboundPayload, 'input'); + let inputDelta: any[] = []; + if (hasInput) { + try { + inputDelta = normalizeNativeResponsesInput(outboundPayload.input); + } catch (inputError: any) { + response.status(400).json({ + error: + inputError?.message || + 'Bad Request: invalid responses input payload.', + timestamp + }); + return; + } + } + + let mergedInput = cloneResponsesHistoryValue(inputDelta); + let previousEntry: StoredResponsesHistoryEntry | null = null; + const previousResponseId = + typeof outboundPayload.previous_response_id === 'string' && + outboundPayload.previous_response_id.trim() + ? outboundPayload.previous_response_id.trim() + : ''; + + if (previousResponseId) { + const loadedPreviousEntry = await loadResponsesHistoryEntry(previousResponseId); + previousEntry = isNativeResponsesHistoryEntryUsable( + loadedPreviousEntry, + ownerScope + ) + ? loadedPreviousEntry + : null; + if (!previousEntry) { + response.status(400).json({ + error: `Previous response with id '${previousResponseId}' not found.`, + timestamp + }); + return; + } + try { + const mergedHistory = await mergeResponsesHistoryInput( + previousEntry, + inputDelta + ); + mergedInput = cloneResponsesHistoryValue(mergedHistory.input); + outboundPayload.input = mergedHistory.input; + delete outboundPayload.previous_response_id; + } catch (historyError: any) { + response.status(400).json({ + error: + historyError?.message || + 'Stored responses history could not be reconstructed.', + timestamp + }); + return; + } + } + + nativeResponsesHistoryContext = { + proxyResponseId: createResponsesItemId('resp'), + ownerScope, + inputDelta: cloneResponsesHistoryValue(inputDelta), + mergedInput, + previousEntry + }; + outboundBodyBuffer = Buffer.from(JSON.stringify(outboundPayload), 'utf8'); + } + + if ( + modelId && + request.tierLimits && + !isModelAllowedForTier(modelId, request.tierLimits as TierData) + ) { + const details = buildModelAccessError(modelId, request.tierLimits as TierData); + response.status(details.statusCode).json({ error: details, timestamp }); + return; + } + + const providers = await dataManager.load('providers'); + const candidates = listNativeProviderCandidates(providers, family, modelId); + const availableCandidates = candidates.filter( + provider => !isNativeProviderCoolingDown(provider.id, modelId) + ); + const provider = availableCandidates[0] || selectBestNativeProvider(providers, family, modelId); + if (!provider) { + response.status(404).json({ + error: `No active ${family}-compatible provider is available${modelId ? ` for model '${modelId}'` : ''}.`, + timestamp + }); + return; + } + const attemptProviders = availableCandidates.length > 0 ? availableCandidates : [provider]; + + const stream = + getHeaderValue(request.headers, 'accept')?.includes('text/event-stream') || + parsedBody?.stream === true; + const userApiKey = request.apiKey!; + const queueLane = isNativeResponsesRoute ? 'responses' : 'shared'; + const requestQueue = getRequestQueueForLane(queueLane); + const releaseQueueSlot = await requestQueue.acquire(); + let lastFailure: + | { + statusCode: number; + retryAfter: string | null; + body: { error: string; timestamp: string }; + } + | null = null; + + try { + for (const attemptProvider of attemptProviders) { + try { + let upstreamUrl = buildNativeUpstreamUrl( + family, + String(attemptProvider.provider_url || '').trim(), + subpath, + queryString + ); + if (family === 'gemini') { + const url = new URL(upstreamUrl); + url.searchParams.set('key', String(attemptProvider.apiKey || '').trim()); + upstreamUrl = url.toString(); + } + + const headers = buildUpstreamHeaders( + request, + family, + attemptProvider, + Boolean(stream) + ); + const upstreamRes = await fetchWithTimeout(upstreamUrl, { + method: request.method, + headers, + body: + request.method === 'GET' || request.method === 'HEAD' + ? undefined + : new Uint8Array(outboundBodyBuffer) + }); + + if (!upstreamRes.ok) { + const errorText = await upstreamRes.text(); + const retryAfterHeader = upstreamRes.headers.get('retry-after'); + const retryAfterMs = parseRetryAfterMs(retryAfterHeader); + const shouldFailOver = shouldTryNextNativeProvider( + family, + upstreamRes.status, + errorText + ); + if (shouldFailOver) { + markNativeProviderCooldown( + attemptProvider.id, + modelId, + retryAfterMs + ); + } + await logError( + { + message: shouldFailOver + ? 'Native provider passthrough request failed; trying next provider.' + : 'Native provider passthrough request failed.', + family, + providerId: attemptProvider.id, + modelId, + statusCode: upstreamRes.status, + retryAfterMs, + failover: shouldFailOver, + upstream: errorText + }, + request + ); + lastFailure = { + statusCode: upstreamRes.status, + retryAfter: retryAfterHeader, + body: buildNativeUpstreamErrorBody( + family, + upstreamRes.status, + timestamp + ) + }; + if (shouldFailOver) continue; + + if (retryAfterHeader) { + response.setHeader('Retry-After', retryAfterHeader); + } + response.status(upstreamRes.status).json(lastFailure.body); + return; + } + + response.status(upstreamRes.status); + copyUpstreamHeaders(upstreamRes.headers, response); + + const upstreamContentType = String(upstreamRes.headers.get('content-type') || ''); + if (upstreamContentType.includes('text/event-stream')) { + if (!upstreamRes.body) { + response.end(); + return; + } + const reader = upstreamRes.body.getReader(); + const decoder = new TextDecoder(); + let promptTokens: number | undefined; + let completionTokens: number | undefined; + let totalTokens: number | undefined; + const shouldRewriteNativeResponsesIds = + isNativeResponsesRoute && nativeResponsesHistoryContext !== null; + const proxyResponseId = shouldRewriteNativeResponsesIds + ? nativeResponsesHistoryContext?.proxyResponseId || null + : null; + let streamUpstreamResponseId: string | null = null; + let streamCreatedAt: number | undefined; + let streamOutput: any[] | null = null; + let streamOutputText = ''; + let streamToolCalls: Record[] = []; + const parser = createSseDataParser((dataLine, eventName) => { + if (!dataLine) return; + if (shouldRewriteNativeResponsesIds && dataLine === '[DONE]') { + response.write(serializeSseEvent(undefined, '[DONE]')); + return; + } + let outboundDataLine = dataLine; + try { + const parsed = JSON.parse(dataLine); + const normalizedPayload = + parsed?.response && typeof parsed.response === 'object' + ? parsed.response + : parsed; + const usage = extractNativeUsage(family, normalizedPayload); + if (typeof usage.promptTokens === 'number') promptTokens = usage.promptTokens; + if (typeof usage.completionTokens === 'number') completionTokens = usage.completionTokens; + if (typeof usage.totalTokens === 'number') totalTokens = usage.totalTokens; + + if (shouldRewriteNativeResponsesIds) { + if ( + !streamUpstreamResponseId && + typeof normalizedPayload?.id === 'string' && + normalizedPayload.id.trim() + ) { + streamUpstreamResponseId = normalizedPayload.id.trim(); + } + if ( + typeof normalizedPayload?.created === 'number' && + Number.isFinite(normalizedPayload.created) + ) { + streamCreatedAt = Math.floor(normalizedPayload.created); + } + if (Array.isArray(normalizedPayload?.output)) { + streamOutput = cloneResponsesHistoryValue(normalizedPayload.output); + for (const outputItem of normalizedPayload.output) { + collectNativeResponsesToolCallsFromItem(streamToolCalls, outputItem); + } + } + if ( + eventName === 'response.output_item.added' || + eventName === 'response.output_item.done' + ) { + collectNativeResponsesToolCallsFromItem( + streamToolCalls, + parsed?.item + ); + } + if (eventName === 'response.function_call_arguments.done') { + upsertNativeResponsesToolCall(streamToolCalls, { + id: parsed?.item_id, + call_id: parsed?.call_id, + name: parsed?.name, + arguments: parsed?.arguments, + status: 'completed' + }); + } + const deltaText = + typeof parsed?.output_text_delta === 'string' + ? parsed.output_text_delta + : typeof parsed?.response?.output_text_delta === 'string' + ? parsed.response.output_text_delta + : typeof parsed?.delta === 'string' + ? parsed.delta + : typeof parsed?.response?.delta === 'string' + ? parsed.response.delta + : ''; + if (deltaText) { + streamOutputText += deltaText; + } + if ( + typeof normalizedPayload?.output_text === 'string' + ) { + streamOutputText = normalizedPayload.output_text; + } + if ( + eventName === 'response.completed' && + typeof parsed?.response?.output_text === 'string' + ) { + streamOutputText = parsed.response.output_text; + } + outboundDataLine = JSON.stringify( + rewriteNativeResponsesEventPayload(parsed, proxyResponseId!) + ); + } + } catch { + // Ignore malformed SSE while piping raw bytes. + } + if (shouldRewriteNativeResponsesIds) { + response.write(serializeSseEvent(eventName, outboundDataLine)); + } + }); + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + if (!value || value.length === 0) continue; + const decodedChunk = decoder.decode(value, { stream: true }); + if (shouldRewriteNativeResponsesIds) { + if (decodedChunk) parser(decodedChunk); + continue; + } + response.write(value); + if (decodedChunk) parser(decodedChunk); + } + const finalChunk = decoder.decode(); + if (finalChunk) parser(finalChunk); + } finally { + reader.releaseLock(); + } + if (typeof totalTokens === 'number' && totalTokens > 0) { + await updateUserTokenUsage(totalTokens, userApiKey, { + modelId: modelId || undefined, + promptTokens, + completionTokens + }); + } + if ( + isNativeResponsesRoute && + nativeResponsesHistoryContext && + (streamUpstreamResponseId || + streamOutput || + streamOutputText || + streamToolCalls.length > 0) + ) { + await persistNativeResponsesHistoryEntry({ + context: nativeResponsesHistoryContext, + responsePayload: { + id: nativeResponsesHistoryContext.proxyResponseId, + model: modelId || undefined, + created: streamCreatedAt, + output: streamOutput, + output_text: streamOutputText, + tool_calls: + streamToolCalls.length > 0 + ? cloneResponsesHistoryValue(streamToolCalls) + : undefined + }, + modelId, + request, + routedFamily: family, + providerId: attemptProvider.id, + upstreamResponseId: streamUpstreamResponseId + }); + } + if (!response.completed) response.end(); + return; + } + + const rawText = await upstreamRes.text(); + let parsedResponse: any = null; + try { + parsedResponse = rawText ? JSON.parse(rawText) : null; + } catch { + parsedResponse = null; + } + + if (parsedResponse && typeof parsedResponse === 'object') { + const usage = extractNativeUsage(family, parsedResponse); + if (typeof usage.totalTokens === 'number' && usage.totalTokens > 0) { + await updateUserTokenUsage(usage.totalTokens, userApiKey, { + modelId: modelId || undefined, + promptTokens: usage.promptTokens, + completionTokens: usage.completionTokens + }); + } + let responsePayloadForClient = parsedResponse; + let responseTextForClient = rawText; + let upstreamResponseId: string | undefined; + if (isNativeResponsesRoute && nativeResponsesHistoryContext) { + upstreamResponseId = + typeof parsedResponse?.id === 'string' && parsedResponse.id.trim() + ? parsedResponse.id.trim() + : undefined; + responsePayloadForClient = rewriteNativeResponsesResponseObject( + parsedResponse, + nativeResponsesHistoryContext.proxyResponseId + ); + responseTextForClient = JSON.stringify(responsePayloadForClient); + await persistNativeResponsesHistoryEntry({ + context: nativeResponsesHistoryContext, + responsePayload: responsePayloadForClient, + modelId, + request, + routedFamily: family, + providerId: attemptProvider.id, + upstreamResponseId + }); + } + if (!String(response.getHeader?.('Content-Type') || '').trim()) { + response.setHeader('Content-Type', 'application/json'); + } + response.end(responseTextForClient); + return; + } + + response.end(rawText); + return; + } catch (error: any) { + if (!shouldRetryNativeTransportError(error)) throw error; + markNativeProviderCooldown(attemptProvider.id, modelId); + await logError( + { + message: 'Native provider passthrough transport failed; trying next provider.', + family, + providerId: attemptProvider.id, + modelId, + errorMessage: error?.message || String(error) + }, + request + ); + lastFailure = { + statusCode: 502, + retryAfter: null, + body: { + error: 'Native provider request failed before the upstream response was received.', + timestamp + } + }; + continue; + } + } + + if (lastFailure) { + response.status(lastFailure.statusCode); + if (lastFailure.retryAfter) { + response.setHeader('Retry-After', lastFailure.retryAfter); + } + response.json(lastFailure.body); + return; + } + + response.status(503).json({ + error: `All ${family}-compatible providers are temporarily unavailable${modelId ? ` for model '${modelId}'` : ''}.`, + timestamp + }); + } finally { + releaseQueueSlot(); + } + } catch (error: any) { + await logError( + { + message: 'Native provider passthrough request crashed.', + errorMessage: error?.message || String(error), + errorStack: error?.stack + }, + request + ); + const errorMessage = String(error?.message || '').toLowerCase(); + const backpressureRetryAfterSeconds = + getBackpressureRetryAfterSeconds(error); + const statusCode = + error?.code === 'MEMORY_PRESSURE' || + error?.code === 'QUEUE_OVERLOADED' || + error?.code === 'QUEUE_WAIT_TIMEOUT' + ? 503 + : errorMessage.includes('timed out') || errorMessage.includes('abort') + ? 504 + : 502; + if ( + typeof backpressureRetryAfterSeconds === 'number' && + backpressureRetryAfterSeconds > 0 + ) { + response.setHeader('Retry-After', String(backpressureRetryAfterSeconds)); + } + response.status(statusCode).json({ + error: + statusCode === 503 + ? error?.code === 'MEMORY_PRESSURE' + ? 'Service temporarily unavailable: server is under memory pressure. Retry in a few seconds.' + : 'Service temporarily unavailable: request queue is busy. Retry in a few seconds.' + : statusCode === 504 + ? 'Native provider request timed out.' + : 'Native provider request failed before the upstream response was received.', + timestamp + }); + } +} + +const nativeHandlers = [authAndUsageMiddleware, rateLimitMiddleware, async ( + request: Request, + response: Response +) => { + await handleNativeProviderRequest(request, response); +}] as const; + +for (const method of ['get', 'post', 'put', 'patch', 'delete', 'options', 'head'] as const) { + router[method]('/:family', ...nativeHandlers); + router[method]('/:family/*', ...nativeHandlers); +} + +const nativeProviderRouter = router; +export default nativeProviderRouter; diff --git a/apps/api/server.ts b/apps/api/server.ts index 432167e..74409ad 100644 --- a/apps/api/server.ts +++ b/apps/api/server.ts @@ -52,6 +52,7 @@ import groqRouter from './routes/groq.js'; import ollamaRouter from './routes/ollama.js'; import openrouterRouter from './routes/openrouter.js'; import openapiRouter from './routes/openapi.js'; +import nativeProviderRouter from './routes/nativeProviders.js'; import { attachWebSocket } from './ws/wsServer.js'; import { attachRealtimeWebSocket } from './ws/realtime.js'; @@ -542,6 +543,9 @@ async function startServer() { } else { console.log(' 𐄂 OpenRouter compatible routes disabled.'); } + app.use('/native', nativeProviderRouter); + console.log(' ✓ Native passthrough router enabled: /native/:family/* and /native/auto/* (legacy alias: /native/mutual/*)'); + console.log(' ✓ Native family aliases now resolve via passthrough router (e.g. /native/openai/v1, /native/anthropic/v1, /native/gemini/v1beta).'); console.log(''); // Newline for cleaner log output // --- WebSocket Endpoint --- From 0f8336630102caaf971accb8711184a50c6c7bd7 Mon Sep 17 00:00:00 2001 From: skullcmd Date: Sun, 5 Apr 2026 05:31:11 +0000 Subject: [PATCH 2/4] feat(exposure-studio): add cleaned research studio --- apps/exposure-studio/.gitignore | 12 + .../exposure-studio/.lane-secrets.env.example | 6 + apps/exposure-studio/Cargo.lock | 1587 +++++++++++++++++ apps/exposure-studio/Cargo.toml | 12 + apps/exposure-studio/PRODUCT_SCOPE.md | 63 + apps/exposure-studio/README.md | 130 ++ .../backend/implementation_lane_runner.py | 258 +++ apps/exposure-studio/backend/lane_shared.py | 442 +++++ .../backend/requirements-youtube-research.txt | 3 + .../backend/website_research_runner.py | 464 +++++ .../backend/youtube_research_runner.py | 935 ++++++++++ apps/exposure-studio/data/assets.json | 80 + .../data/compliance_sources.json | 58 + apps/exposure-studio/data/findings.json | 94 + .../data/research_web_targets.json | 32 + .../data/testing_methodologies.json | 70 + .../data/vulnerability_intelligence.json | 50 + apps/exposure-studio/frontend/app.js | 603 +++++++ apps/exposure-studio/frontend/index.html | 123 ++ apps/exposure-studio/frontend/styles.css | 245 +++ apps/exposure-studio/run.sh | 64 + apps/exposure-studio/src/main.rs | 885 +++++++++ apps/exposure-studio/src/models.rs | 505 ++++++ 23 files changed, 6721 insertions(+) create mode 100644 apps/exposure-studio/.gitignore create mode 100644 apps/exposure-studio/.lane-secrets.env.example create mode 100644 apps/exposure-studio/Cargo.lock create mode 100644 apps/exposure-studio/Cargo.toml create mode 100644 apps/exposure-studio/PRODUCT_SCOPE.md create mode 100644 apps/exposure-studio/README.md create mode 100644 apps/exposure-studio/backend/implementation_lane_runner.py create mode 100644 apps/exposure-studio/backend/lane_shared.py create mode 100644 apps/exposure-studio/backend/requirements-youtube-research.txt create mode 100644 apps/exposure-studio/backend/website_research_runner.py create mode 100644 apps/exposure-studio/backend/youtube_research_runner.py create mode 100644 apps/exposure-studio/data/assets.json create mode 100644 apps/exposure-studio/data/compliance_sources.json create mode 100644 apps/exposure-studio/data/findings.json create mode 100644 apps/exposure-studio/data/research_web_targets.json create mode 100644 apps/exposure-studio/data/testing_methodologies.json create mode 100644 apps/exposure-studio/data/vulnerability_intelligence.json create mode 100644 apps/exposure-studio/frontend/app.js create mode 100644 apps/exposure-studio/frontend/index.html create mode 100644 apps/exposure-studio/frontend/styles.css create mode 100755 apps/exposure-studio/run.sh create mode 100644 apps/exposure-studio/src/main.rs create mode 100644 apps/exposure-studio/src/models.rs diff --git a/apps/exposure-studio/.gitignore b/apps/exposure-studio/.gitignore new file mode 100644 index 0000000..812d14d --- /dev/null +++ b/apps/exposure-studio/.gitignore @@ -0,0 +1,12 @@ +/target +.lane-secrets.env +backend/.venv/ +backend/__pycache__/ +backend/*.py[cod] +data/.generated/ +data/.lane-locks/ +data/.lane-snapshots/ +data/.youtube-lane-cache/ +cookies.txt +*_cookies.txt +*.cookies.txt diff --git a/apps/exposure-studio/.lane-secrets.env.example b/apps/exposure-studio/.lane-secrets.env.example new file mode 100644 index 0000000..3b846f1 --- /dev/null +++ b/apps/exposure-studio/.lane-secrets.env.example @@ -0,0 +1,6 @@ +export YOUTUBE_LANE_COOKIES_FILE="/absolute/path/to/cookies.txt" +export YOUTUBE_LANE_PROXY="http://user:pass@host:port" +export YOUTUBE_LANE_MAX_VIDEOS="20" +export WEBSITE_LANE_CRAWL_DEPTH="1" +export YOUTUBE_LANE_RETRY_ATTEMPTS="3" +export YOUTUBE_LANE_RETRY_BACKOFF_SECONDS="5" diff --git a/apps/exposure-studio/Cargo.lock b/apps/exposure-studio/Cargo.lock new file mode 100644 index 0000000..0271951 --- /dev/null +++ b/apps/exposure-studio/Cargo.lock @@ -0,0 +1,1587 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "axum" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cc" +version = "1.2.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e928d4b69e3077709075a938a05ffbedfa53a84c8f766efbf8220bb1ff60e1" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "http-range-header" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9171a2ea8a68358193d15dd5d70c1c10a2afc3e7e4c5bc92bc9f025cebd7359c" + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "iri-string" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4c90f45aa2e6eacbe8645f77fdea542ac97a494bcd117a67df9ff4d611f995" +dependencies = [ + "cfg-if", + "futures-util", + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.183" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" + +[[package]] +name = "litemap" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "mio" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "potential_utf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustls" +version = "0.23.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "surfacescope" +version = "0.1.0" +dependencies = [ + "axum", + "reqwest", + "serde", + "serde_json", + "tokio", + "tower-http", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinystr" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "http-range-header", + "httpdate", + "iri-string", + "mime", + "mime_guess", + "percent-encoding", + "pin-project-lite", + "tokio", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.115" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6523d69017b7633e396a89c5efab138161ed5aafcbc8d3e5c5a42ae38f50495a" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d1faf851e778dfa54db7cd438b70758eba9755cb47403f3496edd7c8fc212f0" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.115" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e3a6c758eb2f701ed3d052ff5737f5bfe6614326ea7f3bbac7156192dc32e67" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.115" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "921de2737904886b52bcbb237301552d05969a6f9c40d261eb0533c8b055fedf" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.115" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a93e946af942b58934c604527337bad9ae33ba1d5c6900bbb41c2c07c2364a93" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84cde8507f4d7cfcb1185b8cb5890c494ffea65edbe1ba82cfd63661c805ed94" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" + +[[package]] +name = "writeable" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" + +[[package]] +name = "yoke" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/apps/exposure-studio/Cargo.toml b/apps/exposure-studio/Cargo.toml new file mode 100644 index 0000000..9cdd849 --- /dev/null +++ b/apps/exposure-studio/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "surfacescope" +version = "0.1.0" +edition = "2021" + +[dependencies] +axum = "0.8" +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +tokio = { version = "1", features = ["full"] } +tower-http = { version = "0.6", features = ["fs", "trace"] } diff --git a/apps/exposure-studio/PRODUCT_SCOPE.md b/apps/exposure-studio/PRODUCT_SCOPE.md new file mode 100644 index 0000000..f3da15c --- /dev/null +++ b/apps/exposure-studio/PRODUCT_SCOPE.md @@ -0,0 +1,63 @@ +# Product Scope + +## Positioning + +SurfaceScope is a safer alternative to public exposure-indexing workflows: + +- authorized assets only +- evidence-first triage +- disclosure-safe workflow +- compliance-aware reporting + +## Core jobs to be done + +1. Maintain an approved asset inventory with ownership, business criticality, and authorization state. +2. Ingest machine-readable findings from scanners, sensors, or internal feeds. +3. Show which findings need: + - owner confirmation + - remediation + - coordinated disclosure + - audit follow-up +4. Preserve a durable evidence log for each exposure and remediation decision. + +## Product guardrails + +- No anonymous third-party target enumeration features. +- No raw exploit guidance. +- No “better search engine for strangers’ infrastructure” positioning. +- New features should strengthen authorization proof, evidence quality, or disclosure workflow quality. + +## Competitive legal controls to encode + +- Require an explicit authorization basis and an internal reference for every monitored asset. +- Classify finding visibility so newly reported issues can stay in an internal or trusted-researcher window before any wider publication. +- Track report status, intended recipients, and escalation targets for each finding. +- Preserve opt-out, takedown, and abuse-handling workflow as first-class product work, not an afterthought. +- Keep privacy controls visible: data minimization, KYC/reviewer controls, and lawful-contact handling should be represented in the workflow rather than buried in policy text alone. + +## Evidence packet workflow to encode + +Every finding should be able to move through a consistent evidence-first workflow before remediation or disclosure decisions: + +1. **Authorization check** + - Link the finding to the approved asset record and exact approved resource. + - Preserve the authorization basis and internal approval reference used for collection. +2. **Collection metadata** + - Record the collection method, capture timestamp, reviewer, and whether the artifact came from a scanner, manual verification step, or owner-provided proof. + - Track redaction status so screenshots, headers, and response samples can be shared safely. +3. **Evidence readiness** + - Mark whether the evidence is sufficient for owner notification, remediation planning, coordinated disclosure, or audit follow-up. + - Distinguish missing-proof cases from verified exposure cases so the queue does not treat them the same way. +4. **Retention and handoff** + - Keep a durable storage reference for each evidence packet and record who is allowed to view it. + - Preserve the recipient list, escalation path, and disclosure window that applies once the packet is ready. + +This workflow keeps SurfaceScope focused on authorized assets, machine-readable findings, and evidence-backed remediation instead of unbounded public indexing. + +## Suggested next bounded improvements + +- importer for scanner exports +- asset-ownership attestation workflow +- disclosure packet export +- security.txt remediation and disclosure workflow improvements +- authenticated asset-group views diff --git a/apps/exposure-studio/README.md b/apps/exposure-studio/README.md new file mode 100644 index 0000000..8e439e9 --- /dev/null +++ b/apps/exposure-studio/README.md @@ -0,0 +1,130 @@ +# SurfaceScope + +SurfaceScope is an authorized asset exposure management product scaffold. + +It is intentionally **not** a public internet-wide recon engine. The product is designed for: + +- assets you own +- assets you operate +- assets you have explicit written authorization to assess + +The goal is to help a security team inventory exposed services, track machine-readable findings, and move from exposure evidence to owner-approved remediation and disclosure workflows. + +## Product shape + +- Rust backend with Axum in `src/` +- static frontend in `frontend/` +- curated demo inventory and source inputs in `data/` +- generated research lane outputs and caches in `data/.generated/` + +## Safe-use guardrails + +- Keep ownership and authorization state on every asset. +- Preserve evidence and disclosure workflow metadata with every finding. +- Prefer machine-readable imports and scoped findings over raw unauthenticated mass collection. +- Do not add features that optimize for scanning or indexing systems outside an approved asset boundary. + +## Local run + +```bash +bash apps/exposure-studio/run.sh +``` + +Then open `http://127.0.0.1:3325`. + +## Transcript research lane + +Build a transcript-first YouTube research lane with: + +```bash +bash apps/exposure-studio/run.sh youtube-lane \ + --channel-url https://www.youtube.com/@TheAIAutomators/videos +``` + +Build the website text lane with: + +```bash +bash apps/exposure-studio/run.sh website-lane +``` + +The website lane uses an explicit HTTPS allowlist from `data/research_web_targets.json` +as seeds, then performs a bounded same-origin crawl to cover linked pages without +drifting into unrelated sites. + +Build the unified implementation lane with: + +```bash +bash apps/exposure-studio/run.sh implementation-lane +``` + +This refreshes the website and YouTube lanes before merging them into the unified +implementation lane. If you only want to merge the latest generated lane files, +run: + +```bash +bash apps/exposure-studio/run.sh implementation-lane --skip-refresh +``` + +`refresh-all-lanes` is a convenience alias for the same full refresh: + +```bash +bash apps/exposure-studio/run.sh refresh-all-lanes +``` + +Run a preflight doctor check with: + +```bash +bash apps/exposure-studio/run.sh youtube-lane doctor +bash apps/exposure-studio/run.sh website-lane doctor +bash apps/exposure-studio/run.sh implementation-lane doctor +``` + +This bootstraps a local Python virtualenv in `backend/.venv`, downloads audio +for each video, transcribes speech with `faster-whisper`, and writes generated +lane outputs under `data/.generated/`. + +The `website-lane` path only needs the standard-library crawler. The heavier +YouTube transcription packages are installed for `youtube-lane`, +`refresh-all-lanes`, and `implementation-lane` refreshes. + +The lane is intentionally transcript-only for extraction. Video titles remain +visible as operator metadata, but implementation signals are derived from +speech-to-text output rather than title or description fallback. + +On cloud or data-center IPs, YouTube may block direct media access. The runner +keeps those videos in the lane with explicit `transcript_status` and +`error_summary` fields instead of silently falling back to metadata. If needed, +pass a cookies file or proxy: + +```bash +bash apps/exposure-studio/run.sh youtube-lane --cookies-file /path/to/cookies.txt +``` + +You can also point the runner at a browser cookie store directly: + +```bash +bash apps/exposure-studio/run.sh youtube-lane \ + --cookies-browser chrome \ + --cookies-browser-profile /path/to/ChromeProfile +``` + +If no explicit cookie source is configured, the runner tries to auto-detect +common Chrome/Chromium/Brave/Firefox cookie stores in your home directory. + +You can also configure defaults through environment variables: + +```bash +export YOUTUBE_LANE_COOKIES_FILE=/path/to/cookies.txt +export YOUTUBE_LANE_COOKIES_BROWSER=chrome +export YOUTUBE_LANE_COOKIES_PROFILE=/path/to/ChromeProfile +export YOUTUBE_LANE_PROXY=http://127.0.0.1:8080 +export YOUTUBE_LANE_MODEL=tiny +export WEBSITE_LANE_MAX_PAGES=3 +export WEBSITE_LANE_CRAWL_DEPTH=1 +export YOUTUBE_LANE_RETRY_ATTEMPTS=3 +export YOUTUBE_LANE_RETRY_BACKOFF_SECONDS=5 +bash apps/exposure-studio/run.sh youtube-lane doctor +bash apps/exposure-studio/run.sh website-lane +bash apps/exposure-studio/run.sh youtube-lane +bash apps/exposure-studio/run.sh implementation-lane +``` diff --git a/apps/exposure-studio/backend/implementation_lane_runner.py b/apps/exposure-studio/backend/implementation_lane_runner.py new file mode 100644 index 0000000..7b4d0c7 --- /dev/null +++ b/apps/exposure-studio/backend/implementation_lane_runner.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +"""Unified implementation lane runner for website + YouTube research.""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import Any + +from lane_shared import acquire_lane_lock, env_or_default, iso_now, load_last_good_lane, save_last_good_lane, website_page_importance, website_page_importance_legend, weighted_score_description, write_json_file + + +APP_ROOT = Path(__file__).resolve().parents[1] +BACKEND_ROOT = APP_ROOT / "backend" +DEFAULT_GENERATED_DIR = APP_ROOT / "data" / ".generated" +DEFAULT_YOUTUBE_OUTPUT = DEFAULT_GENERATED_DIR / "youtube_implementation_lane.json" +DEFAULT_WEBSITE_OUTPUT = DEFAULT_GENERATED_DIR / "website_implementation_lane.json" +DEFAULT_OUTPUT = DEFAULT_GENERATED_DIR / "implementation_lane.json" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build a unified implementation lane from YouTube and website sources.") + parser.add_argument( + "mode", + nargs="?", + default="run", + choices=("run", "doctor"), + help="Use 'doctor' to run preflight diagnostics across both source lanes.", + ) + parser.add_argument("--youtube-output", default=env_or_default("YOUTUBE_LANE_OUTPUT", str(DEFAULT_YOUTUBE_OUTPUT)), help="Path to the YouTube lane JSON.") + parser.add_argument("--website-output", default=env_or_default("WEBSITE_LANE_OUTPUT", str(DEFAULT_WEBSITE_OUTPUT)), help="Path to the website lane JSON.") + parser.add_argument("--output", default=env_or_default("IMPLEMENTATION_LANE_OUTPUT", str(DEFAULT_OUTPUT)), help="Path to the unified lane JSON.") + parser.add_argument("--skip-refresh", action="store_true", help="Merge existing lane files without rerunning source-specific runners.") + return parser.parse_args() + + +def run_child(command: list[str], allow_failure: bool = False) -> subprocess.CompletedProcess[str]: + result = subprocess.run(command, cwd=str(BACKEND_ROOT), text=True, capture_output=True) + if result.stderr: + sys.stderr.write(result.stderr) + if not allow_failure and result.returncode != 0: + raise RuntimeError(f"command failed ({result.returncode}): {' '.join(command)}") + return result + + +def load_json(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def compute_status(coverage: dict[str, int]) -> str: + youtube_total = int(coverage.get("youtube_video_count") or 0) + youtube_success = int(coverage.get("youtube_transcribed_video_count") or 0) + youtube_fail = int(coverage.get("youtube_failed_video_count") or 0) + website_total = int(coverage.get("website_page_count") or 0) + website_success = int(coverage.get("website_ingested_page_count") or 0) + website_fail = int(coverage.get("website_failed_page_count") or 0) + + active_sources = 0 + successful_sources = 0 + + if youtube_total > 0: + active_sources += 1 + if youtube_success > 0: + successful_sources += 1 + if website_total > 0: + active_sources += 1 + if website_success > 0: + successful_sources += 1 + + if successful_sources == 0: + return "unhealthy" + + if active_sources > successful_sources: + return "degraded" + + if youtube_fail > youtube_success: + return "degraded" + if website_fail > website_success: + return "degraded" + + return "healthy" + + +def merge_recommendations(youtube_lane: dict[str, Any], website_lane: dict[str, Any]) -> list[dict[str, Any]]: + merged: dict[str, dict[str, Any]] = {} + website_pages = { + str(page.get("page_id") or ""): page + for page in website_lane.get("pages") or [] + if isinstance(page, dict) + } + + def upsert(source: str, recommendation: dict[str, Any], item_ids: list[str], weight: float) -> None: + signal_id = str(recommendation.get("id") or "").strip() + if not signal_id: + return + bucket = merged.setdefault( + signal_id, + { + "id": signal_id, + "title": recommendation.get("title") or signal_id, + "priority": recommendation.get("priority") or "medium", + "rationale": recommendation.get("rationale") or "", + "suggested_paths": list(recommendation.get("suggested_paths") or []), + "suggested_paths_by_source": {}, + "supporting_item_ids": [], + "supporting_source_types": [], + "signal_count": 0, + "weighted_signal_score": 0.0, + "source_count": 0, + }, + ) + bucket["signal_count"] += int(recommendation.get("signal_count") or len(item_ids)) + bucket["weighted_signal_score"] += weight + for item_id in item_ids: + if item_id not in bucket["supporting_item_ids"]: + bucket["supporting_item_ids"].append(item_id) + if source not in bucket["supporting_source_types"]: + bucket["supporting_source_types"].append(source) + bucket["source_count"] = len(bucket["supporting_source_types"]) + source_paths = recommendation.get("suggested_paths_by_source", {}).get(source) if isinstance(recommendation.get("suggested_paths_by_source"), dict) else None + if source_paths: + bucket["suggested_paths_by_source"][source] = list(source_paths) + elif recommendation.get("suggested_paths"): + bucket["suggested_paths_by_source"][source] = list(recommendation.get("suggested_paths") or []) + for path in recommendation.get("suggested_paths") or []: + if path not in bucket["suggested_paths"]: + bucket["suggested_paths"].append(path) + + for recommendation in youtube_lane.get("recommendations") or []: + item_ids = list(recommendation.get("supporting_video_ids") or []) + upsert("youtube", recommendation, item_ids, float(len(item_ids) or 0)) + + for recommendation in website_lane.get("recommendations") or []: + item_ids = list(recommendation.get("supporting_page_ids") or []) + total_weight = 0.0 + for item_id in item_ids: + total_weight += float(website_pages.get(item_id, {}).get("importance_score") or website_page_importance(website_pages.get(item_id, {}).get("category"))) + upsert("website", recommendation, item_ids, total_weight) + + recommendations = list(merged.values()) + recommendations.sort(key=lambda item: (-item["source_count"], -item["weighted_signal_score"], -item["signal_count"], item["title"])) + return recommendations + + +def preserve_last_good_implementation_lane(output_path: Path, current_lane: dict[str, Any]) -> dict[str, Any]: + if str(current_lane.get("status") or "").strip().lower() == "healthy": + return current_lane + + previous_lane = load_last_good_lane(output_path) + if not previous_lane: + return current_lane + + if str(previous_lane.get("status") or "").strip().lower() != "healthy": + return current_lane + + preserved_lane = dict(previous_lane) + preserved_lane["restored_at"] = iso_now() + preserved_lane["restored_reason"] = ( + f"Preserved last good unified lane because the latest refresh status was " + f"{current_lane.get('status') or 'unknown'} while the previous snapshot was healthy." + ) + preserved_lane["last_attempt"] = { + "generated_at": current_lane.get("generated_at"), + "status": current_lane.get("status"), + "coverage": current_lane.get("coverage"), + } + return preserved_lane + + +def run_doctor(args: argparse.Namespace) -> int: + youtube = run_child([sys.executable, str(BACKEND_ROOT / "youtube_research_runner.py"), "doctor"], allow_failure=True) + website = run_child([sys.executable, str(BACKEND_ROOT / "website_research_runner.py"), "doctor"], allow_failure=True) + + youtube_report = json.loads(youtube.stdout or "{}") + website_report = json.loads(website.stdout or "{}") + + statuses = [youtube_report.get("status"), website_report.get("status")] + healthy_count = sum(status == "healthy" for status in statuses) + degraded_count = sum(status == "degraded" for status in statuses) + if healthy_count == len(statuses): + status = "healthy" + elif healthy_count > 0 or degraded_count > 0: + status = "degraded" + else: + status = "unhealthy" + + report = { + "status": status, + "checked_at": iso_now(), + "sources": { + "youtube": youtube_report, + "website": website_report, + }, + } + print(json.dumps(report, indent=2, ensure_ascii=False)) + if status == "healthy": + return 0 + if status == "degraded": + return 2 + return 1 + + +def run(args: argparse.Namespace) -> int: + output_path = Path(args.output) + with acquire_lane_lock(output_path): + return _run_locked(args, output_path) + + +def _run_locked(args: argparse.Namespace, output_path: Path) -> int: + if not args.skip_refresh: + run_child([sys.executable, str(BACKEND_ROOT / "website_research_runner.py")], allow_failure=False) + run_child([sys.executable, str(BACKEND_ROOT / "youtube_research_runner.py")], allow_failure=False) + + youtube_lane = load_json(Path(args.youtube_output)) + website_lane = load_json(Path(args.website_output)) + + coverage = { + "youtube_video_count": int(youtube_lane.get("total_video_count") or 0), + "youtube_transcribed_video_count": int(youtube_lane.get("transcribed_video_count") or 0), + "youtube_failed_video_count": int(youtube_lane.get("failed_video_count") or 0), + "website_target_count": int(website_lane.get("target_count") or 0), + "website_page_count": int(website_lane.get("page_count") or len(website_lane.get("pages") or [])), + "website_discovered_page_count": int(website_lane.get("discovered_page_count") or 0), + "website_ingested_page_count": int(website_lane.get("ingested_page_count") or 0), + "website_failed_page_count": int(website_lane.get("failed_page_count") or 0), + } + + lane = { + "source_name": "Unified implementation research lane", + "generated_at": iso_now(), + "status": compute_status(coverage), + "importance_legend": website_page_importance_legend(), + "weighted_score_description": weighted_score_description(), + "coverage": coverage, + "recommendations": merge_recommendations(youtube_lane, website_lane), + } + + lane = preserve_last_good_implementation_lane(output_path, lane) + if str(lane.get("status") or "").strip().lower() == "healthy": + save_last_good_lane(output_path, lane) + write_json_file(output_path, lane) + print(f"[implementation-lane] wrote {output_path} with {len(lane['recommendations'])} merged recommendations", file=sys.stderr) + return 0 + + +def main() -> int: + args = parse_args() + if args.mode == "doctor": + return run_doctor(args) + return run(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/apps/exposure-studio/backend/lane_shared.py b/apps/exposure-studio/backend/lane_shared.py new file mode 100644 index 0000000..199371f --- /dev/null +++ b/apps/exposure-studio/backend/lane_shared.py @@ -0,0 +1,442 @@ +#!/usr/bin/env python3 +"""Shared helpers and signal definitions for SurfaceScope research lanes. + +This module is intentionally free of heavy dependencies (faster-whisper, +yt-dlp) so that the website lane and the unified implementation lane can +import it without pulling in the full YouTube transcription stack. +""" + +from __future__ import annotations + +import contextlib +import json +import os +import re +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + + +@dataclass(frozen=True) +class SignalDefinition: + id: str + title: str + priority: str + rationale: str + note: str + suggested_paths: tuple[str, ...] + keywords: tuple[str, ...] + + +SIGNAL_DEFINITIONS: tuple[SignalDefinition, ...] = ( + SignalDefinition( + id="durable_ingestion_jobs", + title="Persist long-running ingestion jobs", + priority="high", + rationale="Research evidence points to long-running agents, retries, queues, or resumability.", + note="Add checkpointed, resumable import runs instead of single-shot fetch scripts.", + suggested_paths=( + "apps/exposure-studio/backend/lane_shared.py", + "apps/exposure-studio/run.sh", + "apps/exposure-studio/src/main.rs", + ), + keywords=("long running", "langlebig", "resume", "resum", "retry", "checkpoint", "queue", "worker", "planner", "scheduled task", "state"), + ), + SignalDefinition( + id="evals_and_regressions", + title="Add research-backed evals and regressions", + priority="high", + rationale="Research evidence references evals, benchmarks, graders, or test harnesses.", + note="Build deterministic tests for research lanes so parser changes do not silently degrade output quality.", + suggested_paths=( + "apps/exposure-studio/src/main.rs", + "apps/exposure-studio/data/testing_methodologies.json", + ), + keywords=("eval", "evaluation", "benchmark", "deepeval", "grader", "regression", "accuracy", "judge", "test harness"), + ), + SignalDefinition( + id="structured_outputs", + title="Normalize lane output into strict JSON contracts", + priority="high", + rationale="Research evidence references schemas, extraction, parsing, or structured outputs.", + note="Keep each stage machine-readable so downstream product surfaces can trust the lane output.", + suggested_paths=( + "apps/exposure-studio/src/models.rs", + "apps/exposure-studio/data/.generated/implementation_lane.json", + ), + keywords=("structured output", "structured outputs", "schema", "json", "parser", "extract", "classification", "normalize"), + ), + SignalDefinition( + id="human_approval_loops", + title="Keep approval gates around high-risk actions", + priority="medium", + rationale="Research evidence references review, approval, or human-in-the-loop controls.", + note="Route externalized decisions through reviewer approval rather than letting automation publish or escalate by itself.", + suggested_paths=( + "apps/exposure-studio/src/main.rs", + "apps/exposure-studio/data/findings.json", + ), + keywords=("human in the loop", "approval", "review", "confirm", "owner confirmation", "human approval", "human review"), + ), + SignalDefinition( + id="retrieval_memory", + title="Build a searchable research memory", + priority="medium", + rationale="Research evidence references RAG, memory, retrieval, or context management.", + note="Persist research-derived notes so later product features can reuse collected evidence without reprocessing raw sources.", + suggested_paths=( + "apps/exposure-studio/data/.generated/implementation_lane.json", + "apps/exposure-studio/src/main.rs", + ), + keywords=("rag", "retrieval", "knowledge base", "vector", "supabase", "docling", "memory", "context", "notes"), + ), + SignalDefinition( + id="observability_provenance", + title="Track lane observability and evidence provenance", + priority="medium", + rationale="Research evidence references tracing, observability, monitoring, or audit trails.", + note="Store generation timestamps, transcript status, and evidence snippets so operators can audit what the lane actually learned.", + suggested_paths=( + "apps/exposure-studio/src/main.rs", + "apps/exposure-studio/backend/lane_shared.py", + ), + keywords=("langsmith", "trace", "observability", "monitor", "telemetry", "audit trail", "logging"), + ), + SignalDefinition( + id="multi_stage_decomposition", + title="Decompose the lane into small, reviewable stages", + priority="medium", + rationale="Research evidence references sub-agents, specialization, or parallel decomposition.", + note="Keep fetch, transcribe, enrich, and aggregate as separate stages with explicit handoffs.", + suggested_paths=( + "apps/exposure-studio/backend/lane_shared.py", + "apps/exposure-studio/src/models.rs", + ), + keywords=("sub-agent", "subagent", "multi-agent", "specialized", "parallel", "handoff", "decompose", "decomposition"), + ), + SignalDefinition( + id="workflow_adapters", + title="Wrap external feeds behind adapter-style importers", + priority="low", + rationale="Research evidence references workflow tools, triggers, or automation adapters.", + note="Keep source-specific acquisition code behind a narrow importer interface so the product can add more channels later.", + suggested_paths=( + "apps/exposure-studio/backend/lane_shared.py", + "apps/exposure-studio/run.sh", + ), + keywords=("workflow", "automation", "n8n", "make.com", "integration", "trigger", "webhook"), + ), +) + + +# --------------------------------------------------------------------------- +# Page-category exclusion for signal extraction +# --------------------------------------------------------------------------- + +_LOW_SIGNAL_PATH_SEGMENTS = frozenset(( + "privacy-policy", + "privacy", + "cookie-policy", + "cookie", + "terms-of-service", + "terms-and-conditions", + "terms", + "legal", + "disclaimer", + "imprint", + "impressum", + "contact", + "about-us", + "gdpr", + "dmca", + "accessibility", + "sitemap", +)) + + +def is_low_signal_url(url: str) -> bool: + """Return True if *url* points to a page category that should not + contribute to implementation signal extraction (legal, policy, contact, + etc.).""" + if not url: + return False + parsed = urlparse(url) + path_lower = parsed.path.strip("/").lower() + if not path_lower: + return False + segments = path_lower.split("/") + return any(segment in _LOW_SIGNAL_PATH_SEGMENTS for segment in segments) + + +def website_page_importance(category: str) -> float: + normalized = str(category or "").strip().lower() + if normalized == "article": + return 1.0 + if normalized == "landing_page": + return 0.8 + if normalized == "discovered_internal_link": + return 0.6 + if normalized == "blog_index": + return 0.5 + if normalized == "category_archive": + return 0.4 + return 0.5 + + +def website_page_importance_legend() -> dict[str, float]: + return { + "article": 1.0, + "landing_page": 0.8, + "discovered_internal_link": 0.6, + "blog_index": 0.5, + "category_archive": 0.4, + "default": 0.5, + } + + +def weighted_score_description() -> str: + return "Weighted recommendation score favors richer source pages over aggregate or navigational ones." + + +# --------------------------------------------------------------------------- +# Shared text helpers +# --------------------------------------------------------------------------- + +def iso_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def env_or_default(name: str, fallback: str) -> str: + value = os.environ.get(name, "").strip() + return value or fallback + + +def sanitize_text(value: str) -> str: + text = re.sub(r"\s+", " ", value or "").strip() + return text + + +def split_sentences(text: str) -> list[str]: + normalized = sanitize_text(text) + if not normalized: + return [] + sentences = re.split(r"(?<=[.!?])\s+", normalized) + return [sentence.strip() for sentence in sentences if sentence.strip()] + + +def extract_signals( + text: str, + *, + page_url: str = "", +) -> tuple[list[str], list[str], list[str]]: + """Extract implementation signals from *text*. + + If *page_url* resolves to a low-signal page category (privacy policy, + terms, contact, etc.), the function short-circuits and returns empty + results so that legal/policy boilerplate does not inflate signal counts. + """ + if page_url and is_low_signal_url(page_url): + return [], [], [] + + sentences = split_sentences(text) + lowered_sentences = [sentence.lower() for sentence in sentences] + signal_ids: list[str] = [] + notes: list[str] = [] + evidence: list[str] = [] + + for definition in SIGNAL_DEFINITIONS: + matches: list[str] = [] + for sentence, lowered in zip(sentences, lowered_sentences): + if any(keyword in lowered for keyword in definition.keywords): + matches.append(sentence) + if not matches: + continue + signal_ids.append(definition.id) + notes.append(definition.note) + for snippet in matches[:2]: + if snippet not in evidence: + evidence.append(snippet) + + return signal_ids, notes, evidence[:6] + + +_SOURCE_SUGGESTED_PATH_OVERRIDES: dict[str, dict[str, tuple[str, ...]]] = { + "youtube": { + "durable_ingestion_jobs": ( + "apps/exposure-studio/backend/youtube_research_runner.py", + "apps/exposure-studio/.lane-secrets.env", + "apps/exposure-studio/run.sh", + ), + "structured_outputs": ( + "apps/exposure-studio/src/models.rs", + "apps/exposure-studio/data/.generated/youtube_implementation_lane.json", + ), + "retrieval_memory": ( + "apps/exposure-studio/data/.generated/youtube_implementation_lane.json", + "apps/exposure-studio/src/main.rs", + ), + "observability_provenance": ( + "apps/exposure-studio/backend/youtube_research_runner.py", + "apps/exposure-studio/data/.generated/youtube-cache", + ), + "multi_stage_decomposition": ( + "apps/exposure-studio/backend/youtube_research_runner.py", + "apps/exposure-studio/src/models.rs", + ), + "workflow_adapters": ( + "apps/exposure-studio/backend/youtube_research_runner.py", + "apps/exposure-studio/run.sh", + ), + }, + "website": { + "durable_ingestion_jobs": ( + "apps/exposure-studio/backend/website_research_runner.py", + "apps/exposure-studio/data/research_web_targets.json", + "apps/exposure-studio/run.sh", + ), + "structured_outputs": ( + "apps/exposure-studio/src/models.rs", + "apps/exposure-studio/data/.generated/website_implementation_lane.json", + ), + "retrieval_memory": ( + "apps/exposure-studio/data/.generated/website_implementation_lane.json", + "apps/exposure-studio/src/main.rs", + ), + "observability_provenance": ( + "apps/exposure-studio/backend/website_research_runner.py", + "apps/exposure-studio/data/.generated/website_implementation_lane.json", + ), + "multi_stage_decomposition": ( + "apps/exposure-studio/backend/website_research_runner.py", + "apps/exposure-studio/src/models.rs", + ), + "workflow_adapters": ( + "apps/exposure-studio/backend/website_research_runner.py", + "apps/exposure-studio/data/research_web_targets.json", + ), + }, + "implementation": { + "durable_ingestion_jobs": ( + "apps/exposure-studio/backend/implementation_lane_runner.py", + "apps/exposure-studio/run.sh", + "apps/exposure-studio/data/.generated/implementation_lane.json", + ), + "structured_outputs": ( + "apps/exposure-studio/src/models.rs", + "apps/exposure-studio/data/.generated/implementation_lane.json", + ), + "retrieval_memory": ( + "apps/exposure-studio/data/.generated/implementation_lane.json", + "apps/exposure-studio/src/main.rs", + ), + "observability_provenance": ( + "apps/exposure-studio/backend/implementation_lane_runner.py", + "apps/exposure-studio/data/.generated/.lane-snapshots", + ), + "multi_stage_decomposition": ( + "apps/exposure-studio/backend/implementation_lane_runner.py", + "apps/exposure-studio/src/models.rs", + ), + "workflow_adapters": ( + "apps/exposure-studio/backend/implementation_lane_runner.py", + "apps/exposure-studio/run.sh", + ), + }, +} + + +def resolve_suggested_paths(definition: SignalDefinition, source_type: str) -> list[str]: + source_key = str(source_type or "").strip().lower() + override = _SOURCE_SUGGESTED_PATH_OVERRIDES.get(source_key, {}).get(definition.id) + if override: + return list(override) + return list(definition.suggested_paths) + + +def read_json_file(path: Path) -> dict[str, Any] | None: + if not path.exists(): + return None + loaded = json.loads(path.read_text(encoding="utf-8")) + return loaded if isinstance(loaded, dict) else None + + +def write_json_file(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + +def lane_snapshot_path(output_path: Path) -> Path: + snapshot_dir = output_path.parent / ".lane-snapshots" + return snapshot_dir / f"{output_path.stem}.last-good.json" + + +def load_last_good_lane(output_path: Path) -> dict[str, Any] | None: + return read_json_file(lane_snapshot_path(output_path)) + + +def save_last_good_lane(output_path: Path, payload: dict[str, Any]) -> None: + write_json_file(lane_snapshot_path(output_path), payload) + + +def lane_lock_path(output_path: Path) -> Path: + lock_dir = output_path.parent / ".lane-locks" + return lock_dir / f"{output_path.stem}.lock" + + +def lock_owner_is_active(lock_path: Path) -> bool: + metadata = read_json_file(lock_path) + if not isinstance(metadata, dict): + return False + try: + pid = int(metadata.get("pid") or 0) + except (TypeError, ValueError): + return False + if pid <= 0: + return False + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +@contextlib.contextmanager +def acquire_lane_lock(output_path: Path, *, timeout_seconds: float = 10.0, poll_seconds: float = 0.25): + lock_path = lane_lock_path(output_path) + lock_path.parent.mkdir(parents=True, exist_ok=True) + started_at = time.monotonic() + fd: int | None = None + + while fd is None: + try: + fd = os.open(str(lock_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY) + payload = { + "pid": os.getpid(), + "acquired_at": iso_now(), + "target": str(output_path), + } + os.write(fd, json.dumps(payload, ensure_ascii=False).encode("utf-8")) + os.close(fd) + fd = 0 + except FileExistsError: + if not lock_owner_is_active(lock_path): + lock_path.unlink(missing_ok=True) + continue + if timeout_seconds <= 0 or (time.monotonic() - started_at) >= timeout_seconds: + raise RuntimeError( + f"Lane lock is already held for {output_path.name} ({lock_path})." + ) + time.sleep(max(0.01, poll_seconds)) + + try: + yield lock_path + finally: + try: + lock_path.unlink(missing_ok=True) + except Exception: + pass diff --git a/apps/exposure-studio/backend/requirements-youtube-research.txt b/apps/exposure-studio/backend/requirements-youtube-research.txt new file mode 100644 index 0000000..781819c --- /dev/null +++ b/apps/exposure-studio/backend/requirements-youtube-research.txt @@ -0,0 +1,3 @@ +faster-whisper>=1.1.1,<2 +yt-dlp>=2025.3.31 +youtube-transcript-api>=1.2.3,<2 diff --git a/apps/exposure-studio/backend/website_research_runner.py b/apps/exposure-studio/backend/website_research_runner.py new file mode 100644 index 0000000..7706b90 --- /dev/null +++ b/apps/exposure-studio/backend/website_research_runner.py @@ -0,0 +1,464 @@ +#!/usr/bin/env python3 +"""Website text research runner for SurfaceScope.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +from collections import deque +import urllib.request +from html import unescape +from pathlib import Path +from typing import Any +from urllib.parse import urldefrag, urljoin, urlparse + +from lane_shared import ( + acquire_lane_lock, + SIGNAL_DEFINITIONS, + env_or_default, + extract_signals, + iso_now, + load_last_good_lane, + resolve_suggested_paths, + save_last_good_lane, + sanitize_text, + website_page_importance, + website_page_importance_legend, + weighted_score_description, + write_json_file, +) + + +APP_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_TARGETS_FILE = APP_ROOT / "data" / "research_web_targets.json" +DEFAULT_OUTPUT = APP_ROOT / "data" / ".generated" / "website_implementation_lane.json" +DEFAULT_TIMEOUT_SECONDS = 15.0 +DEFAULT_CRAWL_DEPTH = 1 +DEFAULT_DISCOVERED_LINK_LIMIT = 10 + +NON_HTML_SUFFIXES = ( + ".jpg", + ".jpeg", + ".png", + ".gif", + ".webp", + ".svg", + ".ico", + ".pdf", + ".zip", + ".gz", + ".tar", + ".mp3", + ".mp4", + ".mov", + ".avi", + ".webm", + ".css", + ".js", + ".json", + ".xml", + ".txt", +) + +EXCLUDED_PATH_SEGMENTS = ( + "/feed", + "/wp-json", + "/xmlrpc", + "/wp-admin", + "/wp-login", +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build a website text implementation lane.") + parser.add_argument( + "mode", + nargs="?", + default="run", + choices=("run", "doctor"), + help="Use 'doctor' to run preflight diagnostics without building the lane.", + ) + parser.add_argument("--targets-file", default=env_or_default("WEBSITE_LANE_TARGETS_FILE", str(DEFAULT_TARGETS_FILE)), help="JSON file describing website targets.") + parser.add_argument("--output", default=env_or_default("WEBSITE_LANE_OUTPUT", str(DEFAULT_OUTPUT)), help="Lane JSON output path.") + parser.add_argument("--timeout-seconds", type=float, default=float(env_or_default("WEBSITE_LANE_TIMEOUT_SECONDS", str(DEFAULT_TIMEOUT_SECONDS))), help="HTTP timeout per page.") + parser.add_argument("--max-pages", type=int, default=int(env_or_default("WEBSITE_LANE_MAX_PAGES", "0")), help="Optional cap for processed pages.") + parser.add_argument("--crawl-depth", type=int, default=int(env_or_default("WEBSITE_LANE_CRAWL_DEPTH", str(DEFAULT_CRAWL_DEPTH))), help="How many same-origin link levels to follow from seed pages.") + parser.add_argument("--discovered-links-per-page", type=int, default=int(env_or_default("WEBSITE_LANE_DISCOVERED_LINKS_PER_PAGE", str(DEFAULT_DISCOVERED_LINK_LIMIT))), help="Cap discovered same-origin links per fetched page.") + return parser.parse_args() + + +def load_targets(path: Path) -> list[dict[str, Any]]: + raw = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(raw, list): + raise ValueError(f"{path} must contain a JSON list") + return [item for item in raw if isinstance(item, dict) and str(item.get("url") or "").startswith("https://")] + + +def strip_html_to_text(body: str) -> str: + cleaned = re.sub(r"(?is)", " ", body) + cleaned = re.sub(r"(?is)", " ", cleaned) + cleaned = re.sub(r"(?is)", " ", cleaned) + cleaned = re.sub(r"(?is)", " ", cleaned) + cleaned = re.sub(r"(?is)<[^>]+>", " ", cleaned) + return sanitize_text(unescape(cleaned)) + + +def extract_title(body: str) -> str: + match = re.search(r"(?is)(.*?)", body) + return sanitize_text(unescape(match.group(1))) if match else "" + + +def fetch_page(url: str, timeout_seconds: float) -> tuple[dict[str, Any], str]: + request = urllib.request.Request(url, headers={"User-Agent": "SurfaceScope/0.1 website-research-lane"}) + with urllib.request.urlopen(request, timeout=timeout_seconds) as response: + body = response.read().decode("utf-8", "replace") + metadata = { + "final_url": response.geturl(), + "http_status": getattr(response, "status", 200), + "content_type": response.headers.get("content-type", ""), + } + return metadata, body + + +def normalize_candidate_url(raw_url: str, base_url: str, allowed_hosts: set[str]) -> str | None: + if not raw_url: + return None + joined = urljoin(base_url, raw_url.strip()) + without_fragment, _ = urldefrag(joined) + parsed = urlparse(without_fragment) + if parsed.scheme not in {"http", "https"}: + return None + if not parsed.netloc or parsed.netloc not in allowed_hosts: + return None + lowered_path = parsed.path.lower() + if lowered_path.endswith(NON_HTML_SUFFIXES): + return None + stripped_segments = {segment.strip("/") for segment in EXCLUDED_PATH_SEGMENTS} + path_segments = [segment for segment in lowered_path.strip("/").split("/") if segment] + if any(segment in stripped_segments for segment in path_segments): + return None + if any(lowered_path == segment or lowered_path.startswith(segment) for segment in EXCLUDED_PATH_SEGMENTS): + return None + normalized = parsed._replace(query="", fragment="") + return normalized.geturl() + + +def extract_internal_links(body: str, base_url: str, allowed_hosts: set[str], limit: int) -> list[str]: + seen: list[str] = [] + for match in re.finditer(r"""href\s*=\s*["']([^"']+)["']""", body, flags=re.IGNORECASE): + normalized = normalize_candidate_url(match.group(1), base_url, allowed_hosts) + if not normalized or normalized in seen: + continue + seen.append(normalized) + if limit > 0 and len(seen) >= limit: + break + return seen + + +def build_page_id(url: str, fallback: str) -> str: + parsed = urlparse(url) + slug = re.sub(r"[^a-z0-9]+", "-", f"{parsed.netloc}{parsed.path}".lower()).strip("-") + if not slug: + return fallback + return f"page-{slug[:72]}" + + +def build_recommendations(pages: list[dict[str, Any]]) -> list[dict[str, Any]]: + definitions = {definition.id: definition for definition in SIGNAL_DEFINITIONS} + grouped: dict[str, list[str]] = {} + page_index = { + str(page.get("page_id") or ""): page + for page in pages + if isinstance(page, dict) + } + for page in pages: + for signal_id in page.get("implementation_signals", []): + grouped.setdefault(signal_id, []).append(page["page_id"]) + + recommendations: list[dict[str, Any]] = [] + for signal_id, page_ids in grouped.items(): + definition = definitions.get(signal_id) + if not definition: + continue + suggested_paths = resolve_suggested_paths(definition, "website") + weighted_signal_score = sum( + float(page_index.get(page_id, {}).get("importance_score") or 0.0) + for page_id in page_ids + ) + recommendations.append( + { + "id": definition.id, + "title": definition.title, + "priority": definition.priority, + "rationale": definition.rationale, + "suggested_paths": suggested_paths, + "suggested_paths_by_source": { + "website": suggested_paths, + }, + "supporting_source_types": ["website"], + "supporting_page_ids": page_ids, + "signal_count": len(page_ids), + "weighted_signal_score": weighted_signal_score, + "source_count": 1, + } + ) + + recommendations.sort(key=lambda item: (-item["weighted_signal_score"], -item["signal_count"], item["title"])) + return recommendations + + +def preserve_last_good_website_lane(output_path: Path, current_lane: dict[str, Any]) -> dict[str, Any]: + current_success = int(current_lane.get("ingested_page_count") or 0) + if current_success > 0: + return current_lane + + previous_lane = load_last_good_lane(output_path) + if not previous_lane: + return current_lane + + previous_success = int(previous_lane.get("ingested_page_count") or 0) + if previous_success <= current_success: + return current_lane + + preserved_lane = dict(previous_lane) + preserved_lane["restored_at"] = iso_now() + preserved_lane["restored_reason"] = ( + f"Preserved last good website lane because the latest refresh produced " + f"{current_success} ingested pages and the previous snapshot had {previous_success}." + ) + preserved_lane["last_attempt"] = { + "generated_at": current_lane.get("generated_at"), + "target_count": current_lane.get("target_count"), + "page_count": current_lane.get("page_count"), + "ingested_page_count": current_lane.get("ingested_page_count"), + "failed_page_count": current_lane.get("failed_page_count"), + } + return preserved_lane + + +def run_doctor(args: argparse.Namespace) -> int: + targets_path = Path(args.targets_file) + checks: list[dict[str, Any]] = [] + + def add_check(name: str, ok: bool, detail: str, level: str = "info") -> None: + checks.append({"name": name, "ok": ok, "level": level, "detail": detail}) + + add_check("targets_file", targets_path.exists() and targets_path.is_file(), f"Using targets file {targets_path}", level="error" if not targets_path.exists() else "info") + + targets: list[dict[str, Any]] = [] + if targets_path.exists(): + try: + targets = load_targets(targets_path) + add_check("target_count", len(targets) > 0, f"Loaded {len(targets)} website targets.", level="error" if not targets else "info") + except Exception as error: + add_check("target_count", False, str(error), level="error") + + if targets: + sample = targets[0] + try: + metadata, body = fetch_page(sample["url"], args.timeout_seconds) + body_text = strip_html_to_text(body) + add_check("page_fetch", True, f"{sample['url']} returned {metadata['http_status']} with {len(body_text.split())} words.") + allowed_hosts = {urlparse(str(target.get("url") or "")).netloc for target in targets if str(target.get("url") or "").startswith("https://")} + discovered = extract_internal_links(body, metadata["final_url"], allowed_hosts, args.discovered_links_per_page) + add_check("internal_link_discovery", len(discovered) > 0, f"Discovered {len(discovered)} same-origin links from the sample page.", level="warning" if not discovered else "info") + except Exception as error: + add_check("page_fetch", False, str(error), level="error") + + status = "healthy" + if any(not check["ok"] and check["level"] == "error" for check in checks): + status = "unhealthy" + elif any(not check["ok"] and check["level"] == "warning" for check in checks): + status = "degraded" + + print(json.dumps({"status": status, "checked_at": iso_now(), "targets_file": str(targets_path), "checks": checks}, indent=2, ensure_ascii=False)) + if status == "healthy": + return 0 + if status == "degraded": + return 2 + return 1 + + +def run(args: argparse.Namespace) -> int: + output_path = Path(args.output) + with acquire_lane_lock(output_path): + return _run_locked(args, output_path) + + +def _run_locked(args: argparse.Namespace, output_path: Path) -> int: + targets = load_targets(Path(args.targets_file)) + allowed_hosts = { + urlparse(str(target.get("url") or "").strip()).netloc + for target in targets + if str(target.get("url") or "").startswith("https://") + } + + pages: list[dict[str, Any]] = [] + queue: deque[dict[str, Any]] = deque() + seen_urls: set[str] = set() + + for index, target in enumerate(targets, start=1): + url = str(target.get("url") or "").strip() + if not url or url in seen_urls: + continue + seen_urls.add(url) + queue.append( + { + "page_id": str(target.get("id") or f"seed-{index}"), + "title": sanitize_text(str(target.get("title") or "")), + "url": url, + "category": sanitize_text(str(target.get("category") or "")), + "crawl_depth": 0, + "discovered_from_page_id": "", + "seed_target_id": str(target.get("id") or f"seed-{index}"), + } + ) + + processed = 0 + while queue: + if args.max_pages > 0 and processed >= args.max_pages: + break + + target = queue.popleft() + processed += 1 + page_id = str(target.get("page_id") or build_page_id(str(target.get("url") or ""), f"page-{processed}")) + title = sanitize_text(str(target.get("title") or "")) + url = str(target.get("url") or "").strip() + category = sanitize_text(str(target.get("category") or "")) + crawl_depth = int(target.get("crawl_depth") or 0) + discovered_from_page_id = sanitize_text(str(target.get("discovered_from_page_id") or "")) + seed_target_id = sanitize_text(str(target.get("seed_target_id") or page_id)) + print(f"[website-lane] {processed} fetching {url} (depth {crawl_depth})", file=sys.stderr) + + fetch_status = "failed" + http_status = None + final_url = "" + text_word_count = 0 + implementation_signals: list[str] = [] + implementation_notes: list[str] = [] + evidence_snippets: list[str] = [] + page_excerpt = "" + error_summary = "" + + try: + metadata, body = fetch_page(url, args.timeout_seconds) + final_url = metadata["final_url"] + http_status = metadata["http_status"] + + # Post-redirect host validation: reject body if the final URL + # landed on a host outside the configured allowlist. + final_host = urlparse(final_url).netloc if final_url else "" + if final_host and final_host not in allowed_hosts: + fetch_status = "failed" + error_summary = f"Redirect landed on out-of-scope host {final_host} (from {url})." + pages.append( + { + "page_id": page_id, + "title": title, + "url": url, + "category": category, + "fetch_status": fetch_status, + "final_url": final_url, + "http_status": http_status, + "text_word_count": 0, + "importance_score": website_page_importance(category), + "implementation_signals": [], + "implementation_notes": [], + "evidence_snippets": [], + "page_excerpt": "", + "error_summary": error_summary, + "crawl_depth": crawl_depth, + "discovered_from_page_id": discovered_from_page_id, + "seed_target_id": seed_target_id, + } + ) + continue + + fetch_status = "ok" if metadata["http_status"] == 200 else "reachable_with_error" + extracted_title = extract_title(body) + if extracted_title: + title = extracted_title + body_text = strip_html_to_text(body) + text_word_count = len(body_text.split()) + if body_text: + effective_url = final_url or url + implementation_signals, implementation_notes, evidence_snippets = extract_signals(body_text, page_url=effective_url) + page_excerpt = body_text[:320] + if crawl_depth < max(0, args.crawl_depth): + for discovered_url in extract_internal_links(body, final_url or url, allowed_hosts, args.discovered_links_per_page): + if discovered_url in seen_urls: + continue + seen_urls.add(discovered_url) + queue.append( + { + "page_id": build_page_id(discovered_url, f"page-{len(seen_urls)}"), + "title": "", + "url": discovered_url, + "category": "discovered_internal_link", + "crawl_depth": crawl_depth + 1, + "discovered_from_page_id": page_id, + "seed_target_id": seed_target_id, + } + ) + else: + fetch_status = "failed" + error_summary = "No usable body text was extracted from the page." + except Exception as error: + error_summary = str(error) + + pages.append( + { + "page_id": page_id, + "title": title, + "url": url, + "category": category, + "fetch_status": fetch_status, + "final_url": final_url, + "http_status": http_status, + "text_word_count": text_word_count, + "importance_score": website_page_importance(category), + "implementation_signals": implementation_signals, + "implementation_notes": implementation_notes, + "evidence_snippets": evidence_snippets, + "page_excerpt": page_excerpt, + "error_summary": error_summary, + "crawl_depth": crawl_depth, + "discovered_from_page_id": discovered_from_page_id, + "seed_target_id": seed_target_id, + } + ) + + lane = { + "source_name": "Website text research lane", + "generated_at": iso_now(), + "importance_legend": website_page_importance_legend(), + "weighted_score_description": weighted_score_description(), + "target_count": len(targets), + "page_count": len(pages), + "discovered_page_count": sum(1 for page in pages if page.get("discovered_from_page_id")), + "ingested_page_count": sum(1 for page in pages if page["fetch_status"] == "ok"), + "failed_page_count": sum(1 for page in pages if page["fetch_status"] != "ok"), + "recommendations": build_recommendations(pages), + "pages": pages, + } + + lane = preserve_last_good_website_lane(output_path, lane) + if int(lane.get("ingested_page_count") or 0) > 0: + save_last_good_lane(output_path, lane) + write_json_file(output_path, lane) + print(f"[website-lane] wrote {output_path} with {lane['ingested_page_count']} fetched website documents", file=sys.stderr) + return 0 + + +def main() -> int: + args = parse_args() + if args.mode == "doctor": + return run_doctor(args) + return run(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/apps/exposure-studio/backend/youtube_research_runner.py b/apps/exposure-studio/backend/youtube_research_runner.py new file mode 100644 index 0000000..29e8fd4 --- /dev/null +++ b/apps/exposure-studio/backend/youtube_research_runner.py @@ -0,0 +1,935 @@ +#!/usr/bin/env python3 +"""Transcript-first YouTube research runner for SurfaceScope. + +This lane intentionally derives implementation signals only from speech-to-text +transcripts. Video titles and descriptions are kept as metadata for operator +review, but they are not used as fallback extraction sources. +""" + +from __future__ import annotations + +import argparse +import json +import re +import shutil +import sys +import time +from collections import defaultdict +from importlib.metadata import version as package_version +from importlib.metadata import PackageNotFoundError +from pathlib import Path +from typing import Any + +from lane_shared import ( + acquire_lane_lock, + SIGNAL_DEFINITIONS, + SignalDefinition, + env_or_default, + extract_signals, + iso_now, + load_last_good_lane, + read_json_file, + resolve_suggested_paths, + save_last_good_lane, + sanitize_text, + split_sentences, + write_json_file, +) + +try: + from faster_whisper import WhisperModel +except ImportError as error: # pragma: no cover - import guard + raise SystemExit( + "faster-whisper is required. Install backend/requirements-youtube-research.txt first." + ) from error + +try: + from yt_dlp import YoutubeDL +except ImportError as error: # pragma: no cover - import guard + raise SystemExit( + "yt-dlp is required. Install backend/requirements-youtube-research.txt first." + ) from error + +try: + from youtube_transcript_api import YouTubeTranscriptApi + from youtube_transcript_api.formatters import TextFormatter +except ImportError: # pragma: no cover - optional fallback + YouTubeTranscriptApi = None + TextFormatter = None + + +APP_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_CHANNEL_URL = "https://www.youtube.com/@TheAIAutomators/videos" +DEFAULT_GENERATED_DIR = APP_ROOT / "data" / ".generated" +DEFAULT_OUTPUT = DEFAULT_GENERATED_DIR / "youtube_implementation_lane.json" +DEFAULT_CACHE_DIR = DEFAULT_GENERATED_DIR / "youtube-cache" +DEFAULT_COOKIES_BROWSER = "" +DEFAULT_COOKIES_PROFILE = "" +DEFAULT_COOKIES_KEYRING = "" +DEFAULT_RETRY_ATTEMPTS = 3 +DEFAULT_RETRY_BACKOFF_SECONDS = 5.0 + +BROWSER_COOKIE_CANDIDATES = ( + ("chrome", Path("~/.config/google-chrome").expanduser(), "Cookies"), + ("chromium", Path("~/.config/chromium").expanduser(), "Cookies"), + ("brave", Path("~/.config/BraveSoftware/Brave-Browser").expanduser(), "Cookies"), + ("firefox", Path("~/.mozilla/firefox").expanduser(), "cookies.sqlite"), +) + + + +# iso_now, env_or_default, sanitize_text, split_sentences, extract_signals, +# SIGNAL_DEFINITIONS, and SignalDefinition are imported from lane_shared. + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build a transcript-first YouTube implementation lane.") + parser.add_argument( + "mode", + nargs="?", + default="run", + choices=("run", "doctor"), + help="Use 'doctor' to run preflight diagnostics without building the lane.", + ) + parser.add_argument("--channel-url", default=env_or_default("YOUTUBE_LANE_CHANNEL_URL", DEFAULT_CHANNEL_URL), help="YouTube channel videos URL.") + parser.add_argument("--output", default=env_or_default("YOUTUBE_LANE_OUTPUT", str(DEFAULT_OUTPUT)), help="Lane JSON output path.") + parser.add_argument("--cache-dir", default=env_or_default("YOUTUBE_LANE_CACHE_DIR", str(DEFAULT_CACHE_DIR)), help="Cache directory for transcripts and temporary audio.") + parser.add_argument("--model", default=env_or_default("YOUTUBE_LANE_MODEL", "small"), help="faster-whisper model name (for example: tiny, small, medium).") + parser.add_argument("--device", default=env_or_default("YOUTUBE_LANE_DEVICE", "auto"), help="Whisper device selection.") + parser.add_argument("--compute-type", default=env_or_default("YOUTUBE_LANE_COMPUTE_TYPE", "int8"), help="Whisper compute type.") + parser.add_argument("--max-videos", type=int, default=int(env_or_default("YOUTUBE_LANE_MAX_VIDEOS", "0")), help="Optional cap for video processing.") + parser.add_argument("--sleep-seconds", type=float, default=float(env_or_default("YOUTUBE_LANE_SLEEP_SECONDS", "0")), help="Optional delay between downloads.") + parser.add_argument("--retry-attempts", type=int, default=int(env_or_default("YOUTUBE_LANE_RETRY_ATTEMPTS", str(DEFAULT_RETRY_ATTEMPTS))), help="Retry attempts for transient YouTube download failures.") + parser.add_argument("--retry-backoff-seconds", type=float, default=float(env_or_default("YOUTUBE_LANE_RETRY_BACKOFF_SECONDS", str(DEFAULT_RETRY_BACKOFF_SECONDS))), help="Base backoff in seconds between YouTube retries.") + parser.add_argument("--keep-audio", action="store_true", help="Keep downloaded audio files in cache.") + parser.add_argument("--cookies-file", default=env_or_default("YOUTUBE_LANE_COOKIES_FILE", ""), help="Optional Netscape cookies file for yt-dlp.") + parser.add_argument("--cookies-browser", default=env_or_default("YOUTUBE_LANE_COOKIES_BROWSER", DEFAULT_COOKIES_BROWSER), help="Optional browser name for yt-dlp cookies-from-browser.") + parser.add_argument("--cookies-browser-profile", default=env_or_default("YOUTUBE_LANE_COOKIES_PROFILE", DEFAULT_COOKIES_PROFILE), help="Optional browser profile path/name for cookies-from-browser.") + parser.add_argument("--cookies-browser-keyring", default=env_or_default("YOUTUBE_LANE_COOKIES_KEYRING", DEFAULT_COOKIES_KEYRING), help="Optional keyring selector for cookies-from-browser.") + parser.add_argument("--proxy", default=env_or_default("YOUTUBE_LANE_PROXY", ""), help="Optional outbound proxy URL for yt-dlp.") + return parser.parse_args() + + +def ensure_dir(path: Path) -> None: + path.mkdir(parents=True, exist_ok=True) + + +def load_json(path: Path) -> Any | None: + return read_json_file(path) + + +def write_json(path: Path, payload: Any) -> None: + write_json_file(path, payload) + + +def build_js_runtime_config() -> dict[str, dict[str, str]]: + if shutil.which("node"): + return {"node": {}} + return {} + + +def find_browser_cookie_source() -> dict[str, str]: + for browser_name, root_dir, cookie_filename in BROWSER_COOKIE_CANDIDATES: + if not root_dir.exists(): + continue + for cookie_path in root_dir.rglob(cookie_filename): + return { + "mode": "browser", + "browser": browser_name, + "profile": str(cookie_path.parent), + "keyring": "", + } + return {"mode": "none", "browser": "", "profile": "", "keyring": ""} + + +def resolve_cookie_source(args: argparse.Namespace) -> dict[str, str]: + cookies_file = args.cookies_file.strip() + if cookies_file: + return { + "mode": "file", + "browser": "", + "profile": cookies_file, + "keyring": "", + } + + cookies_browser = args.cookies_browser.strip() + cookies_profile = args.cookies_browser_profile.strip() + cookies_keyring = args.cookies_browser_keyring.strip() + if cookies_browser: + return { + "mode": "browser", + "browser": cookies_browser, + "profile": cookies_profile, + "keyring": cookies_keyring, + } + + return find_browser_cookie_source() + + +def build_ydl_options(base: dict[str, Any], args: argparse.Namespace) -> dict[str, Any]: + options = dict(base) + cookie_source = resolve_cookie_source(args) + if cookie_source["mode"] == "file": + options["cookiefile"] = cookie_source["profile"] + elif cookie_source["mode"] == "browser": + options["cookiesfrombrowser"] = ( + cookie_source["browser"], + cookie_source["profile"] or None, + cookie_source["keyring"] or None, + None, + ) + + if args.proxy: + options["proxy"] = args.proxy + + js_runtimes = build_js_runtime_config() + if js_runtimes: + options["js_runtimes"] = js_runtimes + + return options + + +def print_runtime_summary(args: argparse.Namespace) -> None: + cookie_source = resolve_cookie_source(args) + runtime = { + "mode": args.mode, + "channel_url": args.channel_url, + "output": str(Path(args.output)), + "cache_dir": str(Path(args.cache_dir)), + "model": args.model, + "device": args.device, + "compute_type": args.compute_type, + "max_videos": args.max_videos, + "retry_attempts": args.retry_attempts, + "retry_backoff_seconds": args.retry_backoff_seconds, + "cookies_file_configured": bool(args.cookies_file), + "cookies_browser_configured": bool(args.cookies_browser), + "cookies_browser_profile_configured": bool(args.cookies_browser_profile), + "cookie_source_mode": cookie_source["mode"], + "cookie_source_browser": cookie_source["browser"], + "proxy_configured": bool(args.proxy), + "js_runtime_node_available": bool(shutil.which("node")), + } + print(f"[youtube-lane] runtime {json.dumps(runtime, ensure_ascii=False)}", file=sys.stderr) + + +def installed_package_version(name: str) -> str: + try: + return package_version(name) + except PackageNotFoundError: + return "" + + + +# sanitize_text, split_sentences, and extract_signals are imported from lane_shared. + + +def format_duration(seconds: int | None) -> str: + if not seconds: + return "" + hours, remainder = divmod(int(seconds), 3600) + minutes, secs = divmod(remainder, 60) + if hours: + return f"{hours}:{minutes:02d}:{secs:02d}" + return f"{minutes}:{secs:02d}" + + +def to_iso_date(upload_date: str | None) -> str: + if not upload_date or len(upload_date) != 8 or not upload_date.isdigit(): + return "" + return f"{upload_date[0:4]}-{upload_date[4:6]}-{upload_date[6:8]}" + + +def load_cached_transcript(cache_path: Path, model_name: str) -> dict[str, Any] | None: + cached = load_json(cache_path) + if not isinstance(cached, dict) or cached.get("transcript_status") != "complete": + return None + source = str(cached.get("transcript_source") or "") + if source == "youtube_auto_captions": + return cached + if cached.get("model_name") == model_name: + return cached + return None + + +def load_any_complete_transcript(cache_path: Path) -> dict[str, Any] | None: + cached = load_json(cache_path) + if not isinstance(cached, dict): + return None + if cached.get("transcript_status") != "complete": + return None + return cached + + +def is_retryable_youtube_error(error: Exception | str) -> bool: + message = str(error or "").lower() + retry_markers = ( + "http error 403", + "unable to download video data", + "sign in to confirm you're not a bot", + "sign in to confirm you’re not a bot", + "timed out", + "timeout", + "connection reset", + "temporarily unavailable", + "remote end closed connection", + "proxy", + ) + return any(marker in message for marker in retry_markers) + + +def run_with_retry(task_label: str, operation, *, attempts: int, backoff_seconds: float): + last_error: Exception | None = None + total_attempts = max(1, attempts) + total_backoff = 0.0 + for attempt in range(1, total_attempts + 1): + try: + return operation(), { + "attempt_count": attempt, + "backoff_seconds": total_backoff, + "recovered_after_retry": attempt > 1, + } + except Exception as error: # pragma: no cover - network-dependent path + last_error = error + if attempt >= total_attempts or not is_retryable_youtube_error(error): + raise + delay = max(0.0, backoff_seconds) * (2 ** (attempt - 1)) + total_backoff += delay + print( + f"[youtube-lane] retrying {task_label} after attempt {attempt}/{total_attempts}: {error}", + file=sys.stderr, + ) + if delay > 0: + time.sleep(delay) + if last_error: + raise last_error + raise RuntimeError(f"{task_label} failed without an exception") + + +def list_channel_entries(channel_url: str, args: argparse.Namespace) -> tuple[str, str, list[dict[str, Any]]]: + options = build_ydl_options({ + "quiet": True, + "skip_download": True, + "extract_flat": "in_playlist", + "playlistend": args.max_videos or None, + "ignoreerrors": True, + "noplaylist": False, + }, args) + with YoutubeDL(options) as ydl: + payload = ydl.extract_info(channel_url, download=False) + + entries = [ + entry + for entry in (payload.get("entries") or []) + if isinstance(entry, dict) and entry.get("id") + ] + channel_title = payload.get("title") or "Unknown channel" + channel_id = payload.get("channel_id") or payload.get("id") or "" + return channel_title, channel_id, entries + + +def build_processing_plan(entries: list[dict[str, Any]], transcript_dir: Path) -> list[dict[str, Any]]: + seen_video_ids: set[str] = set() + cached_entries: list[dict[str, Any]] = [] + uncached_entries: list[dict[str, Any]] = [] + + for original_index, entry in enumerate(entries): + video_id = str(entry.get("id") or "").strip() + if not video_id or video_id in seen_video_ids: + continue + seen_video_ids.add(video_id) + + plan_item = { + "entry": entry, + "original_index": original_index, + "video_id": video_id, + "duration_seconds": int(entry.get("duration")) if isinstance(entry.get("duration"), (int, float)) else None, + } + transcript_cache = transcript_dir / f"{video_id}.json" + if load_any_complete_transcript(transcript_cache): + cached_entries.append(plan_item) + else: + uncached_entries.append(plan_item) + + uncached_entries.sort( + key=lambda item: ( + item["duration_seconds"] is None, + item["duration_seconds"] or 0, + item["original_index"], + ) + ) + return cached_entries + uncached_entries + + +def probe_audio_access(video_url: str, args: argparse.Namespace) -> tuple[bool, str]: + options = build_ydl_options({ + "quiet": True, + "no_warnings": True, + "skip_download": True, + "noplaylist": True, + "format": "bestaudio/best", + "ignoreerrors": False, + }, args) + try: + with YoutubeDL(options) as ydl: + info = ydl.extract_info(video_url, download=False) + title = sanitize_text(str((info or {}).get("title") or "")) + return True, title or "audio metadata accessible" + except Exception as error: # pragma: no cover - network-dependent + return False, str(error) + + +def find_cached_audio(audio_dir: Path, video_id: str) -> Path | None: + matches = sorted(audio_dir.glob(f"{video_id}.*")) + return matches[0] if matches else None + + +def download_audio(video_url: str, video_id: str, audio_dir: Path, args: argparse.Namespace) -> tuple[Path, dict[str, Any]]: + ensure_dir(audio_dir) + cached = find_cached_audio(audio_dir, video_id) + if cached: + return cached, {} + + options = build_ydl_options({ + "quiet": True, + "no_warnings": True, + "noplaylist": True, + "format": "bestaudio/best", + "outtmpl": str(audio_dir / f"{video_id}.%(ext)s"), + "restrictfilenames": True, + "ignoreerrors": False, + }, args) + with YoutubeDL(options) as ydl: + info = ydl.extract_info(video_url, download=True) + + requested_downloads = info.get("requested_downloads") or [] + file_path = None + if requested_downloads: + file_path = requested_downloads[0].get("filepath") + if not file_path: + file_path = ydl.prepare_filename(info) + return Path(file_path), info + + +def transcribe_audio( + model: WhisperModel, + audio_path: Path, + cache_path: Path, + model_name: str, + acquisition_meta: dict[str, Any] | None = None, +) -> dict[str, Any]: + cached = load_cached_transcript(cache_path, model_name) + if cached: + return cached + + segments, info = model.transcribe( + str(audio_path), + beam_size=1, + vad_filter=True, + condition_on_previous_text=False, + ) + transcript_parts: list[str] = [] + for segment in segments: + text = sanitize_text(segment.text) + if text: + transcript_parts.append(text) + + transcript_text = sanitize_text(" ".join(transcript_parts)) + payload = { + "model_name": model_name, + "transcript_source": "faster_whisper", + "transcript_status": "complete" if transcript_text else "failed", + "transcript_language": getattr(info, "language", "") or "", + "transcript_text": transcript_text, + "acquisition_retry_attempt_count": int((acquisition_meta or {}).get("attempt_count") or 1), + "acquisition_retry_backoff_seconds": float((acquisition_meta or {}).get("backoff_seconds") or 0.0), + "acquisition_retry_recovered": bool((acquisition_meta or {}).get("recovered_after_retry")), + "generated_at": iso_now(), + } + write_json(cache_path, payload) + return payload + + +def fetch_auto_captions(video_id: str, cache_path: Path) -> dict[str, Any]: + cached = load_cached_transcript(cache_path, "") + if cached and cached.get("transcript_source") == "youtube_auto_captions": + return cached + if YouTubeTranscriptApi is None or TextFormatter is None: + raise RuntimeError("youtube-transcript-api is not installed.") + + transcript = YouTubeTranscriptApi().fetch(video_id, languages=["en", "de"]) + transcript_text = sanitize_text(TextFormatter().format_transcript(transcript)) + payload = { + "model_name": "", + "transcript_source": "youtube_auto_captions", + "transcript_status": "complete" if transcript_text else "failed", + "transcript_language": getattr(transcript, "language_code", "") or "", + "transcript_text": transcript_text, + "acquisition_retry_attempt_count": 0, + "acquisition_retry_backoff_seconds": 0.0, + "acquisition_retry_recovered": False, + "generated_at": iso_now(), + } + write_json(cache_path, payload) + return payload + + +def probe_auto_captions(video_id: str) -> tuple[bool, str]: + if YouTubeTranscriptApi is None or TextFormatter is None: + return False, "youtube-transcript-api is not installed." + try: + transcript = YouTubeTranscriptApi().fetch(video_id, languages=["en", "de"]) + transcript_text = sanitize_text(TextFormatter().format_transcript(transcript)) + if not transcript_text: + return False, "Transcript request returned an empty payload." + return True, f"{len(transcript_text.split())} words" + except Exception as error: # pragma: no cover - network-dependent + return False, str(error) + + +def empty_lane(channel_url: str) -> dict[str, Any]: + return { + "source_name": "YouTube transcript research lane", + "channel_title": "The AI Automators", + "channel_id": "", + "channel_url": channel_url, + "generated_at": "", + "model_name": "", + "total_video_count": 0, + "transcribed_video_count": 0, + "failed_video_count": 0, + "recommendations": [], + "videos": [], + } + + +def build_recommendations(videos: list[dict[str, Any]]) -> list[dict[str, Any]]: + supporting_ids: dict[str, list[str]] = defaultdict(list) + for video in videos: + for signal_id in video["implementation_signals"]: + supporting_ids[signal_id].append(video["video_id"]) + + recommendations: list[dict[str, Any]] = [] + for definition in SIGNAL_DEFINITIONS: + ids = supporting_ids.get(definition.id, []) + if not ids: + continue + count = len(ids) + suggested_paths = resolve_suggested_paths(definition, "youtube") + recommendations.append( + { + "id": definition.id, + "title": definition.title, + "priority": definition.priority, + "rationale": definition.rationale, + "suggested_paths": suggested_paths, + "suggested_paths_by_source": { + "youtube": suggested_paths, + }, + "supporting_video_ids": ids, + "supporting_source_types": ["youtube"], + "signal_count": count, + "weighted_signal_score": float(count), + "source_count": 1, + } + ) + + recommendations.sort(key=lambda item: (-item["signal_count"], item["title"])) + return recommendations + + +def preserve_last_good_youtube_lane(output_path: Path, current_lane: dict[str, Any]) -> dict[str, Any]: + current_success = int(current_lane.get("transcribed_video_count") or 0) + if current_success > 0: + return current_lane + + previous_lane = load_last_good_lane(output_path) + if not previous_lane: + return current_lane + + previous_success = int(previous_lane.get("transcribed_video_count") or 0) + if previous_success <= current_success: + return current_lane + + preserved_lane = dict(previous_lane) + preserved_lane["restored_at"] = iso_now() + preserved_lane["restored_reason"] = ( + f"Preserved last good YouTube lane because the latest refresh produced " + f"{current_success} transcript-backed videos and the previous snapshot had {previous_success}." + ) + preserved_lane["last_attempt"] = { + "generated_at": current_lane.get("generated_at"), + "total_video_count": current_lane.get("total_video_count"), + "transcribed_video_count": current_lane.get("transcribed_video_count"), + "failed_video_count": current_lane.get("failed_video_count"), + } + return preserved_lane + + +def build_doctor_report(args: argparse.Namespace) -> dict[str, Any]: + cache_dir = Path(args.cache_dir) + ensure_dir(cache_dir) + + checks: list[dict[str, Any]] = [] + + def add_check(name: str, ok: bool, detail: str, level: str = "info") -> None: + checks.append({ + "name": name, + "ok": ok, + "level": level, + "detail": detail, + }) + + package_versions = { + "yt_dlp": installed_package_version("yt-dlp"), + "faster_whisper": installed_package_version("faster-whisper"), + "youtube_transcript_api": installed_package_version("youtube-transcript-api"), + } + + add_check( + "cache_dir", + cache_dir.exists() and cache_dir.is_dir(), + f"Using cache dir {cache_dir}", + ) + + cookie_source = resolve_cookie_source(args) + if cookie_source["mode"] == "file": + cookie_path = Path(cookie_source["profile"]) + add_check( + "cookies_file", + cookie_path.exists() and cookie_path.is_file(), + f"Configured cookies file: {cookie_path}", + level="warning" if not (cookie_path.exists() and cookie_path.is_file()) else "info", + ) + elif cookie_source["mode"] == "browser": + profile_hint = cookie_source["profile"] or "" + add_check( + "cookies_browser", + True, + f"Using browser cookies from {cookie_source['browser']} profile {profile_hint}", + ) + else: + add_check( + "cookies_source", + False, + "No cookies source configured or auto-detected. This is often why YouTube transcript/audio access is blocked on cloud IPs.", + level="warning", + ) + + if args.proxy.strip(): + add_check("proxy", True, f"Proxy configured: {args.proxy}") + else: + add_check( + "proxy", + False, + "No proxy configured.", + level="warning", + ) + + channel_title = "" + channel_id = "" + entries: list[dict[str, Any]] = [] + try: + channel_title, channel_id, entries = list_channel_entries( + args.channel_url, + argparse.Namespace(**{**vars(args), "max_videos": max(1, args.max_videos)}), + ) + add_check( + "channel_listing", + len(entries) > 0, + f"Resolved channel '{channel_title}' ({channel_id}) with {len(entries)} visible entries.", + level="error" if not entries else "info", + ) + except Exception as error: # pragma: no cover - network-dependent + add_check("channel_listing", False, str(error), level="error") + + sample_videos = entries[: min(3, len(entries))] + if sample_videos: + audio_successes = 0 + audio_failures: list[str] = [] + captions_successes = 0 + captions_failures: list[str] = [] + + for entry in sample_videos: + video_id = str(entry.get("id") or "").strip() + video_url = entry.get("url") + if not isinstance(video_url, str) or not video_url: + video_url = f"https://www.youtube.com/watch?v={video_id}" + + audio_ok, audio_detail = probe_audio_access(video_url, args) + if audio_ok: + audio_successes += 1 + else: + audio_failures.append(f"{video_id}: {audio_detail}") + + captions_ok, captions_detail = probe_auto_captions(video_id) + if captions_ok: + captions_successes += 1 + else: + captions_failures.append(f"{video_id}: {captions_detail}") + + add_check( + "audio_access", + audio_successes == len(sample_videos), + f"Audio access succeeded for {audio_successes}/{len(sample_videos)} sampled videos." + + (f" Failures: {' | '.join(audio_failures[:2])}" if audio_failures else ""), + level="error" if audio_successes == 0 else ("warning" if audio_successes < len(sample_videos) else "info"), + ) + + add_check( + "auto_captions", + captions_successes == len(sample_videos), + f"Auto captions succeeded for {captions_successes}/{len(sample_videos)} sampled videos." + + (f" Failures: {' | '.join(captions_failures[:2])}" if captions_failures else ""), + level="info" if audio_successes > 0 else ("warning" if captions_successes < len(sample_videos) else "info"), + ) + else: + add_check( + "sample_videos", + False, + "No sample video available for transcript diagnostics.", + level="error", + ) + + hard_failures = [check for check in checks if check["level"] == "error" and not check["ok"]] + warnings = [check for check in checks if check["level"] == "warning" and not check["ok"]] + if hard_failures: + status = "unhealthy" + elif warnings: + status = "degraded" + else: + status = "healthy" + + return { + "status": status, + "checked_at": iso_now(), + "channel_url": args.channel_url, + "package_versions": package_versions, + "checks": checks, + } + + +def main() -> int: + args = parse_args() + print_runtime_summary(args) + + if args.mode == "doctor": + report = build_doctor_report(args) + print(json.dumps(report, indent=2, ensure_ascii=False)) + if report["status"] == "healthy": + return 0 + if report["status"] == "degraded": + return 2 + return 1 + + output_path = Path(args.output) + with acquire_lane_lock(output_path): + return _run_locked(args, output_path) + +def _run_locked(args: argparse.Namespace, output_path: Path) -> int: + cache_dir = Path(args.cache_dir) + audio_dir = cache_dir / "audio" + transcript_dir = cache_dir / "transcripts" + ensure_dir(audio_dir) + ensure_dir(transcript_dir) + + model = WhisperModel(args.model, device=args.device, compute_type=args.compute_type) + channel_title, channel_id, entries = list_channel_entries( + args.channel_url, + args, + ) + processing_plan = build_processing_plan(entries, transcript_dir) + + lane = empty_lane(args.channel_url) + lane["channel_title"] = channel_title + lane["channel_id"] = channel_id + lane["generated_at"] = iso_now() + lane["model_name"] = args.model + lane["total_video_count"] = len(processing_plan) + + videos: list[dict[str, Any]] = [] + failures = 0 + + for index, plan_item in enumerate(processing_plan, start=1): + entry = plan_item["entry"] + video_id = str(plan_item["video_id"] or "").strip() + if not video_id: + continue + video_url = entry.get("url") + if not isinstance(video_url, str) or not video_url: + video_url = f"https://www.youtube.com/watch?v={video_id}" + + title = sanitize_text(str(entry.get("title") or "")) + published_at = to_iso_date(entry.get("upload_date")) + duration_seconds = entry.get("duration") + view_count = entry.get("view_count") + transcript_cache = transcript_dir / f"{video_id}.json" + + print(f"[youtube-lane] {index}/{len(processing_plan)} transcribing {video_id} :: {title}", file=sys.stderr) + transcript_status = "failed" + transcript_language = "" + transcript_text = "" + transcript_source = "" + error_summary = "" + retry_attempt_count = 0 + retry_backoff_seconds = 0.0 + retry_recovered = False + transcript_cache_hit = False + + cached_transcript = load_any_complete_transcript(transcript_cache) + if cached_transcript: + transcript_status = str(cached_transcript.get("transcript_status") or "complete") + transcript_language = str(cached_transcript.get("transcript_language") or "") + transcript_text = sanitize_text(str(cached_transcript.get("transcript_text") or "")) + transcript_source = str(cached_transcript.get("transcript_source") or "") + error_summary = "" + retry_attempt_count = int(cached_transcript.get("acquisition_retry_attempt_count") or 0) + retry_backoff_seconds = float(cached_transcript.get("acquisition_retry_backoff_seconds") or 0.0) + retry_recovered = bool(cached_transcript.get("acquisition_retry_recovered")) + transcript_cache_hit = True + + if transcript_text: + signal_ids, notes, evidence = extract_signals(transcript_text) + transcript_words = len(transcript_text.split()) + videos.append( + { + "_sort_index": int(plan_item["original_index"]), + "video_id": video_id, + "video_url": video_url, + "title": title, + "published_at": published_at, + "duration_seconds": int(duration_seconds) if isinstance(duration_seconds, (int, float)) else None, + "duration_text": format_duration(duration_seconds if isinstance(duration_seconds, (int, float)) else None), + "view_count": int(view_count) if isinstance(view_count, (int, float)) else None, + "transcript_status": transcript_status, + "transcript_source": transcript_source, + "transcript_language": transcript_language, + "transcript_word_count": transcript_words, + "retry_attempt_count": retry_attempt_count, + "retry_backoff_seconds": retry_backoff_seconds, + "retry_recovered": retry_recovered, + "transcript_cache_hit": transcript_cache_hit, + "implementation_signals": signal_ids, + "implementation_notes": notes, + "evidence_snippets": evidence, + "transcript_excerpt": transcript_text[:320], + "error_summary": error_summary, + } + ) + if args.sleep_seconds > 0: + time.sleep(args.sleep_seconds) + continue + + try: + (audio_path, info), retry_meta = run_with_retry( + f"audio download for {video_id}", + lambda: download_audio(video_url, video_id, audio_dir, args), + attempts=args.retry_attempts, + backoff_seconds=args.retry_backoff_seconds, + ) + retry_attempt_count = int(retry_meta.get("attempt_count") or 0) + retry_backoff_seconds = float(retry_meta.get("backoff_seconds") or 0.0) + retry_recovered = bool(retry_meta.get("recovered_after_retry")) + if not published_at: + published_at = to_iso_date(info.get("upload_date")) + if not duration_seconds: + duration_seconds = info.get("duration") + if not view_count: + view_count = info.get("view_count") + transcript_payload = transcribe_audio( + model, + audio_path, + transcript_cache, + args.model, + acquisition_meta=retry_meta, + ) + transcript_status = str(transcript_payload.get("transcript_status") or "failed") + transcript_language = str(transcript_payload.get("transcript_language") or "") + transcript_text = sanitize_text(str(transcript_payload.get("transcript_text") or "")) + transcript_source = str(transcript_payload.get("transcript_source") or "faster_whisper") + if not args.keep_audio and audio_path.exists(): + audio_path.unlink(missing_ok=True) + except Exception as error: # pragma: no cover - network/model failure path + error_summary = str(error) + + if not transcript_text: + try: + transcript_payload = fetch_auto_captions(video_id, transcript_cache) + transcript_status = str(transcript_payload.get("transcript_status") or "failed") + transcript_language = str(transcript_payload.get("transcript_language") or "") + transcript_text = sanitize_text(str(transcript_payload.get("transcript_text") or "")) + transcript_source = str(transcript_payload.get("transcript_source") or "youtube_auto_captions") + error_summary = "" + except Exception as error: # pragma: no cover - network/caption failure path + failures += 1 + error_summary = error_summary or str(error) + write_json( + transcript_cache, + { + "model_name": args.model, + "transcript_source": "", + "transcript_status": "failed", + "transcript_language": "", + "transcript_text": "", + "generated_at": iso_now(), + "error": error_summary, + }, + ) + transcript_status = "failed" + transcript_language = "" + transcript_text = "" + transcript_source = "" + + signal_ids, notes, evidence = extract_signals(transcript_text) + transcript_words = len(transcript_text.split()) if transcript_text else 0 + + videos.append( + { + "_sort_index": int(plan_item["original_index"]), + "video_id": video_id, + "video_url": video_url, + "title": title, + "published_at": published_at, + "duration_seconds": int(duration_seconds) if isinstance(duration_seconds, (int, float)) else None, + "duration_text": format_duration(duration_seconds if isinstance(duration_seconds, (int, float)) else None), + "view_count": int(view_count) if isinstance(view_count, (int, float)) else None, + "transcript_status": transcript_status, + "transcript_source": transcript_source, + "transcript_language": transcript_language, + "transcript_word_count": transcript_words, + "retry_attempt_count": retry_attempt_count, + "retry_backoff_seconds": retry_backoff_seconds, + "retry_recovered": retry_recovered, + "transcript_cache_hit": transcript_cache_hit, + "implementation_signals": signal_ids, + "implementation_notes": notes, + "evidence_snippets": evidence, + "transcript_excerpt": transcript_text[:320], + "error_summary": error_summary, + } + ) + + if args.sleep_seconds > 0: + time.sleep(args.sleep_seconds) + + videos.sort(key=lambda video: int(video.get("_sort_index") or 0)) + for video in videos: + video.pop("_sort_index", None) + lane["videos"] = videos + lane["transcribed_video_count"] = sum(1 for video in videos if video["transcript_status"] == "complete") + lane["failed_video_count"] = sum(1 for video in videos if video["transcript_status"] != "complete") + lane["recommendations"] = build_recommendations(videos) + + lane = preserve_last_good_youtube_lane(output_path, lane) + if int(lane.get("transcribed_video_count") or 0) > 0: + save_last_good_lane(output_path, lane) + write_json(output_path, lane) + print( + f"[youtube-lane] wrote {output_path} with {lane['transcribed_video_count']} transcript-backed video records", + file=sys.stderr, + ) + return 0 + + +if __name__ == "__main__": # pragma: no cover - CLI entrypoint + raise SystemExit(main()) diff --git a/apps/exposure-studio/data/assets.json b/apps/exposure-studio/data/assets.json new file mode 100644 index 0000000..e637d3d --- /dev/null +++ b/apps/exposure-studio/data/assets.json @@ -0,0 +1,80 @@ +[ + { + "id": "asset-edge-01", + "name": "app.anygpt.example", + "kind": "public_web", + "owner": "Platform Engineering", + "authorization_state": "owned", + "authorization_basis": "first-party production asset", + "authorization_reference": "CMDB service record APP-001 and security lead approval", + "resource_scope": "domain", + "last_verified_at": "2026-03-29T00:00:00Z", + "exposure_risk": "high", + "notes": "Primary customer-facing surface with CDN and API dependencies.", + "approved_resources": [ + { + "id": "resource-edge-homepage", + "label": "Local demo homepage", + "resource_type": "url", + "target": "http://127.0.0.1:3325/", + "owner_contact": "platform-engineering@anygpt.example", + "scan_policy": "safe-http-metadata", + "approved_by": "Security lead", + "approval_reference": "demo-fixture-approval-homepage", + "notes": "Safe local demo target for HTTP metadata and security.txt checks." + } + ] + }, + { + "id": "asset-api-02", + "name": "api.anygpt.example", + "kind": "public_api", + "owner": "API Reliability", + "authorization_state": "owned", + "authorization_basis": "first-party production API", + "authorization_reference": "CMDB service record API-002 and owner attestation", + "resource_scope": "domain", + "last_verified_at": "2026-03-29T00:00:00Z", + "exposure_risk": "critical", + "notes": "Public API with machine-readable findings ingestion and remediation SLA.", + "approved_resources": [ + { + "id": "resource-api-healthz", + "label": "Local demo health endpoint", + "resource_type": "url", + "target": "http://127.0.0.1:3325/healthz", + "owner_contact": "api-reliability@anygpt.example", + "scan_policy": "safe-http-metadata", + "approved_by": "API owner", + "approval_reference": "demo-fixture-approval-api", + "notes": "Safe local demo target for response-shape and reachability checks." + } + ] + }, + { + "id": "asset-vendor-03", + "name": "status.vendor.example", + "kind": "third_party_dependency", + "owner": "Security Operations", + "authorization_state": "delegated", + "authorization_basis": "contractual monitoring delegation", + "authorization_reference": "MSA section 4.2 plus vendor security addendum", + "resource_scope": "delegated-service", + "last_verified_at": "2026-03-28T18:30:00Z", + "exposure_risk": "medium", + "notes": "Monitored under contractual authorization only.", + "approved_resources": [ + { + "id": "resource-vendor-status", + "label": "Delegated vendor status surface", + "resource_type": "domain", + "target": "status.vendor.example", + "owner_contact": "vendor-security@vendor.example", + "scan_policy": "contractual-monitoring-only", + "approved_by": "Vendor management", + "approval_reference": "msa-4.2-vendor-status", + "notes": "Delegated target included to preserve lawful monitoring and opt-out workflow examples." + } + ] + } +] diff --git a/apps/exposure-studio/data/compliance_sources.json b/apps/exposure-studio/data/compliance_sources.json new file mode 100644 index 0000000..55f6037 --- /dev/null +++ b/apps/exposure-studio/data/compliance_sources.json @@ -0,0 +1,58 @@ +[ + { + "id": "cisa-vulnerability-management", + "title": "CISA Vulnerability Management", + "category": "governance", + "url": "https://www.cisa.gov/vulnerability-management", + "requirement": "Treat vulnerability management and coordinated disclosure as measurable risk-reduction work, not just passive indexing.", + "product_response": "SurfaceScope keeps ownership, evidence, and remediation workflow attached to every finding." + }, + { + "id": "fedramp-vulnerability-scanning", + "title": "FedRAMP Vulnerability Scanning", + "category": "scanning", + "url": "https://www.fedramp.gov/docs/rev5/playbook/csp/continuous-monitoring/vulnerability-scanning/", + "requirement": "Preserve authorization state, machine-readable findings, and asset inventory linkage for scans.", + "product_response": "The product tracks authorization status per asset and expects structured findings tied to inventory identifiers." + }, + { + "id": "rfc-9116-security-txt", + "title": "RFC 9116 security.txt", + "category": "disclosure", + "url": "https://www.rfc-editor.org/rfc/rfc9116", + "requirement": "Support discoverable disclosure contacts and policy metadata for coordinated reporting.", + "product_response": "The roadmap includes security.txt ingestion and disclosure packet export." + }, + { + "id": "owasp-asvs", + "title": "OWASP ASVS", + "category": "delivery", + "url": "https://owasp.org/www-project-application-security-verification-standard/", + "requirement": "Use application security verification guidance to turn exposure evidence into developer-facing remediation work.", + "product_response": "The dashboard is built around verified findings, evidence retention, and remediation ownership." + }, + { + "id": "leakix-terms-permissible-use", + "title": "LeakIX Terms and Conditions", + "category": "lawful-ops", + "url": "https://leakix.net/terms-and-conditions", + "requirement": "Restrict platform use to authorized, professional security work and forbid harm, extortion, bulk extraction, and publication without permission.", + "product_response": "SurfaceScope stores authorization proof, recipient contacts, and disclosure workflow state on every monitored asset and finding." + }, + { + "id": "leakix-about-disclosure-window", + "title": "LeakIX About and disclosure window", + "category": "disclosure", + "url": "https://leakix.net/about", + "requirement": "Keep newly discovered issues in a restricted visibility window before wider publication, and define escalation paths for critical cases.", + "product_response": "Findings now carry visibility classes, grace-period windows, report status, and escalation targets." + }, + { + "id": "leakix-privacy-gdpr", + "title": "LeakIX Privacy Policy", + "category": "privacy", + "url": "https://leakix.net/privacy-policy", + "requirement": "Document data-minimization, KYC, retention, and data-subject-rights handling for a public exposure platform.", + "product_response": "SurfaceScope models explicit authorization references and is scoped for owned or contractually delegated assets instead of anonymous public indexing." + } +] diff --git a/apps/exposure-studio/data/findings.json b/apps/exposure-studio/data/findings.json new file mode 100644 index 0000000..4e9b74a --- /dev/null +++ b/apps/exposure-studio/data/findings.json @@ -0,0 +1,94 @@ +[ + { + "id": "finding-001", + "asset_id": "asset-api-02", + "title": "Missing disclosure contact metadata", + "severity": "medium", + "status": "triaging", + "source": "security_txt_probe", + "evidence_url": null, + "needs_disclosure": false, + "needs_owner_confirmation": true, + "report_status": "drafting-owner-contact", + "visibility": "internal-review", + "grace_period_days": 30, + "summary": "The API surface does not publish a verified disclosure contact path, so inbound coordinated disclosure handling is ambiguous.", + "related_intelligence_ids": [ + "intel-disclosure-channel" + ], + "recommended_method_ids": [ + "method-security-txt-verification", + "method-ownership-attestation" + ], + "evidence_summary": "security.txt lookup returned no valid Contact or Policy metadata on the approved API host.", + "remediation_owner": "API Reliability", + "recipient_contacts": [ + "security@anygpt.example", + "api-reliability@anygpt.example" + ], + "escalation_targets": [ + "platform-owner-oncall", + "compliance-review" + ] + }, + { + "id": "finding-002", + "asset_id": "asset-edge-01", + "title": "Unexpected admin panel exposure on public edge", + "severity": "critical", + "status": "remediating", + "source": "authorized_surface_scan", + "evidence_url": null, + "needs_disclosure": true, + "needs_owner_confirmation": false, + "report_status": "notified-owner", + "visibility": "trusted-researchers-only", + "grace_period_days": 30, + "summary": "An authenticated, approved scan found an administrative surface reachable from the public edge that should be boundary-restricted.", + "related_intelligence_ids": [ + "intel-adminer-ssrf" + ], + "recommended_method_ids": [ + "method-exposed-admin-surface-review", + "method-kev-priority-review" + ], + "evidence_summary": "Boundary review confirmed the admin route resolves on the public edge when it should be internally segmented.", + "remediation_owner": "Platform Engineering", + "recipient_contacts": [ + "security@anygpt.example", + "platform-engineering@anygpt.example" + ], + "escalation_targets": [ + "hosting-abuse-desk", + "regional-cert" + ] + }, + { + "id": "finding-003", + "asset_id": "asset-vendor-03", + "title": "Delegated dependency lacks machine-readable finding export", + "severity": "low", + "status": "new", + "source": "vendor_posture_review", + "evidence_url": null, + "needs_disclosure": false, + "needs_owner_confirmation": true, + "report_status": "awaiting-authorization-proof", + "visibility": "internal-review", + "grace_period_days": 14, + "summary": "The dependency is in scope under delegated authorization, but the current workflow cannot ingest its findings in structured form.", + "related_intelligence_ids": [], + "recommended_method_ids": [ + "method-ownership-attestation" + ], + "evidence_summary": "The dependency review captured ownership and authorization, but no machine-readable export format was available for ingestion.", + "remediation_owner": "Security Operations", + "recipient_contacts": [ + "vendor-security@vendor.example", + "security-operations@anygpt.example" + ], + "escalation_targets": [ + "procurement-owner" + ] + } +] diff --git a/apps/exposure-studio/data/research_web_targets.json b/apps/exposure-studio/data/research_web_targets.json new file mode 100644 index 0000000..83303c9 --- /dev/null +++ b/apps/exposure-studio/data/research_web_targets.json @@ -0,0 +1,32 @@ +[ + { + "id": "ai-automators-home", + "title": "The AI Automators home", + "category": "landing_page", + "url": "https://www.theaiautomators.com/" + }, + { + "id": "ai-automators-blog", + "title": "The AI Automators blog index", + "category": "blog_index", + "url": "https://www.theaiautomators.com/blog/" + }, + { + "id": "ai-automators-automations", + "title": "The AI Automators automations archive", + "category": "category_archive", + "url": "https://www.theaiautomators.com/category/automations/" + }, + { + "id": "ai-automators-agent-skills", + "title": "Anthropic's Agent Skills article", + "category": "article", + "url": "https://www.theaiautomators.com/anthropics-agent-skills/" + }, + { + "id": "ai-automators-agent-harness", + "title": "Anthropic harness article", + "category": "article", + "url": "https://www.theaiautomators.com/unlock-deep-agents-with-anthropics-agent-harness-in-n8n/" + } +] diff --git a/apps/exposure-studio/data/testing_methodologies.json b/apps/exposure-studio/data/testing_methodologies.json new file mode 100644 index 0000000..1c0db8b --- /dev/null +++ b/apps/exposure-studio/data/testing_methodologies.json @@ -0,0 +1,70 @@ +[ + { + "id": "method-security-txt-verification", + "title": "security.txt and disclosure-channel verification", + "category": "disclosure", + "safety_posture": "safe-passive", + "objective": "Verify that an owned or authorized web asset publishes a usable disclosure contact and policy path before findings are escalated.", + "operator_steps": [ + "Fetch /.well-known/security.txt from the approved asset.", + "Record whether Contact, Expires, and Policy fields are present.", + "If the file is missing or stale, create a disclosure-workflow finding rather than probing deeper." + ], + "evidence_outputs": [ + "retrieved security.txt body", + "timestamped disclosure contact evidence", + "policy-link presence or absence" + ] + }, + { + "id": "method-exposed-admin-surface-review", + "title": "authorized admin-surface review", + "category": "exposure", + "safety_posture": "safe-authz-required", + "objective": "Confirm whether a known administrative surface is reachable from the public edge and whether access boundaries are correct.", + "operator_steps": [ + "Use only an owned or explicitly authorized target.", + "Confirm whether the administrative surface resolves on the public edge.", + "Capture boundary evidence, but stop before any unapproved destructive action." + ], + "evidence_outputs": [ + "public-edge reachability evidence", + "service/banner metadata", + "route ownership confirmation" + ] + }, + { + "id": "method-kev-priority-review", + "title": "KEV and CVE prioritization review", + "category": "vulnerability-intel", + "safety_posture": "safe-passive", + "objective": "Map exposed products to known exploited or high-priority CVEs and route remediation based on authoritative public intelligence.", + "operator_steps": [ + "Normalize the exposed service to vendor/product context.", + "Compare the product against KEV and NVD-backed records already ingested by SurfaceScope.", + "Flag the finding for remediation priority when a mapped CVE is present." + ], + "evidence_outputs": [ + "product-to-CVE mapping", + "public reference URLs", + "remediation-priority note" + ] + }, + { + "id": "method-ownership-attestation", + "title": "asset ownership attestation", + "category": "governance", + "safety_posture": "safe-governance", + "objective": "Prevent ambiguous or third-party assets from entering the scanner workflow without explicit authorization evidence.", + "operator_steps": [ + "Confirm owner and business unit for the asset.", + "Record whether the asset is owned, delegated, or pending authorization.", + "Block active testing for assets that remain pending." + ], + "evidence_outputs": [ + "owner name", + "authorization state", + "review timestamp" + ] + } +] diff --git a/apps/exposure-studio/data/vulnerability_intelligence.json b/apps/exposure-studio/data/vulnerability_intelligence.json new file mode 100644 index 0000000..b631aee --- /dev/null +++ b/apps/exposure-studio/data/vulnerability_intelligence.json @@ -0,0 +1,50 @@ +[ + { + "id": "intel-adminer-ssrf", + "cve_id": "CVE-2021-21311", + "title": "Adminer server-side request forgery vulnerability", + "source_catalog": "CISA KEV", + "vendor": "Adminer", + "product": "Adminer", + "weakness": "CWE-918", + "risk_signal": "known-exploited", + "public_reference_url": "https://www.cisa.gov/known-exploited-vulnerabilities-catalog", + "remediation_focus": "Prioritize remediation or removal when an exposed Adminer surface is identified.", + "testing_method_ids": [ + "method-kev-priority-review", + "method-exposed-admin-surface-review" + ] + }, + { + "id": "intel-tplink-auth-bypass", + "cve_id": "CVE-2023-50224", + "title": "TP-Link TL-WR841N authentication bypass by spoofing", + "source_catalog": "CISA KEV", + "vendor": "TP-Link", + "product": "TL-WR841N", + "weakness": "authentication-bypass", + "risk_signal": "known-exploited", + "public_reference_url": "https://www.cisa.gov/news-events/alerts/2025/09/03/cisa-adds-two-known-exploited-vulnerabilities-catalog", + "remediation_focus": "Treat exposed internet-facing management surfaces for the affected product as urgent remediation candidates.", + "testing_method_ids": [ + "method-kev-priority-review", + "method-exposed-admin-surface-review" + ] + }, + { + "id": "intel-disclosure-channel", + "cve_id": "RFC-9116", + "title": "security.txt disclosure-channel support", + "source_catalog": "RFC 9116", + "vendor": "IETF", + "product": "security.txt", + "weakness": "disclosure-process-gap", + "risk_signal": "coordination-readiness", + "public_reference_url": "https://www.rfc-editor.org/rfc/rfc9116", + "remediation_focus": "If an exposed service lacks a clear disclosure channel, create a workflow finding to restore coordinated reporting readiness.", + "testing_method_ids": [ + "method-security-txt-verification", + "method-ownership-attestation" + ] + } +] diff --git a/apps/exposure-studio/frontend/app.js b/apps/exposure-studio/frontend/app.js new file mode 100644 index 0000000..19baaa4 --- /dev/null +++ b/apps/exposure-studio/frontend/app.js @@ -0,0 +1,603 @@ +async function loadJson(path) { + const response = await fetch(path); + if (!response.ok) { + throw new Error(`Failed to load ${path}: ${response.status}`); + } + return response.json(); +} + +function renderMetrics(summary) { + const metrics = [ + ['Authorized assets only', summary.authorized_assets_only ? 'Yes' : 'No'], + ['Tracked assets', String(summary.asset_count)], + ['Open findings', String(summary.findings_open)], + ['Critical findings', String(summary.critical_findings)], + ['Disclosure queue', String(summary.disclosure_queue)], + ['Compliance sources', String(summary.compliance_sources)], + ['Testing methods', String(summary.testing_methodologies)], + ['Intel records', String(summary.vulnerability_intelligence_items)], + ]; + + return metrics.map(([label, value]) => ` +
+ ${label} + ${value} +
+ `).join(''); +} + +function renderAssets(assets) { + return ` +
    + ${assets.map((asset) => ` +
  • +
    + ${asset.name} + ${asset.authorization_state} +
    +

    ${asset.kind} · ${asset.owner}

    + ${asset.resource_scope ? `

    Scope: ${asset.resource_scope}

    ` : ''} + ${asset.authorization_basis ? `

    Authorization basis: ${asset.authorization_basis}

    ` : ''} + ${asset.authorization_reference ? `

    Authorization proof: ${asset.authorization_reference}

    ` : ''} +

    ${asset.notes}

    +
    • + `).join('')} +
+ `; +} + +function renderApprovedResources(resources) { + return ` +
    + ${resources.map((item) => ` +
  • +
    + ${item.resource.label} + ${item.authorization_state} +
    +

    ${item.asset_name} · ${item.resource.resource_type}

    +

    Target: ${item.resource.target}

    +

    Owner contact: ${item.resource.owner_contact || 'n/a'}

    +

    Approval: ${item.resource.approved_by || 'n/a'} · ${item.resource.approval_reference || 'n/a'}

    +

    ${item.resource.notes || item.authorization_basis}

    +
    • + `).join('')} +
+ `; +} + +function renderFindings(findings) { + return ` +
    + ${findings.map((finding) => ` +
  • +
    + ${finding.title} + ${finding.severity} +
    +

    ${finding.source} · ${finding.status}${finding.asset ? ` · ${finding.asset.name}` : ''}

    +

    ${finding.summary}

    + ${finding.report_status ? `

    Report status: ${finding.report_status}

    ` : ''} + ${finding.visibility ? `

    Visibility: ${finding.visibility}${finding.grace_period_days ? ` · Grace window: ${finding.grace_period_days} day(s)` : ''}

    ` : ''} + ${finding.evidence_summary ? `

    Evidence: ${finding.evidence_summary}

    ` : ''} + ${finding.remediation_owner ? `

    Owner: ${finding.remediation_owner}

    ` : ''} + ${finding.asset ? `

    Authorization: ${finding.asset.authorization_state} · Risk: ${finding.asset.exposure_risk}

    ` : ''} + ${finding.asset?.authorization_basis ? `

    Basis: ${finding.asset.authorization_basis}

    ` : ''} + ${finding.asset?.authorization_reference ? `

    Proof: ${finding.asset.authorization_reference}

    ` : ''} + ${finding.asset?.resource_scope ? `

    Resource scope: ${finding.asset.resource_scope}

    ` : ''} +

    + ${finding.needs_owner_confirmation ? 'Owner confirmation required' : 'Owner confirmed'} + · + ${finding.needs_disclosure ? 'Disclosure queue' : 'No disclosure packet yet'} +

    + ${finding.recipient_contacts?.length ? `

    Recipients: ${finding.recipient_contacts.join(', ')}

    ` : ''} + ${finding.escalation_targets?.length ? `

    Escalation: ${finding.escalation_targets.join(', ')}

    ` : ''} + ${finding.related_intelligence?.length ? ` +
    + ${finding.related_intelligence + .map((item) => `${item.cve_id}`) + .join('')} +
    + ` : ''} + ${finding.recommended_methods?.length ? ` +
    + ${finding.recommended_methods + .map((item) => `${item.title}`) + .join('')} +
    + ` : ''} +
    • + `).join('')} +
+ `; +} + +function renderSources(sources) { + return ` +
    + ${sources.map((source) => ` +
  • +
    + ${source.title} + ${source.category} +
    +

    ${source.requirement}

    +

    ${source.product_response}

    + Open source +
    • + `).join('')} +
+ `; +} + +function renderTestingMethodologies(methods) { + return ` +
    + ${methods.map((method) => ` +
  • +
    + ${method.title} + ${method.category} +
    +

    ${method.objective}

    +

    Safety posture: ${method.safety_posture}

    +

    Evidence: ${method.evidence_outputs.join(', ')}

    +
    • + `).join('')} +
+ `; +} + +function renderVulnerabilityIntelligence(items) { + return ` +
    + ${items.map((item) => ` +
  • +
    + ${item.cve_id} + ${item.source_catalog} +
    +

    ${item.title}

    +

    ${item.vendor} · ${item.product} · ${item.weakness}

    +

    ${item.remediation_focus}

    + Open reference +
    • + `).join('')} +
+ `; +} + +function renderProductBrief(productBrief) { + return ` +
+
+

${productBrief.name}

+

${productBrief.positioning}

+
+
+
+

Guardrails

+
    + ${productBrief.guardrails.map((item) => `

  • ${item}

    • `).join('')} +
+
+
+

Next Steps

+
    + ${productBrief.next_steps.map((item) => `

  • ${item}

    • `).join('')} +
+
+
+
+ `; +} + +function renderApprovedScanResults(results) { + return ` +
    + ${results.map((result) => ` +
  • +
    + ${result.resource.label} + ${result.reachability} +
    +

    ${result.asset_name} · ${result.resource.target}

    +

    Checked: ${result.checked_at}

    + ${result.final_url ? `

    Final URL: ${result.final_url}

    ` : ''} + ${result.http_status ? `

    HTTP status: ${result.http_status}

    ` : ''} + ${result.page_title ? `

    Page title: ${result.page_title}

    ` : ''} + ${result.server_header ? `

    Server: ${result.server_header}

    ` : ''} + ${result.resolved_ips?.length ? `

    Resolved IPs: ${result.resolved_ips.join(', ')}

    ` : ''} + ${result.security_txt ? `

    security.txt: ${result.security_txt.status}${result.security_txt.contact ? ` · ${result.security_txt.contact}` : ''}

    ` : ''} + ${result.notes?.length ? `

    ${result.notes.join(' | ')}

    ` : ''} +
    • + `).join('')} +
+ `; +} + +function recommendationSupportCount(recommendation) { + const sourceSpecific = [ + ...(recommendation.supporting_item_ids || []), + ...(recommendation.supporting_video_ids || []), + ...(recommendation.supporting_page_ids || []), + ]; + return new Set(sourceSpecific).size; +} + +function renderRestoreNotice(lane) { + if (!lane?.restored_reason) { + return ''; + } + + const lastAttempt = lane?.last_attempt && typeof lane.last_attempt === 'object' + ? lane.last_attempt + : null; + + const attemptBits = []; + if (lastAttempt?.generated_at) attemptBits.push(`Attempted ${lastAttempt.generated_at}`); + if (lastAttempt?.status) attemptBits.push(`status ${lastAttempt.status}`); + if (typeof lastAttempt?.transcribed_video_count === 'number') attemptBits.push(`${lastAttempt.transcribed_video_count} transcribed`); + if (typeof lastAttempt?.ingested_page_count === 'number') attemptBits.push(`${lastAttempt.ingested_page_count} ingested`); + + return ` +
+
+ Last Good Snapshot Restored + rollback +
+ ${lane?.restored_at ? `

Restored at: ${lane.restored_at}

` : ''} +

${lane.restored_reason}

+ ${attemptBits.length ? `

Last attempt: ${attemptBits.join(' · ')}

` : ''} +
+ `; +} + +function renderSuggestedPathsBySource(recommendation) { + const mapping = recommendation?.suggested_paths_by_source; + if (!mapping || typeof mapping !== 'object') { + return ''; + } + + const entries = Object.entries(mapping).filter(([, paths]) => Array.isArray(paths) && paths.length > 0); + if (!entries.length) { + return ''; + } + + return entries.map(([source, paths]) => ` +

${source} paths: ${paths.join(', ')}

+ `).join(''); +} + +function renderRankingLegend(lane) { + const legend = lane?.importance_legend && typeof lane.importance_legend === 'object' + ? lane.importance_legend + : {}; + const parts = [ + ['article', legend.article], + ['landing page', legend.landing_page], + ['discovered internal page', legend.discovered_internal_link], + ['blog index', legend.blog_index], + ['category archive', legend.category_archive], + ].filter(([, value]) => typeof value === 'number'); + const weightText = parts.length + ? parts.map(([label, value]) => `${label} ${Number(value).toFixed(1)}`).join(' · ') + : 'article 1.0 · landing page 0.8 · discovered internal page 0.6 · blog index 0.5 · category archive 0.4'; + const scoreDescription = lane?.weighted_score_description || 'Weighted recommendation score favors richer source pages over aggregate or navigational ones.'; + + return ` +
+
+ Ranking Legend + weights +
+

${scoreDescription}

+

Page importance: ${weightText}

+

Weighted score: merged recommendation strength after applying the page importance weights above.

+
+ `; +} + +function renderYoutubeAcquisitionDetail(video) { + if (video?.transcript_cache_hit) { + if (video?.retry_recovered) { + return `cache hit · originally recovered after ${video.retry_attempt_count} attempts (${Number(video.retry_backoff_seconds || 0).toFixed(1)}s backoff)`; + } + if (typeof video?.retry_attempt_count === 'number' && video.retry_attempt_count > 0) { + return `cache hit · originally first try (${video.retry_attempt_count} attempt)`; + } + return 'cache hit'; + } + if (video?.retry_recovered) { + return `recovered after ${video.retry_attempt_count} attempts (${Number(video.retry_backoff_seconds || 0).toFixed(1)}s backoff)`; + } + if (typeof video?.retry_attempt_count === 'number' && video.retry_attempt_count > 0) { + return `first try (${video.retry_attempt_count} attempt)`; + } + if (video?.transcript_status === 'failed') { + return 'failed before transcript capture'; + } + return ''; +} + +function renderRecommendationCards(recommendations) { + return ` +
    + ${recommendations.map((recommendation) => ` +
  • +
    + ${recommendation.title} + ${recommendation.priority || 'info'} +
    +

    ${recommendation.rationale}

    +

    Signals: ${recommendation.signal_count}

    + ${typeof recommendation.weighted_signal_score === 'number' && recommendation.weighted_signal_score > 0 ? `

    Weighted score: ${recommendation.weighted_signal_score.toFixed(1)}

    ` : ''} + ${recommendation.source_count ? `

    Sources: ${recommendation.source_count}

    ` : ''} + ${recommendation.suggested_paths?.length ? `

    Suggested paths: ${recommendation.suggested_paths.join(', ')}

    ` : ''} + ${renderSuggestedPathsBySource(recommendation)} + ${recommendation.supporting_source_types?.length ? `

    Backed by ${recommendation.supporting_source_types.join(', ')}.

    ` : ''} + ${recommendationSupportCount(recommendation) ? `

    Linked items: ${recommendationSupportCount(recommendation)}

    ` : ''} +
    • + `).join('')} +
+ `; +} + +function renderImplementationLane(lane) { + const coverage = lane?.coverage || {}; + const recommendations = lane?.recommendations || []; + const metrics = [ + ['Status', lane?.status || 'unknown'], + ['Website targets', String(coverage.website_target_count || 0)], + ['Website pages', String(coverage.website_page_count || 0)], + ['Website discovered', String(coverage.website_discovered_page_count || 0)], + ['Website ingested', String(coverage.website_ingested_page_count || 0)], + ['YouTube videos', String(coverage.youtube_video_count || 0)], + ['YouTube transcribed', String(coverage.youtube_transcribed_video_count || 0)], + ]; + + return ` + ${renderRestoreNotice(lane)} + ${renderRankingLegend(lane)} +
+ ${metrics.map(([label, value]) => ` +
+ ${label} + ${value} +
+ `).join('')} +
+ ${lane?.generated_at ? `

Generated ${lane.generated_at}

` : ''} + ${recommendations.length ? renderRecommendationCards(recommendations) : ` +
+

No merged implementation recommendations are available yet. Run bash apps/exposure-studio/run.sh implementation-lane.

+
+ `} + `; +} + +function renderWebsiteLanePages(lane) { + const pages = lane?.pages || []; + if (!pages.length) { + return ` +
+

No website research documents have been indexed yet. Run bash apps/exposure-studio/run.sh website-lane.

+
+ `; + } + + return ` + ${renderRestoreNotice(lane)} + ${renderRankingLegend(lane)} +

+ ${lane?.generated_at ? `Generated ${lane.generated_at} · ` : ''} + ${typeof lane?.target_count === 'number' ? `${lane.target_count} targets` : ''} + ${typeof lane?.page_count === 'number' ? ` · ${lane.page_count} pages` : ''} + ${typeof lane?.discovered_page_count === 'number' ? ` · ${lane.discovered_page_count} discovered` : ''} +

+
    + ${pages.slice(0, 12).map((page) => ` +
  • +
    + ${page.title} + ${page.fetch_status} +
    +

    ${page.category || 'website'}${page.http_status ? ` · HTTP ${page.http_status}` : ''}${page.text_word_count ? ` · ${page.text_word_count} words` : ''}${typeof page.crawl_depth === 'number' ? ` · depth ${page.crawl_depth}` : ''}

    + ${typeof page.importance_score === 'number' && page.importance_score > 0 ? `

    Importance: ${page.importance_score.toFixed(1)}

    ` : ''} + ${page.discovered_from_page_id ? `

    Discovered from: ${page.discovered_from_page_id}

    ` : `

    Seed target: ${page.seed_target_id || page.page_id}

    `} + ${page.implementation_notes?.length ? `

    Implementation notes: ${page.implementation_notes.join(' | ')}

    ` : ''} + ${page.evidence_snippets?.length ? `

    ${page.evidence_snippets.join(' | ')}

    ` : ''} + ${page.error_summary ? `

    ${page.error_summary}

    ` : ''} + ${page.implementation_signals?.length ? ` +
    + ${page.implementation_signals.map((signal) => `${signal}`).join('')} +
    + ` : ''} + ${page.url ? `Open page` : ''} +
    • + `).join('')} +
+ `; +} + +function renderYouTubeLaneRecommendations(lane) { + const recommendations = lane?.recommendations || []; + const blockedHint = typeof lane?.failed_video_count === 'number' + && lane.failed_video_count > 0 + && lane.transcribed_video_count === 0 + ? ` +
+

+ Transcript acquisition is currently blocked on this machine. Configure + YOUTUBE_LANE_COOKIES_FILE or YOUTUBE_LANE_PROXY, run + bash apps/exposure-studio/run.sh youtube-lane doctor, then rerun + bash apps/exposure-studio/run.sh youtube-lane. +

+
+ ` + : ''; + if (!recommendations.length) { + return ` + ${blockedHint} +
+

No transcript-backed lane output has been generated yet. Run the youtube lane refresh command to populate this panel.

+
+ `; + } + + const meta = [ + lane?.generated_at ? `Generated ${lane.generated_at}` : '', + lane?.model_name ? `Model ${lane.model_name}` : '', + typeof lane?.transcribed_video_count === 'number' ? `${lane.transcribed_video_count} transcribed` : '', + ].filter(Boolean); + + return ` + ${blockedHint} + ${renderRestoreNotice(lane)} + ${meta.length ? `

${meta.join(' · ')}

` : ''} + ${renderRecommendationCards(recommendations)} + `; +} + +function renderYouTubeLaneVideos(lane) { + const videos = lane?.videos || []; + if (!videos.length) { + return ` +
+

No transcript-backed videos have been indexed yet.

+
+ `; + } + + return ` +
    + ${videos.slice(0, 12).map((video) => ` +
  • +
    + ${video.title} + ${video.transcript_status} +
    +

    ${video.published_at || 'unknown publish date'}${video.duration_text ? ` · ${video.duration_text}` : ''}${video.transcript_language ? ` · ${video.transcript_language}` : ''}${video.transcript_source ? ` · ${video.transcript_source}` : ''}

    +

    Transcript words: ${video.transcript_word_count}

    + ${renderYoutubeAcquisitionDetail(video) ? `

    Acquisition: ${renderYoutubeAcquisitionDetail(video)}

    ` : ''} + ${video.implementation_notes?.length ? `

    Implementation notes: ${video.implementation_notes.join(' | ')}

    ` : ''} + ${video.evidence_snippets?.length ? `

    ${video.evidence_snippets.join(' | ')}

    ` : ''} + ${video.error_summary ? `

    ${video.error_summary}

    ` : ''} + ${video.implementation_signals?.length ? ` +
    + ${video.implementation_signals.map((signal) => `${signal}`).join('')} +
    + ` : ''} + ${video.video_url ? `Open video` : ''} +
    • + `).join('')} +
+ `; +} + +function renderWorkflowHighlights(assets, findings) { + const ownedAssets = assets.filter((asset) => asset.authorization_state === 'owned').length; + const delegatedAssets = assets.filter((asset) => asset.authorization_state === 'delegated').length; + const ownerConfirmationQueue = findings.filter((finding) => finding.needs_owner_confirmation).length; + const disclosureQueue = findings.filter((finding) => finding.needs_disclosure).length; + const metrics = [ + ['Owned assets', String(ownedAssets)], + ['Delegated assets', String(delegatedAssets)], + ['Owner confirmations', String(ownerConfirmationQueue)], + ['Disclosure-ready findings', String(disclosureQueue)], + ]; + + return ` +
+
+

Workflow Highlights

+

What the team should verify next in the authorized exposure workflow.

+
+
+ ${metrics.map(([label, value]) => ` +
+ ${label} + ${value} +
+ `).join('')} +
+
+ `; +} + +function renderOperatorChecklist(summary, findings, methods, intel) { + const disclosureQueue = findings.filter((finding) => finding.needs_disclosure).length; + const ownerConfirmations = findings.filter((finding) => finding.needs_owner_confirmation).length; + const checklist = [ + ['Triage critical findings', summary.critical_findings > 0 ? `${summary.critical_findings} critical finding(s) need remediation sequencing.` : 'No critical findings are currently open.'], + ['Prepare disclosure packets', disclosureQueue > 0 ? `${disclosureQueue} finding(s) are marked for disclosure-safe follow-up.` : 'No disclosure packets are currently required.'], + ['Confirm asset ownership', ownerConfirmations > 0 ? `${ownerConfirmations} finding(s) still need owner confirmation before escalation.` : 'All current findings have an assigned owner confirmation state.'], + ['Validate testing coverage', methods.length > 0 ? `${methods.length} approved testing methodology record(s) are available for safe verification.` : 'Testing methodology coverage still needs to be documented.'], + ['Track public intelligence', intel.length > 0 ? `${intel.length} vulnerability intelligence record(s) are linked for remediation context.` : 'No public vulnerability intelligence has been linked yet.'], + ]; + + return ` +
+
+

Operator Checklist

+

Concrete next actions for asset owners, remediation leads, and disclosure coordinators.

+
+
    + ${checklist.map(([title, detail]) => ` +
  • +
    + ${title} + action +
    +

    ${detail}

    +
    • + `).join('')} +
+
+ `; +} + +async function boot() { + const [summary, assets, findings, enrichedFindings, sources, methods, intel, productBrief, approvedResources, approvedScanResults, websiteLane, youtubeLane, implementationLane] = await Promise.all([ + loadJson('/api/summary'), + loadJson('/api/assets'), + loadJson('/api/findings'), + loadJson('/api/findings/enriched'), + loadJson('/api/compliance/sources'), + loadJson('/api/testing-methodologies'), + loadJson('/api/vulnerability-intelligence'), + loadJson('/api/product-brief'), + loadJson('/api/assets/approved-resources'), + loadJson('/api/scans/approved'), + loadJson('/api/research/website-lane'), + loadJson('/api/research/youtube-lane'), + loadJson('/api/research/implementation-lane'), + ]); + + document.getElementById('metrics').innerHTML = renderMetrics(summary); + document.getElementById('assets').innerHTML = renderAssets(assets); + document.getElementById('findings').innerHTML = renderFindings(enrichedFindings); + document.getElementById('approved-resources').innerHTML = renderApprovedResources(approvedResources); + document.getElementById('approved-scan-results').innerHTML = renderApprovedScanResults(approvedScanResults); + document.getElementById('sources').innerHTML = renderSources(sources); + document.getElementById('methods').innerHTML = renderTestingMethodologies(methods); + document.getElementById('intel').innerHTML = renderVulnerabilityIntelligence(intel); + document.getElementById('implementation-lane').innerHTML = renderImplementationLane(implementationLane); + document.getElementById('website-lane-pages').innerHTML = renderWebsiteLanePages(websiteLane); + document.getElementById('youtube-lane-recommendations').innerHTML = renderYouTubeLaneRecommendations(youtubeLane); + document.getElementById('youtube-lane-videos').innerHTML = renderYouTubeLaneVideos(youtubeLane); + + const hero = document.querySelector('.hero'); + if (hero && !document.querySelector('.product-brief')) { + hero.insertAdjacentHTML('afterend', renderProductBrief(productBrief)); + hero.insertAdjacentHTML('afterend', renderWorkflowHighlights(assets, findings)); + hero.insertAdjacentHTML('afterend', renderOperatorChecklist(summary, findings, methods, intel)); + } +} + +boot().catch((error) => { + document.body.insertAdjacentHTML( + 'beforeend', + `
Failed to load SurfaceScope demo data: ${error instanceof Error ? error.message : String(error)}
`, + ); +}); diff --git a/apps/exposure-studio/frontend/index.html b/apps/exposure-studio/frontend/index.html new file mode 100644 index 0000000..f12433c --- /dev/null +++ b/apps/exposure-studio/frontend/index.html @@ -0,0 +1,123 @@ + + + + + + SurfaceScope + + + +
+
+

Authorized Exposure Management

+

SurfaceScope

+

+ A safer exposure-management product for teams that need inventory, evidence, + disclosure workflow, and remediation tracking for assets they actually own or are allowed to test. +

+
+ +
+ +
+
+
+

Asset Inventory

+

Ownership and authorization come first.

+
+
+
+ +
+
+

Open Findings

+

Evidence-first triage with disclosure workflow flags.

+
+
+
+
+ +
+
+
+

Approved Resources

+

Only resources with explicit approval proofs are eligible for active checks.

+
+
+
+ +
+
+

Authorized Scan Preview

+

Bounded HTTP, DNS, and security.txt checks against approved resources only.

+
+
+
+
+ +
+
+

Compliance Sources

+

Authoritative references the product is designed around.

+
+
+
+ +
+
+
+

Testing Methodologies

+

Approved methods for authorized validation and evidence capture.

+
+
+
+ +
+
+

Vulnerability Intelligence

+

Public CVE and KEV signals mapped into the exposure workflow.

+
+
+
+
+ +
+
+
+

Unified Implementation Lane

+

Merged recommendations across website research and YouTube transcripts.

+
+
+
+ +
+
+

Website Research Lane

+

Page-text derived implementation signals from tracked websites.

+
+
+
+
+ +
+
+
+

YouTube Implementation Lane

+

Transcript-derived recommendations extracted from tracked channel research.

+
+
+
+ +
+
+

Transcript-backed Video Research

+

Speech-to-text only. No title or description fallback for implementation signals.

+
+
+
+
+
+ + + + diff --git a/apps/exposure-studio/frontend/styles.css b/apps/exposure-studio/frontend/styles.css new file mode 100644 index 0000000..e564f32 --- /dev/null +++ b/apps/exposure-studio/frontend/styles.css @@ -0,0 +1,245 @@ +:root { + color-scheme: light; + --bg: #f3f6ef; + --panel: rgba(255, 255, 255, 0.82); + --line: rgba(18, 38, 32, 0.14); + --text: #112620; + --muted: #48635d; + --accent: #0f7b6c; + --accent-soft: rgba(15, 123, 108, 0.12); + --critical: #9e2733; + --high: #bb5f1b; + --medium: #6c7f1d; + --low: #35636a; + font-family: "IBM Plex Sans", "Segoe UI", sans-serif; +} + +* { + box-sizing: border-box; +} + +body { + margin: 0; + background: + radial-gradient(circle at top left, rgba(15, 123, 108, 0.15), transparent 26rem), + linear-gradient(180deg, #f7faef 0%, var(--bg) 100%); + color: var(--text); +} + +.shell { + max-width: 1180px; + margin: 0 auto; + padding: 3rem 1.25rem 4rem; +} + +.hero { + padding: 2rem 0 1rem; +} + +.eyebrow { + margin: 0 0 0.6rem; + text-transform: uppercase; + letter-spacing: 0.14em; + color: var(--accent); + font-size: 0.8rem; + font-weight: 700; +} + +h1, +h2, +p { + margin: 0; +} + +h1 { + font-size: clamp(2.4rem, 5vw, 4.6rem); + line-height: 0.95; + margin-bottom: 0.9rem; +} + +.lede { + max-width: 56rem; + color: var(--muted); + font-size: 1.08rem; + line-height: 1.6; +} + +.metrics { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); + gap: 0.85rem; + margin: 2rem 0; +} + +.metric, +.panel, +.card { + border: 1px solid var(--line); + background: var(--panel); + backdrop-filter: blur(14px); + border-radius: 22px; + box-shadow: 0 16px 50px rgba(17, 38, 32, 0.06); +} + +.metric { + padding: 1rem 1rem 1.1rem; +} + +.metric-label, +.muted, +.meta, +.panel header p { + color: var(--muted); +} + +.metric-label { + display: block; + margin-bottom: 0.45rem; + font-size: 0.88rem; +} + +.metric-value { + font-size: 1.75rem; +} + +.panel-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(320px, 1fr)); + gap: 1rem; + margin-bottom: 1rem; +} + +.panel { + padding: 1.2rem; +} + +.panel header { + display: flex; + flex-direction: column; + gap: 0.3rem; + margin-bottom: 1rem; +} + +.list { + list-style: none; + padding: 0; + margin: 0; + display: grid; + gap: 0.85rem; +} + +.card { + padding: 0.95rem 1rem; +} + +.card-row { + display: flex; + justify-content: space-between; + gap: 0.8rem; + align-items: center; + margin-bottom: 0.5rem; +} + +.pill { + display: inline-flex; + align-items: center; + border-radius: 999px; + padding: 0.2rem 0.65rem; + background: var(--accent-soft); + color: var(--accent); + font-size: 0.75rem; + font-weight: 700; +} + +.pill-critical { + background: rgba(158, 39, 51, 0.14); + color: var(--critical); +} + +.pill-high { + background: rgba(187, 95, 27, 0.14); + color: var(--high); +} + +.pill-medium { + background: rgba(108, 127, 29, 0.14); + color: var(--medium); +} + +.pill-low { + background: rgba(53, 99, 106, 0.14); + color: var(--low); +} + +.pill-owned, +.pill-delegated, +.pill-pending, +.pill-generic { + background: var(--accent-soft); + color: var(--accent); +} + +.meta { + margin-top: 0.45rem; + font-size: 0.88rem; +} + +.detail { + margin-top: 0.5rem; + color: var(--text); + font-size: 0.95rem; +} + +.tag-row { + display: flex; + flex-wrap: wrap; + gap: 0.45rem; + margin-top: 0.65rem; +} + +.tag { + display: inline-flex; + align-items: center; + border-radius: 999px; + padding: 0.18rem 0.62rem; + background: rgba(17, 38, 32, 0.07); + color: var(--text); + font-size: 0.75rem; + font-weight: 600; +} + +.tag-critical { + background: rgba(158, 39, 51, 0.12); + color: var(--critical); +} + +a { + color: var(--accent); + font-weight: 600; + text-decoration: none; +} + +a:hover { + text-decoration: underline; +} + +.error-banner { + position: sticky; + bottom: 1rem; + margin: 1rem; + padding: 0.9rem 1rem; + border-radius: 16px; + background: rgba(158, 39, 51, 0.94); + color: white; +} + +@media (max-width: 640px) { + .shell { + padding: 2rem 0.9rem 3rem; + } + + .card-row { + align-items: flex-start; + flex-direction: column; + } +} diff --git a/apps/exposure-studio/run.sh b/apps/exposure-studio/run.sh new file mode 100755 index 0000000..7c23557 --- /dev/null +++ b/apps/exposure-studio/run.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PORT="${PORT:-3325}" +export PORT + +SECRETS_FILE="$ROOT_DIR/.lane-secrets.env" +if [[ -f "$SECRETS_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + source "$SECRETS_FILE" + set +a +fi + +cd "$ROOT_DIR" + +if [[ "${1:-}" == "youtube-lane" || "${1:-}" == "website-lane" || "${1:-}" == "implementation-lane" || "${1:-}" == "refresh-all-lanes" ]]; then + LANE_NAME="$1" + shift + PYTHON_BIN="${PYTHON_BIN:-python3}" + VENV_DIR="$ROOT_DIR/backend/.venv" + REQUIREMENTS_FILE="$ROOT_DIR/backend/requirements-youtube-research.txt" + INSTALL_YOUTUBE_STACK=0 + case "$LANE_NAME" in + youtube-lane) + RUNNER_FILE="$ROOT_DIR/backend/youtube_research_runner.py" + INSTALL_YOUTUBE_STACK=1 + ;; + website-lane) + RUNNER_FILE="$ROOT_DIR/backend/website_research_runner.py" + ;; + implementation-lane) + RUNNER_FILE="$ROOT_DIR/backend/implementation_lane_runner.py" + INSTALL_YOUTUBE_STACK=1 + SKIP_REFRESH=0 + IS_DOCTOR=0 + for arg in "$@"; do + if [[ "$arg" == "--skip-refresh" ]]; then + SKIP_REFRESH=1 + elif [[ "$arg" == "doctor" ]]; then + IS_DOCTOR=1 + fi + done + if [[ "$SKIP_REFRESH" == "1" && "$IS_DOCTOR" == "0" ]]; then + INSTALL_YOUTUBE_STACK=0 + fi + ;; + refresh-all-lanes) + RUNNER_FILE="$ROOT_DIR/backend/implementation_lane_runner.py" + set -- run + INSTALL_YOUTUBE_STACK=1 + ;; + esac + + "$PYTHON_BIN" -m venv "$VENV_DIR" + "$VENV_DIR/bin/python" -m pip install --quiet --upgrade pip + if [[ "$INSTALL_YOUTUBE_STACK" == "1" ]]; then + "$VENV_DIR/bin/python" -m pip install --quiet -r "$REQUIREMENTS_FILE" + fi + exec "$VENV_DIR/bin/python" "$RUNNER_FILE" "$@" +fi + +exec cargo run --release diff --git a/apps/exposure-studio/src/main.rs b/apps/exposure-studio/src/main.rs new file mode 100644 index 0000000..fefa2d8 --- /dev/null +++ b/apps/exposure-studio/src/main.rs @@ -0,0 +1,885 @@ +use std::{collections::HashMap, env, net::SocketAddr, path::PathBuf, sync::Arc}; + +use axum::{ + extract::State, + response::IntoResponse, + routing::get, + Json, Router, +}; +use reqwest::{header, Client, Url}; +use serde::de::DeserializeOwned; +use tokio::fs; +use tokio::net::lookup_host; +use tower_http::services::{ServeDir, ServeFile}; + +mod models; + +use models::{ + ApprovedAssetResource, + ApprovedResource, + ApprovedResourceScanResult, + Asset, + ComplianceSource, + DashboardSummary, + EnrichedFinding, + FindingAssetContext, + Finding, + ImplementationResearchLane, + ProductBrief, + WebsiteResearchLane, + YouTubeResearchLane, + SecurityTxtSummary, + TestingMethodology, + VulnerabilityIntelligence, +}; + +#[derive(Clone)] +struct AppState { + assets: Arc>, + findings: Arc>, + compliance_sources: Arc>, + testing_methodologies: Arc>, + vulnerability_intelligence: Arc>, + website_research_lane: Arc, + youtube_research_lane: Arc, + implementation_research_lane: Arc, + summary: Arc, + product_brief: Arc, +} + +fn app_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) +} + +fn data_root() -> PathBuf { + app_root().join("data") +} + +fn generated_lane_root() -> PathBuf { + data_root().join(".generated") +} + +fn lane_output_path(env_var_name: &str, file_name: &str) -> PathBuf { + env::var(env_var_name) + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .map(PathBuf::from) + .unwrap_or_else(|| generated_lane_root().join(file_name)) +} + +fn build_scan_client() -> Result { + Client::builder() + .redirect(reqwest::redirect::Policy::limited(5)) + .timeout(std::time::Duration::from_secs(5)) + .user_agent("SurfaceScope/0.1 approved-resource-scan") + .build() + .map_err(|error| format!("failed to build scan client: {error}")) +} + +async fn load_json(file_name: &str) -> Result, String> +where + T: DeserializeOwned, +{ + let path = data_root().join(file_name); + let raw = fs::read_to_string(&path) + .await + .map_err(|error| format!("failed to read {}: {}", path.display(), error))?; + serde_json::from_str(&raw) + .map_err(|error| format!("failed to parse {}: {}", path.display(), error)) +} + +async fn load_optional_struct(path: PathBuf) -> Result, String> +where + T: DeserializeOwned, +{ + match fs::read_to_string(&path).await { + Ok(raw) => serde_json::from_str(&raw) + .map(Some) + .map_err(|error| format!("failed to parse {}: {}", path.display(), error)), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(format!("failed to read {}: {}", path.display(), error)), + } +} + +async fn build_state() -> Result { + let assets: Vec = load_json("assets.json").await?; + let findings: Vec = load_json("findings.json").await?; + let compliance_sources: Vec = load_json("compliance_sources.json").await?; + let testing_methodologies: Vec = load_json("testing_methodologies.json").await?; + let vulnerability_intelligence: Vec = load_json("vulnerability_intelligence.json").await?; + let website_research_lane = load_optional_struct::(lane_output_path( + "WEBSITE_LANE_OUTPUT", + "website_implementation_lane.json", + )) + .await? + .unwrap_or_default(); + let youtube_research_lane = load_optional_struct::(lane_output_path( + "YOUTUBE_LANE_OUTPUT", + "youtube_implementation_lane.json", + )) + .await? + .unwrap_or_default(); + let implementation_research_lane = + load_optional_struct::(lane_output_path( + "IMPLEMENTATION_LANE_OUTPUT", + "implementation_lane.json", + )) + .await? + .unwrap_or_default(); + + let summary = DashboardSummary { + product_name: "SurfaceScope".to_string(), + authorized_assets_only: true, + asset_count: assets.len(), + findings_open: findings.iter().filter(|finding| finding.status != "closed").count(), + disclosure_queue: findings.iter().filter(|finding| finding.needs_disclosure).count(), + critical_findings: findings.iter().filter(|finding| finding.severity == "critical").count(), + compliance_sources: compliance_sources.len(), + testing_methodologies: testing_methodologies.len(), + vulnerability_intelligence_items: vulnerability_intelligence.len(), + }; + + let product_brief = ProductBrief { + name: "SurfaceScope".to_string(), + positioning: "Authorized asset exposure management with evidence-first remediation and disclosure workflow.".to_string(), + guardrails: vec![ + "Owned or explicitly authorized assets only.".to_string(), + "Machine-readable findings and evidence retention.".to_string(), + "Disclosure-safe workflow instead of public third-party indexing.".to_string(), + "Private report handling with reviewer-only visibility windows before any wider publication.".to_string(), + "Documented authorization proof, recipient contacts, and escalation paths on every monitored asset.".to_string(), + ], + next_steps: vec![ + "Scanner import adapters".to_string(), + "Owner attestation workflow".to_string(), + "Disclosure packet export".to_string(), + "Scheduled research-lane refresh and review".to_string(), + "CVE and KEV intelligence enrichment".to_string(), + "Approved testing methodology playbooks".to_string(), + "Trusted-researcher grace windows and report visibility controls".to_string(), + "Opt-out, takedown, and abuse-handling queue".to_string(), + "Resource-claim verification for domains, networks, and delegated services".to_string(), + ], + }; + + Ok(AppState { + assets: Arc::new(assets), + findings: Arc::new(findings), + compliance_sources: Arc::new(compliance_sources), + testing_methodologies: Arc::new(testing_methodologies), + vulnerability_intelligence: Arc::new(vulnerability_intelligence), + website_research_lane: Arc::new(website_research_lane), + youtube_research_lane: Arc::new(youtube_research_lane), + implementation_research_lane: Arc::new(implementation_research_lane), + summary: Arc::new(summary), + product_brief: Arc::new(product_brief), + }) +} + +fn build_enriched_findings( + assets: &[Asset], + findings: &[Finding], + testing_methodologies: &[TestingMethodology], + vulnerability_intelligence: &[VulnerabilityIntelligence], +) -> Vec { + let asset_index: HashMap<&str, &Asset> = assets.iter().map(|asset| (asset.id.as_str(), asset)).collect(); + let method_index: HashMap<&str, &TestingMethodology> = testing_methodologies + .iter() + .map(|method| (method.id.as_str(), method)) + .collect(); + let intelligence_index: HashMap<&str, &VulnerabilityIntelligence> = vulnerability_intelligence + .iter() + .map(|item| (item.id.as_str(), item)) + .collect(); + + findings + .iter() + .map(|finding| EnrichedFinding { + id: finding.id.clone(), + asset_id: finding.asset_id.clone(), + title: finding.title.clone(), + severity: finding.severity.clone(), + status: finding.status.clone(), + source: finding.source.clone(), + evidence_url: finding.evidence_url.clone(), + needs_disclosure: finding.needs_disclosure, + needs_owner_confirmation: finding.needs_owner_confirmation, + report_status: finding.report_status.clone(), + visibility: finding.visibility.clone(), + grace_period_days: finding.grace_period_days, + summary: finding.summary.clone(), + evidence_summary: finding.evidence_summary.clone(), + remediation_owner: finding.remediation_owner.clone(), + recipient_contacts: finding.recipient_contacts.clone(), + escalation_targets: finding.escalation_targets.clone(), + asset: asset_index.get(finding.asset_id.as_str()).map(|asset| FindingAssetContext { + id: asset.id.clone(), + name: asset.name.clone(), + owner: asset.owner.clone(), + authorization_state: asset.authorization_state.clone(), + authorization_basis: asset.authorization_basis.clone(), + authorization_reference: asset.authorization_reference.clone(), + resource_scope: asset.resource_scope.clone(), + exposure_risk: asset.exposure_risk.clone(), + }), + related_intelligence: finding + .related_intelligence_ids + .iter() + .filter_map(|id| intelligence_index.get(id.as_str()).cloned().cloned()) + .collect(), + recommended_methods: finding + .recommended_method_ids + .iter() + .filter_map(|id| method_index.get(id.as_str()).cloned().cloned()) + .collect(), + }) + .collect() +} + +fn flatten_approved_resources(assets: &[Asset]) -> Vec { + assets + .iter() + .flat_map(|asset| { + asset.approved_resources.iter().cloned().map(|resource| ApprovedAssetResource { + asset_id: asset.id.clone(), + asset_name: asset.name.clone(), + authorization_state: asset.authorization_state.clone(), + authorization_basis: asset.authorization_basis.clone(), + resource_scope: asset.resource_scope.clone(), + resource, + }) + }) + .collect() +} + +fn candidate_urls_for_resource(resource: &ApprovedResource) -> Vec { + let target = resource.target.trim(); + if target.is_empty() { + return Vec::new(); + } + if target.starts_with("http://") || target.starts_with("https://") { + return vec![target.to_string()]; + } + match resource.resource_type.as_str() { + "domain" | "hostname" | "public_web" | "public_api" => vec![ + format!("https://{target}"), + format!("http://{target}"), + ], + "ip" => vec![format!("http://{target}")], + _ => vec![format!("https://{target}")], + } +} + +async fn resolve_resource_ips(resource: &ApprovedResource) -> Vec { + let target = resource.target.trim(); + if target.is_empty() { + return Vec::new(); + } + + let host = if target.starts_with("http://") || target.starts_with("https://") { + Url::parse(target) + .ok() + .and_then(|url| url.host_str().map(|host| host.to_string())) + .unwrap_or_default() + } else { + target.to_string() + }; + + if host.is_empty() { + return Vec::new(); + } + + let port = if resource.resource_type == "public_api" { 443 } else { 80 }; + let lookup_result = lookup_host((host.as_str(), port)).await; + match lookup_result { + Ok(addrs) => { + let mut ips: Vec = addrs.map(|addr| addr.ip().to_string()).collect(); + ips.sort(); + ips.dedup(); + ips + } + Err(_) => Vec::new(), + } +} + +fn extract_html_title(body: &str) -> String { + let lower = body.to_lowercase(); + let open = lower.find(""); + let close = lower.find(""); + match (open, close) { + (Some(start), Some(end)) if end > start + 7 => body[start + 7..end].trim().to_string(), + _ => String::new(), + } +} + +fn parse_security_txt(body: &str, fetched_from: &str, status: &str) -> SecurityTxtSummary { + let mut contact = String::new(); + let mut policy = String::new(); + let mut expires = String::new(); + + for line in body.lines() { + let trimmed = line.trim(); + if trimmed.is_empty() || trimmed.starts_with('#') { + continue; + } + let lower = trimmed.to_lowercase(); + if lower.starts_with("contact:") && contact.is_empty() { + contact = trimmed[8..].trim().to_string(); + } else if lower.starts_with("policy:") && policy.is_empty() { + policy = trimmed[7..].trim().to_string(); + } else if lower.starts_with("expires:") && expires.is_empty() { + expires = trimmed[8..].trim().to_string(); + } + } + + SecurityTxtSummary { + status: status.to_string(), + fetched_from: fetched_from.to_string(), + contact, + policy, + expires, + } +} + +async fn fetch_security_txt(client: &Client, resource: &ApprovedResource) -> Option { + let first_url = candidate_urls_for_resource(resource).into_iter().next()?; + let base = Url::parse(&first_url).ok()?; + let security_url = base.join("/.well-known/security.txt").ok()?; + + match client.get(security_url.clone()).send().await { + Ok(response) => { + let status_code = response.status().as_u16(); + if !response.status().is_success() { + return Some(parse_security_txt("", security_url.as_str(), if status_code == 404 { "missing" } else { "unreachable" })); + } + let content_type = response + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_string(); + let body = response.text().await.unwrap_or_default(); + let parsed = parse_security_txt(&body, security_url.as_str(), "present"); + let looks_like_html = content_type.contains("text/html") || body.to_lowercase().contains(" Some(parse_security_txt("", security_url.as_str(), "unreachable")), + } +} + +async fn scan_resource(client: &Client, asset: &Asset, resource: &ApprovedResource) -> ApprovedResourceScanResult { + let checked_at = chrono_like_now(); + let resolved_ips = resolve_resource_ips(resource).await; + let mut notes = Vec::new(); + if resolved_ips.is_empty() { + notes.push("DNS resolution did not return any IPs during this scan window.".to_string()); + } + + let mut reachability = "unreachable".to_string(); + let mut final_url = String::new(); + let mut http_status = None; + let mut page_title = String::new(); + let mut server_header = String::new(); + + for url in candidate_urls_for_resource(resource) { + match client.get(url.clone()).send().await { + Ok(response) => { + final_url = response.url().to_string(); + http_status = Some(response.status().as_u16()); + server_header = response + .headers() + .get(header::SERVER) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_string(); + let body = response.text().await.unwrap_or_default(); + page_title = extract_html_title(&body); + reachability = if http_status == Some(200) { + "reachable".to_string() + } else { + "reachable-with-error".to_string() + }; + if page_title.is_empty() { + notes.push("HTTP response did not include an HTML title tag.".to_string()); + } + break; + } + Err(error) => { + notes.push(format!("HTTP probe failed for {url}: {error}")); + } + } + } + + let security_txt = fetch_security_txt(client, resource).await; + if let Some(summary) = &security_txt { + if summary.status == "missing" { + notes.push("security.txt was missing at the approved resource origin.".to_string()); + } else if summary.status == "invalid" { + notes.push("security.txt was reachable but did not contain recognizable Contact, Policy, or Expires fields.".to_string()); + } else if summary.status == "unreachable" { + notes.push("security.txt could not be fetched from the approved resource origin.".to_string()); + } + } + + ApprovedResourceScanResult { + asset_id: asset.id.clone(), + asset_name: asset.name.clone(), + authorization_state: asset.authorization_state.clone(), + authorization_basis: asset.authorization_basis.clone(), + approval_reference: asset.authorization_reference.clone(), + resource_scope: asset.resource_scope.clone(), + resource: resource.clone(), + checked_at, + reachability, + resolved_ips, + final_url, + http_status, + page_title, + server_header, + security_txt, + notes, + } +} + +async fn scan_approved_resources(assets: &[Asset]) -> Result, String> { + let client = build_scan_client()?; + let mut results = Vec::new(); + for asset in assets { + for resource in &asset.approved_resources { + results.push(scan_resource(&client, asset, resource).await); + } + } + Ok(results) +} + +fn chrono_like_now() -> String { + use std::time::{SystemTime, UNIX_EPOCH}; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + let secs = now.as_secs(); + let tm = chrono_like_gmtime(secs); + format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}Z", + tm.year, tm.month, tm.day, tm.hour, tm.minute, tm.second + ) +} + +struct SimpleDateTime { + year: i32, + month: u32, + day: u32, + hour: u32, + minute: u32, + second: u32, +} + +fn chrono_like_gmtime(timestamp: u64) -> SimpleDateTime { + let days = (timestamp / 86_400) as i64; + let secs_of_day = (timestamp % 86_400) as u32; + let z = days + 719_468; + let era = if z >= 0 { z } else { z - 146_096 } / 146_097; + let doe = z - era * 146_097; + let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365; + let y = yoe + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let day = doy - (153 * mp + 2) / 5 + 1; + let month = mp + if mp < 10 { 3 } else { -9 }; + let year = y + if month <= 2 { 1 } else { 0 }; + + SimpleDateTime { + year: year as i32, + month: month as u32, + day: day as u32, + hour: secs_of_day / 3600, + minute: (secs_of_day % 3600) / 60, + second: secs_of_day % 60, + } +} + +async fn healthz() -> impl IntoResponse { + Json(serde_json::json!({ "status": "ok" })) +} + +async fn summary(State(state): State) -> impl IntoResponse { + Json((*state.summary).clone()) +} + +async fn assets(State(state): State) -> impl IntoResponse { + Json((*state.assets).clone()) +} + +async fn approved_resources(State(state): State) -> impl IntoResponse { + Json(flatten_approved_resources(state.assets.as_ref())) +} + +async fn findings(State(state): State) -> impl IntoResponse { + Json((*state.findings).clone()) +} + +async fn enriched_findings(State(state): State) -> impl IntoResponse { + Json(build_enriched_findings( + state.assets.as_ref(), + state.findings.as_ref(), + state.testing_methodologies.as_ref(), + state.vulnerability_intelligence.as_ref(), + )) +} + +async fn compliance_sources(State(state): State) -> impl IntoResponse { + Json((*state.compliance_sources).clone()) +} + +async fn testing_methodologies(State(state): State) -> impl IntoResponse { + Json((*state.testing_methodologies).clone()) +} + +async fn vulnerability_intelligence(State(state): State) -> impl IntoResponse { + Json((*state.vulnerability_intelligence).clone()) +} + +async fn website_research_lane(State(state): State) -> impl IntoResponse { + Json((*state.website_research_lane).clone()) +} + +async fn youtube_research_lane(State(state): State) -> impl IntoResponse { + Json((*state.youtube_research_lane).clone()) +} + +async fn implementation_research_lane(State(state): State) -> impl IntoResponse { + Json((*state.implementation_research_lane).clone()) +} + +async fn product_brief(State(state): State) -> impl IntoResponse { + Json((*state.product_brief).clone()) +} + +async fn approved_scans(State(state): State) -> impl IntoResponse { + match scan_approved_resources(state.assets.as_ref()).await { + Ok(results) => Json(results).into_response(), + Err(error) => ( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::json!({ "error": error })), + ) + .into_response(), + } +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let port = env::var("PORT") + .ok() + .and_then(|value| value.parse::().ok()) + .unwrap_or(3325); + let state = build_state().await?; + + let frontend_dir = app_root().join("frontend"); + let index_file = frontend_dir.join("index.html"); + + let app = Router::new() + .route("/healthz", get(healthz)) + .route("/api/summary", get(summary)) + .route("/api/assets", get(assets)) + .route("/api/assets/approved-resources", get(approved_resources)) + .route("/api/findings", get(findings)) + .route("/api/findings/enriched", get(enriched_findings)) + .route("/api/scans/approved", get(approved_scans)) + .route("/api/compliance/sources", get(compliance_sources)) + .route("/api/testing-methodologies", get(testing_methodologies)) + .route("/api/vulnerability-intelligence", get(vulnerability_intelligence)) + .route("/api/research/website-lane", get(website_research_lane)) + .route("/api/research/youtube-lane", get(youtube_research_lane)) + .route("/api/research/implementation-lane", get(implementation_research_lane)) + .route("/api/product-brief", get(product_brief)) + .nest_service("/static", ServeDir::new(frontend_dir.clone())) + .fallback_service(ServeFile::new(index_file)) + .with_state(state); + + let addr = SocketAddr::from(([127, 0, 0, 1], port)); + let listener = tokio::net::TcpListener::bind(addr).await?; + println!("SurfaceScope listening on http://{}", addr); + axum::serve(listener, app).await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + + use super::{build_enriched_findings, build_state}; + use crate::models::{ApprovedResource, Asset, Finding, TestingMethodology, VulnerabilityIntelligence}; + + #[test] + fn enriches_findings_with_asset_intel_and_methods() { + let assets = vec![Asset { + id: "asset-1".to_string(), + name: "api.example.test".to_string(), + kind: "public_api".to_string(), + owner: "API Team".to_string(), + authorization_state: "owned".to_string(), + authorization_basis: "service owner approval".to_string(), + authorization_reference: "CMDB record API-001".to_string(), + resource_scope: "domain".to_string(), + last_verified_at: "2026-03-29T00:00:00Z".to_string(), + exposure_risk: "critical".to_string(), + notes: "demo".to_string(), + approved_resources: vec![ApprovedResource { + id: "resource-1".to_string(), + label: "Primary app surface".to_string(), + resource_type: "url".to_string(), + target: "http://127.0.0.1:3325/".to_string(), + owner_contact: "platform@example.test".to_string(), + scan_policy: "safe-http-metadata".to_string(), + approved_by: "Security lead".to_string(), + approval_reference: "ticket-123".to_string(), + notes: "Local test target".to_string(), + }], + }]; + let findings = vec![Finding { + id: "finding-1".to_string(), + asset_id: "asset-1".to_string(), + title: "Missing disclosure contact metadata".to_string(), + severity: "medium".to_string(), + status: "triaging".to_string(), + source: "security_txt_probe".to_string(), + evidence_url: None, + needs_disclosure: false, + needs_owner_confirmation: true, + report_status: "draft".to_string(), + visibility: "internal-review".to_string(), + grace_period_days: 30, + summary: "summary".to_string(), + related_intelligence_ids: vec!["intel-1".to_string()], + recommended_method_ids: vec!["method-1".to_string()], + evidence_summary: "evidence".to_string(), + remediation_owner: "API Team".to_string(), + recipient_contacts: vec!["security@example.test".to_string()], + escalation_targets: vec!["platform-abuse@example.test".to_string()], + }]; + let methods = vec![TestingMethodology { + id: "method-1".to_string(), + title: "security.txt verification".to_string(), + category: "disclosure".to_string(), + safety_posture: "safe-passive".to_string(), + objective: "objective".to_string(), + operator_steps: vec!["step".to_string()], + evidence_outputs: vec!["body".to_string()], + }]; + let intel = vec![VulnerabilityIntelligence { + id: "intel-1".to_string(), + cve_id: "RFC-9116".to_string(), + title: "security.txt disclosure-channel support".to_string(), + source_catalog: "RFC 9116".to_string(), + vendor: "IETF".to_string(), + product: "security.txt".to_string(), + weakness: "disclosure-process-gap".to_string(), + risk_signal: "coordination-readiness".to_string(), + public_reference_url: "https://www.rfc-editor.org/rfc/rfc9116".to_string(), + remediation_focus: "focus".to_string(), + testing_method_ids: vec!["method-1".to_string()], + }]; + + let enriched = build_enriched_findings(&assets, &findings, &methods, &intel); + assert_eq!(enriched.len(), 1); + let item = &enriched[0]; + assert_eq!(item.asset.as_ref().map(|asset| asset.name.as_str()), Some("api.example.test")); + assert_eq!(item.related_intelligence.len(), 1); + assert_eq!(item.related_intelligence[0].cve_id, "RFC-9116"); + assert_eq!(item.recommended_methods.len(), 1); + assert_eq!(item.recommended_methods[0].title, "security.txt verification"); + assert_eq!(item.asset.as_ref().map(|asset| asset.authorization_basis.as_str()), Some("service owner approval")); + assert_eq!(item.report_status, "draft"); + assert_eq!(item.visibility, "internal-review"); + } + + #[tokio::test] + async fn fixture_data_loads_and_references_resolve() { + let state = build_state() + .await + .expect("fixture data should deserialize into application state"); + + assert!(!state.assets.is_empty(), "expected at least one asset fixture"); + assert!(!state.findings.is_empty(), "expected at least one finding fixture"); + assert!( + !state.compliance_sources.is_empty(), + "expected at least one compliance source fixture" + ); + assert!( + !state.testing_methodologies.is_empty(), + "expected at least one testing methodology fixture" + ); + assert!( + !state.vulnerability_intelligence.is_empty(), + "expected at least one vulnerability intelligence fixture" + ); + assert!( + state.website_research_lane.source_name.contains("Website"), + "expected website research lane fixture metadata" + ); + assert!( + state.youtube_research_lane.channel_url.starts_with("https://"), + "expected the youtube research lane to preserve its source URL" + ); + assert!( + !state.implementation_research_lane.status.trim().is_empty(), + "expected the implementation research lane to preserve a status" + ); + + let asset_ids: HashSet<&str> = state.assets.iter().map(|asset| asset.id.as_str()).collect(); + let method_ids: HashSet<&str> = state + .testing_methodologies + .iter() + .map(|method| method.id.as_str()) + .collect(); + let intelligence_ids: HashSet<&str> = state + .vulnerability_intelligence + .iter() + .map(|item| item.id.as_str()) + .collect(); + + for source in state.compliance_sources.iter() { + assert!(!source.id.trim().is_empty(), "compliance source id should not be blank"); + assert!( + !source.title.trim().is_empty(), + "compliance source {} should have a title", + source.id + ); + assert!( + source.url.starts_with("https://"), + "compliance source {} should use an https URL", + source.id + ); + assert!( + !source.requirement.trim().is_empty(), + "compliance source {} should describe the requirement", + source.id + ); + assert!( + !source.product_response.trim().is_empty(), + "compliance source {} should describe the product response", + source.id + ); + } + + for asset in state.assets.iter() { + assert!( + !asset.authorization_basis.trim().is_empty(), + "asset {} should carry an authorization basis", + asset.id + ); + assert!( + !asset.authorization_reference.trim().is_empty(), + "asset {} should carry an authorization reference", + asset.id + ); + assert!( + !asset.resource_scope.trim().is_empty(), + "asset {} should carry a resource scope", + asset.id + ); + assert!( + !asset.approved_resources.is_empty(), + "asset {} should carry at least one approved resource", + asset.id + ); + } + + for finding in state.findings.iter() { + assert!( + !finding.report_status.trim().is_empty(), + "finding {} should carry a report status", + finding.id + ); + assert!( + !finding.visibility.trim().is_empty(), + "finding {} should carry a visibility classification", + finding.id + ); + if finding.needs_disclosure { + assert!( + finding.grace_period_days > 0, + "finding {} should carry a disclosure grace period when disclosure is required", + finding.id + ); + } + assert!( + asset_ids.contains(finding.asset_id.as_str()), + "finding {} references missing asset {}", + finding.id, + finding.asset_id + ); + for intelligence_id in &finding.related_intelligence_ids { + assert!( + intelligence_ids.contains(intelligence_id.as_str()), + "finding {} references missing intelligence {}", + finding.id, + intelligence_id + ); + } + for method_id in &finding.recommended_method_ids { + assert!( + method_ids.contains(method_id.as_str()), + "finding {} references missing testing method {}", + finding.id, + method_id + ); + } + } + + for item in state.vulnerability_intelligence.iter() { + for method_id in &item.testing_method_ids { + assert!( + method_ids.contains(method_id.as_str()), + "intelligence {} references missing testing method {}", + item.id, + method_id + ); + } + } + + let enriched = build_enriched_findings( + state.assets.as_ref(), + state.findings.as_ref(), + state.testing_methodologies.as_ref(), + state.vulnerability_intelligence.as_ref(), + ); + + assert_eq!( + enriched.len(), + state.findings.len(), + "expected every fixture finding to survive enrichment" + ); + + for (finding, enriched_finding) in state.findings.iter().zip(enriched.iter()) { + assert!( + enriched_finding.asset.is_some(), + "expected enriched finding {} to include its asset context", + finding.id + ); + assert_eq!( + enriched_finding.related_intelligence.len(), + finding.related_intelligence_ids.len(), + "expected enriched finding {} to resolve all linked intelligence records", + finding.id + ); + assert_eq!( + enriched_finding.recommended_methods.len(), + finding.recommended_method_ids.len(), + "expected enriched finding {} to resolve all linked testing methods", + finding.id + ); + assert!( + !enriched_finding.visibility.trim().is_empty(), + "expected enriched finding {} to retain visibility state", + finding.id + ); + } + } +} diff --git a/apps/exposure-studio/src/models.rs b/apps/exposure-studio/src/models.rs new file mode 100644 index 0000000..be2ed69 --- /dev/null +++ b/apps/exposure-studio/src/models.rs @@ -0,0 +1,505 @@ +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct ApprovedResource { + pub id: String, + pub label: String, + pub resource_type: String, + pub target: String, + #[serde(default)] + pub owner_contact: String, + #[serde(default)] + pub scan_policy: String, + #[serde(default)] + pub approved_by: String, + #[serde(default)] + pub approval_reference: String, + #[serde(default)] + pub notes: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct Asset { + pub id: String, + pub name: String, + pub kind: String, + pub owner: String, + pub authorization_state: String, + #[serde(default)] + pub authorization_basis: String, + #[serde(default)] + pub authorization_reference: String, + #[serde(default)] + pub resource_scope: String, + pub last_verified_at: String, + pub exposure_risk: String, + #[serde(default)] + pub notes: String, + #[serde(default)] + pub approved_resources: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct Finding { + pub id: String, + pub asset_id: String, + pub title: String, + pub severity: String, + pub status: String, + pub source: String, + pub evidence_url: Option, + pub needs_disclosure: bool, + pub needs_owner_confirmation: bool, + #[serde(default)] + pub report_status: String, + #[serde(default)] + pub visibility: String, + #[serde(default)] + pub grace_period_days: u16, + pub summary: String, + #[serde(default)] + pub related_intelligence_ids: Vec, + #[serde(default)] + pub recommended_method_ids: Vec, + #[serde(default)] + pub evidence_summary: String, + #[serde(default)] + pub remediation_owner: String, + #[serde(default)] + pub recipient_contacts: Vec, + #[serde(default)] + pub escalation_targets: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct ComplianceSource { + pub id: String, + pub title: String, + pub category: String, + pub url: String, + pub requirement: String, + pub product_response: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct TestingMethodology { + pub id: String, + pub title: String, + pub category: String, + pub safety_posture: String, + pub objective: String, + pub operator_steps: Vec, + pub evidence_outputs: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct VulnerabilityIntelligence { + pub id: String, + pub cve_id: String, + pub title: String, + pub source_catalog: String, + pub vendor: String, + pub product: String, + pub weakness: String, + pub risk_signal: String, + pub public_reference_url: String, + pub remediation_focus: String, + pub testing_method_ids: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct ResearchRecommendation { + pub id: String, + pub title: String, + pub priority: String, + pub rationale: String, + #[serde(default)] + pub suggested_paths: Vec, + #[serde(default)] + pub supporting_video_ids: Vec, + #[serde(default)] + pub supporting_page_ids: Vec, + #[serde(default)] + pub supporting_item_ids: Vec, + #[serde(default)] + pub supporting_source_types: Vec, + #[serde(default)] + pub suggested_paths_by_source: BTreeMap>, + pub signal_count: usize, + #[serde(default)] + pub weighted_signal_score: f64, + #[serde(default)] + pub source_count: usize, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct YouTubeResearchVideo { + pub video_id: String, + pub video_url: String, + pub title: String, + #[serde(default)] + pub published_at: String, + pub duration_seconds: Option, + #[serde(default)] + pub duration_text: String, + pub view_count: Option, + #[serde(default)] + pub transcript_status: String, + #[serde(default)] + pub transcript_source: String, + #[serde(default)] + pub transcript_language: String, + pub transcript_word_count: usize, + #[serde(default)] + pub retry_attempt_count: usize, + #[serde(default)] + pub retry_backoff_seconds: f64, + #[serde(default)] + pub retry_recovered: bool, + #[serde(default)] + pub transcript_cache_hit: bool, + #[serde(default)] + pub implementation_signals: Vec, + #[serde(default)] + pub implementation_notes: Vec, + #[serde(default)] + pub evidence_snippets: Vec, + #[serde(default)] + pub transcript_excerpt: String, + #[serde(default)] + pub error_summary: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct YouTubeResearchLane { + pub source_name: String, + pub channel_title: String, + #[serde(default)] + pub channel_id: String, + pub channel_url: String, + #[serde(default)] + pub generated_at: String, + #[serde(default)] + pub model_name: String, + pub total_video_count: usize, + pub transcribed_video_count: usize, + pub failed_video_count: usize, + #[serde(default)] + pub restored_at: String, + #[serde(default)] + pub restored_reason: String, + #[serde(default)] + pub last_attempt: Value, + #[serde(default)] + pub recommendations: Vec, + #[serde(default)] + pub videos: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct WebsiteResearchPage { + pub page_id: String, + pub title: String, + pub url: String, + #[serde(default)] + pub category: String, + #[serde(default)] + pub fetch_status: String, + #[serde(default)] + pub final_url: String, + pub http_status: Option, + pub text_word_count: usize, + #[serde(default)] + pub importance_score: f64, + #[serde(default)] + pub implementation_signals: Vec, + #[serde(default)] + pub implementation_notes: Vec, + #[serde(default)] + pub evidence_snippets: Vec, + #[serde(default)] + pub page_excerpt: String, + #[serde(default)] + pub error_summary: String, + #[serde(default)] + pub crawl_depth: usize, + #[serde(default)] + pub discovered_from_page_id: String, + #[serde(default)] + pub seed_target_id: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct WebsiteResearchLane { + pub source_name: String, + #[serde(default)] + pub generated_at: String, + #[serde(default)] + pub importance_legend: BTreeMap, + #[serde(default)] + pub weighted_score_description: String, + pub target_count: usize, + #[serde(default)] + pub page_count: usize, + #[serde(default)] + pub discovered_page_count: usize, + pub ingested_page_count: usize, + pub failed_page_count: usize, + #[serde(default)] + pub restored_at: String, + #[serde(default)] + pub restored_reason: String, + #[serde(default)] + pub last_attempt: Value, + #[serde(default)] + pub recommendations: Vec, + #[serde(default)] + pub pages: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct ImplementationLaneCoverage { + pub youtube_video_count: usize, + pub youtube_transcribed_video_count: usize, + pub youtube_failed_video_count: usize, + #[serde(default)] + pub website_target_count: usize, + pub website_page_count: usize, + #[serde(default)] + pub website_discovered_page_count: usize, + pub website_ingested_page_count: usize, + pub website_failed_page_count: usize, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct ImplementationResearchLane { + pub source_name: String, + #[serde(default)] + pub generated_at: String, + pub status: String, + #[serde(default)] + pub importance_legend: BTreeMap, + #[serde(default)] + pub weighted_score_description: String, + #[serde(default)] + pub restored_at: String, + #[serde(default)] + pub restored_reason: String, + #[serde(default)] + pub last_attempt: Value, + pub coverage: ImplementationLaneCoverage, + #[serde(default)] + pub recommendations: Vec, +} + +fn default_last_attempt() -> Value { + Value::Object(Default::default()) +} + +fn default_importance_legend() -> BTreeMap { + BTreeMap::from([ + ("article".to_string(), 1.0), + ("blog_index".to_string(), 0.5), + ("category_archive".to_string(), 0.4), + ("default".to_string(), 0.5), + ("discovered_internal_link".to_string(), 0.6), + ("landing_page".to_string(), 0.8), + ]) +} + +fn default_weighted_score_description() -> String { + "Weighted recommendation score favors richer source pages over aggregate or navigational ones.".to_string() +} + +impl Default for YouTubeResearchLane { + fn default() -> Self { + Self { + source_name: "YouTube transcript research lane".to_string(), + channel_title: String::new(), + channel_id: String::new(), + channel_url: "https://www.youtube.com/@TheAIAutomators/videos".to_string(), + generated_at: String::new(), + model_name: String::new(), + total_video_count: 0, + transcribed_video_count: 0, + failed_video_count: 0, + restored_at: String::new(), + restored_reason: String::new(), + last_attempt: default_last_attempt(), + recommendations: Vec::new(), + videos: Vec::new(), + } + } +} + +impl Default for WebsiteResearchLane { + fn default() -> Self { + Self { + source_name: "Website text research lane".to_string(), + generated_at: String::new(), + importance_legend: default_importance_legend(), + weighted_score_description: default_weighted_score_description(), + target_count: 0, + page_count: 0, + discovered_page_count: 0, + ingested_page_count: 0, + failed_page_count: 0, + restored_at: String::new(), + restored_reason: String::new(), + last_attempt: default_last_attempt(), + recommendations: Vec::new(), + pages: Vec::new(), + } + } +} + +impl Default for ImplementationLaneCoverage { + fn default() -> Self { + Self { + youtube_video_count: 0, + youtube_transcribed_video_count: 0, + youtube_failed_video_count: 0, + website_target_count: 0, + website_page_count: 0, + website_discovered_page_count: 0, + website_ingested_page_count: 0, + website_failed_page_count: 0, + } + } +} + +impl Default for ImplementationResearchLane { + fn default() -> Self { + Self { + source_name: "Unified implementation research lane".to_string(), + generated_at: String::new(), + status: "not_generated".to_string(), + importance_legend: default_importance_legend(), + weighted_score_description: default_weighted_score_description(), + restored_at: String::new(), + restored_reason: String::new(), + last_attempt: default_last_attempt(), + coverage: ImplementationLaneCoverage::default(), + recommendations: Vec::new(), + } + } +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "snake_case")] +pub struct DashboardSummary { + pub product_name: String, + pub authorized_assets_only: bool, + pub asset_count: usize, + pub findings_open: usize, + pub disclosure_queue: usize, + pub critical_findings: usize, + pub compliance_sources: usize, + pub testing_methodologies: usize, + pub vulnerability_intelligence_items: usize, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "snake_case")] +pub struct ProductBrief { + pub name: String, + pub positioning: String, + pub guardrails: Vec, + pub next_steps: Vec, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "snake_case")] +pub struct ApprovedAssetResource { + pub asset_id: String, + pub asset_name: String, + pub authorization_state: String, + pub authorization_basis: String, + pub resource_scope: String, + pub resource: ApprovedResource, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "snake_case")] +pub struct SecurityTxtSummary { + pub status: String, + pub fetched_from: String, + pub contact: String, + pub policy: String, + pub expires: String, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "snake_case")] +pub struct ApprovedResourceScanResult { + pub asset_id: String, + pub asset_name: String, + pub authorization_state: String, + pub authorization_basis: String, + pub approval_reference: String, + pub resource_scope: String, + pub resource: ApprovedResource, + pub checked_at: String, + pub reachability: String, + pub resolved_ips: Vec, + pub final_url: String, + pub http_status: Option, + pub page_title: String, + pub server_header: String, + pub security_txt: Option, + pub notes: Vec, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "snake_case")] +pub struct FindingAssetContext { + pub id: String, + pub name: String, + pub owner: String, + pub authorization_state: String, + pub authorization_basis: String, + pub authorization_reference: String, + pub resource_scope: String, + pub exposure_risk: String, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "snake_case")] +pub struct EnrichedFinding { + pub id: String, + pub asset_id: String, + pub title: String, + pub severity: String, + pub status: String, + pub source: String, + pub evidence_url: Option, + pub needs_disclosure: bool, + pub needs_owner_confirmation: bool, + pub report_status: String, + pub visibility: String, + pub grace_period_days: u16, + pub summary: String, + pub evidence_summary: String, + pub remediation_owner: String, + pub recipient_contacts: Vec, + pub escalation_targets: Vec, + pub asset: Option, + pub related_intelligence: Vec, + pub recommended_methods: Vec, +} From 2e0d5299a78b9789e280aac3d1e1cd7a42473b8e Mon Sep 17 00:00:00 2001 From: skullcmd Date: Sun, 5 Apr 2026 05:38:02 +0000 Subject: [PATCH 3/4] chore(api): add mock mode selection to test setup cli --- apps/api/dev/testSetupCli.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/api/dev/testSetupCli.ts b/apps/api/dev/testSetupCli.ts index 3688d9e..9ecbd14 100644 --- a/apps/api/dev/testSetupCli.ts +++ b/apps/api/dev/testSetupCli.ts @@ -9,9 +9,11 @@ const envFile = process.env.NODE_ENV === 'test' ? '.env.test' : '.env'; dotenv.config({ path: path.join(projectRoot, envFile), override: true }); const action = process.argv[2] || 'setup'; +const modeArg = process.argv[3]; +const mode = modeArg === 'anthropic' ? 'anthropic' : 'openai'; if (action === 'setup') { - setupMockProviderConfig(); + setupMockProviderConfig(mode); } else if (action === 'restore') { restoreProviderConfig(); } else { From 248f840ea47f412b6a5b6244898859f55bcd800b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 5 Apr 2026 05:58:21 +0000 Subject: [PATCH 4/4] fix: address CodeQL CWE-916, XSS in frontend, and Rust reviewer issues - nativeProviders.ts: replace createHash('sha256') with createHmac for owner scope ID derivation (fixes CodeQL CWE-916 insufficient hash effort) - app.js: add escapeHtml()/safeHref() helpers and apply throughout all render functions to prevent XSS via innerHTML template interpolation - main.rs: fix extract_html_title to use byte-level ASCII lowercasing (avoids UTF-8 index mismatches from to_lowercase() on non-ASCII input) - main.rs: fix scan_resource loop to continue on non-2xx responses, only break on successful 2xx (allows http fallback after https 404) Agent-Logs-Url: https://github.com/AnyVM-Tech/AnyGPT/sessions/212a346a-4054-4a08-9865-7cb1c6b439f2 Co-authored-by: skullcrushercmd <93234024+skullcrushercmd@users.noreply.github.com> --- apps/api/routes/nativeProviders.ts | 2 +- apps/exposure-studio/frontend/app.js | 210 +++++++++++++++------------ apps/exposure-studio/src/main.rs | 19 ++- 3 files changed, 132 insertions(+), 99 deletions(-) diff --git a/apps/api/routes/nativeProviders.ts b/apps/api/routes/nativeProviders.ts index 12cb3fd..6f36686 100644 --- a/apps/api/routes/nativeProviders.ts +++ b/apps/api/routes/nativeProviders.ts @@ -298,7 +298,7 @@ function buildNativeResponsesOwnerScope( if (!normalizedApiKey) return undefined; try { return `key:${crypto - .createHash('sha256') + .createHmac('sha256', 'anygpt-native-owner-scope') .update(normalizedApiKey) .digest('hex') .slice(0, 24)}`; diff --git a/apps/exposure-studio/frontend/app.js b/apps/exposure-studio/frontend/app.js index 19baaa4..3584be1 100644 --- a/apps/exposure-studio/frontend/app.js +++ b/apps/exposure-studio/frontend/app.js @@ -6,6 +6,28 @@ async function loadJson(path) { return response.json(); } +function escapeHtml(str) { + if (str == null) return ''; + return String(str) + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +} + +function safeHref(url) { + try { + const parsed = new URL(String(url || '')); + if (parsed.protocol === 'http:' || parsed.protocol === 'https:') { + return escapeHtml(parsed.href); + } + } catch { + // not a valid URL + } + return '#'; +} + function renderMetrics(summary) { const metrics = [ ['Authorized assets only', summary.authorized_assets_only ? 'Yes' : 'No'], @@ -32,14 +54,14 @@ function renderAssets(assets) { ${assets.map((asset) => `
  • - ${asset.name} - ${asset.authorization_state} + ${escapeHtml(asset.name)} + ${escapeHtml(asset.authorization_state)}
    -

    ${asset.kind} · ${asset.owner}

    - ${asset.resource_scope ? `

    Scope: ${asset.resource_scope}

    ` : ''} - ${asset.authorization_basis ? `

    Authorization basis: ${asset.authorization_basis}

    ` : ''} - ${asset.authorization_reference ? `

    Authorization proof: ${asset.authorization_reference}

    ` : ''} -

    ${asset.notes}

    +

    ${escapeHtml(asset.kind)} · ${escapeHtml(asset.owner)}

    + ${asset.resource_scope ? `

    Scope: ${escapeHtml(asset.resource_scope)}

    ` : ''} + ${asset.authorization_basis ? `

    Authorization basis: ${escapeHtml(asset.authorization_basis)}

    ` : ''} + ${asset.authorization_reference ? `

    Authorization proof: ${escapeHtml(asset.authorization_reference)}

    ` : ''} +

    ${escapeHtml(asset.notes)}

    • `).join('')} @@ -52,14 +74,14 @@ function renderApprovedResources(resources) { ${resources.map((item) => `
  • - ${item.resource.label} - ${item.authorization_state} + ${escapeHtml(item.resource.label)} + ${escapeHtml(item.authorization_state)}
    -

    ${item.asset_name} · ${item.resource.resource_type}

    -

    Target: ${item.resource.target}

    -

    Owner contact: ${item.resource.owner_contact || 'n/a'}

    -

    Approval: ${item.resource.approved_by || 'n/a'} · ${item.resource.approval_reference || 'n/a'}

    -

    ${item.resource.notes || item.authorization_basis}

    +

    ${escapeHtml(item.asset_name)} · ${escapeHtml(item.resource.resource_type)}

    +

    Target: ${escapeHtml(item.resource.target)}

    +

    Owner contact: ${escapeHtml(item.resource.owner_contact) || 'n/a'}

    +

    Approval: ${escapeHtml(item.resource.approved_by) || 'n/a'} · ${escapeHtml(item.resource.approval_reference) || 'n/a'}

    +

    ${escapeHtml(item.resource.notes || item.authorization_basis)}

    • `).join('')} @@ -72,37 +94,37 @@ function renderFindings(findings) { ${findings.map((finding) => `
  • - ${finding.title} - ${finding.severity} + ${escapeHtml(finding.title)} + ${escapeHtml(finding.severity)}
    -

    ${finding.source} · ${finding.status}${finding.asset ? ` · ${finding.asset.name}` : ''}

    -

    ${finding.summary}

    - ${finding.report_status ? `

    Report status: ${finding.report_status}

    ` : ''} - ${finding.visibility ? `

    Visibility: ${finding.visibility}${finding.grace_period_days ? ` · Grace window: ${finding.grace_period_days} day(s)` : ''}

    ` : ''} - ${finding.evidence_summary ? `

    Evidence: ${finding.evidence_summary}

    ` : ''} - ${finding.remediation_owner ? `

    Owner: ${finding.remediation_owner}

    ` : ''} - ${finding.asset ? `

    Authorization: ${finding.asset.authorization_state} · Risk: ${finding.asset.exposure_risk}

    ` : ''} - ${finding.asset?.authorization_basis ? `

    Basis: ${finding.asset.authorization_basis}

    ` : ''} - ${finding.asset?.authorization_reference ? `

    Proof: ${finding.asset.authorization_reference}

    ` : ''} - ${finding.asset?.resource_scope ? `

    Resource scope: ${finding.asset.resource_scope}

    ` : ''} +

    ${escapeHtml(finding.source)} · ${escapeHtml(finding.status)}${finding.asset ? ` · ${escapeHtml(finding.asset.name)}` : ''}

    +

    ${escapeHtml(finding.summary)}

    + ${finding.report_status ? `

    Report status: ${escapeHtml(finding.report_status)}

    ` : ''} + ${finding.visibility ? `

    Visibility: ${escapeHtml(finding.visibility)}${finding.grace_period_days ? ` · Grace window: ${escapeHtml(finding.grace_period_days)} day(s)` : ''}

    ` : ''} + ${finding.evidence_summary ? `

    Evidence: ${escapeHtml(finding.evidence_summary)}

    ` : ''} + ${finding.remediation_owner ? `

    Owner: ${escapeHtml(finding.remediation_owner)}

    ` : ''} + ${finding.asset ? `

    Authorization: ${escapeHtml(finding.asset.authorization_state)} · Risk: ${escapeHtml(finding.asset.exposure_risk)}

    ` : ''} + ${finding.asset?.authorization_basis ? `

    Basis: ${escapeHtml(finding.asset.authorization_basis)}

    ` : ''} + ${finding.asset?.authorization_reference ? `

    Proof: ${escapeHtml(finding.asset.authorization_reference)}

    ` : ''} + ${finding.asset?.resource_scope ? `

    Resource scope: ${escapeHtml(finding.asset.resource_scope)}

    ` : ''}

    ${finding.needs_owner_confirmation ? 'Owner confirmation required' : 'Owner confirmed'} · ${finding.needs_disclosure ? 'Disclosure queue' : 'No disclosure packet yet'}

    - ${finding.recipient_contacts?.length ? `

    Recipients: ${finding.recipient_contacts.join(', ')}

    ` : ''} - ${finding.escalation_targets?.length ? `

    Escalation: ${finding.escalation_targets.join(', ')}

    ` : ''} + ${finding.recipient_contacts?.length ? `

    Recipients: ${finding.recipient_contacts.map(escapeHtml).join(', ')}

    ` : ''} + ${finding.escalation_targets?.length ? `

    Escalation: ${finding.escalation_targets.map(escapeHtml).join(', ')}

    ` : ''} ${finding.related_intelligence?.length ? `
    ${finding.related_intelligence - .map((item) => `${item.cve_id}`) + .map((item) => `${escapeHtml(item.cve_id)}`) .join('')}
    ` : ''} ${finding.recommended_methods?.length ? `
    ${finding.recommended_methods - .map((item) => `${item.title}`) + .map((item) => `${escapeHtml(item.title)}`) .join('')}
    ` : ''} @@ -118,12 +140,12 @@ function renderSources(sources) { ${sources.map((source) => `
  • - ${source.title} - ${source.category} + ${escapeHtml(source.title)} + ${escapeHtml(source.category)}
    -

    ${source.requirement}

    -

    ${source.product_response}

    - Open source +

    ${escapeHtml(source.requirement)}

    +

    ${escapeHtml(source.product_response)}

    + Open source
    • `).join('')} @@ -136,12 +158,12 @@ function renderTestingMethodologies(methods) { ${methods.map((method) => `
  • - ${method.title} - ${method.category} + ${escapeHtml(method.title)} + ${escapeHtml(method.category)}
    -

    ${method.objective}

    -

    Safety posture: ${method.safety_posture}

    -

    Evidence: ${method.evidence_outputs.join(', ')}

    +

    ${escapeHtml(method.objective)}

    +

    Safety posture: ${escapeHtml(method.safety_posture)}

    +

    Evidence: ${method.evidence_outputs.map(escapeHtml).join(', ')}

    • `).join('')} @@ -154,13 +176,13 @@ function renderVulnerabilityIntelligence(items) { ${items.map((item) => `
  • - ${item.cve_id} - ${item.source_catalog} + ${escapeHtml(item.cve_id)} + ${escapeHtml(item.source_catalog)}
    -

    ${item.title}

    -

    ${item.vendor} · ${item.product} · ${item.weakness}

    -

    ${item.remediation_focus}

    - Open reference +

    ${escapeHtml(item.title)}

    +

    ${escapeHtml(item.vendor)} · ${escapeHtml(item.product)} · ${escapeHtml(item.weakness)}

    +

    ${escapeHtml(item.remediation_focus)}

    + Open reference
    • `).join('')} @@ -171,20 +193,20 @@ function renderProductBrief(productBrief) { return `
    -

    ${productBrief.name}

    -

    ${productBrief.positioning}

    +

    ${escapeHtml(productBrief.name)}

    +

    ${escapeHtml(productBrief.positioning)}

    Guardrails

      - ${productBrief.guardrails.map((item) => `

    • ${item}

      • `).join('')} + ${productBrief.guardrails.map((item) => `

    • ${escapeHtml(item)}

      • `).join('')}

    Next Steps

      - ${productBrief.next_steps.map((item) => `

    • ${item}

      • `).join('')} + ${productBrief.next_steps.map((item) => `

    • ${escapeHtml(item)}

      • `).join('')}
    @@ -198,18 +220,18 @@ function renderApprovedScanResults(results) { ${results.map((result) => `
  • - ${result.resource.label} - ${result.reachability} + ${escapeHtml(result.resource.label)} + ${escapeHtml(result.reachability)}
    -

    ${result.asset_name} · ${result.resource.target}

    -

    Checked: ${result.checked_at}

    - ${result.final_url ? `

    Final URL: ${result.final_url}

    ` : ''} - ${result.http_status ? `

    HTTP status: ${result.http_status}

    ` : ''} - ${result.page_title ? `

    Page title: ${result.page_title}

    ` : ''} - ${result.server_header ? `

    Server: ${result.server_header}

    ` : ''} - ${result.resolved_ips?.length ? `

    Resolved IPs: ${result.resolved_ips.join(', ')}

    ` : ''} - ${result.security_txt ? `

    security.txt: ${result.security_txt.status}${result.security_txt.contact ? ` · ${result.security_txt.contact}` : ''}

    ` : ''} - ${result.notes?.length ? `

    ${result.notes.join(' | ')}

    ` : ''} +

    ${escapeHtml(result.asset_name)} · ${escapeHtml(result.resource.target)}

    +

    Checked: ${escapeHtml(result.checked_at)}

    + ${result.final_url ? `

    Final URL: ${escapeHtml(result.final_url)}

    ` : ''} + ${result.http_status ? `

    HTTP status: ${escapeHtml(result.http_status)}

    ` : ''} + ${result.page_title ? `

    Page title: ${escapeHtml(result.page_title)}

    ` : ''} + ${result.server_header ? `

    Server: ${escapeHtml(result.server_header)}

    ` : ''} + ${result.resolved_ips?.length ? `

    Resolved IPs: ${result.resolved_ips.map(escapeHtml).join(', ')}

    ` : ''} + ${result.security_txt ? `

    security.txt: ${escapeHtml(result.security_txt.status)}${result.security_txt.contact ? ` · ${escapeHtml(result.security_txt.contact)}` : ''}

    ` : ''} + ${result.notes?.length ? `

    ${result.notes.map(escapeHtml).join(' | ')}

    ` : ''}
    • `).join('')} @@ -235,8 +257,8 @@ function renderRestoreNotice(lane) { : null; const attemptBits = []; - if (lastAttempt?.generated_at) attemptBits.push(`Attempted ${lastAttempt.generated_at}`); - if (lastAttempt?.status) attemptBits.push(`status ${lastAttempt.status}`); + if (lastAttempt?.generated_at) attemptBits.push(`Attempted ${escapeHtml(lastAttempt.generated_at)}`); + if (lastAttempt?.status) attemptBits.push(`status ${escapeHtml(lastAttempt.status)}`); if (typeof lastAttempt?.transcribed_video_count === 'number') attemptBits.push(`${lastAttempt.transcribed_video_count} transcribed`); if (typeof lastAttempt?.ingested_page_count === 'number') attemptBits.push(`${lastAttempt.ingested_page_count} ingested`); @@ -246,8 +268,8 @@ function renderRestoreNotice(lane) { Last Good Snapshot Restored rollback - ${lane?.restored_at ? `

    Restored at: ${lane.restored_at}

    ` : ''} -

    ${lane.restored_reason}

    + ${lane?.restored_at ? `

    Restored at: ${escapeHtml(lane.restored_at)}

    ` : ''} +

    ${escapeHtml(lane.restored_reason)}

    ${attemptBits.length ? `

    Last attempt: ${attemptBits.join(' · ')}

    ` : ''} `; @@ -265,7 +287,7 @@ function renderSuggestedPathsBySource(recommendation) { } return entries.map(([source, paths]) => ` -

    ${source} paths: ${paths.join(', ')}

    +

    ${escapeHtml(source)} paths: ${paths.map(escapeHtml).join(', ')}

    `).join(''); } @@ -291,7 +313,7 @@ function renderRankingLegend(lane) { Ranking Legend weights -

    ${scoreDescription}

    +

    ${escapeHtml(scoreDescription)}

    Page importance: ${weightText}

    Weighted score: merged recommendation strength after applying the page importance weights above.

    @@ -326,16 +348,16 @@ function renderRecommendationCards(recommendations) { ${recommendations.map((recommendation) => `
  • - ${recommendation.title} - ${recommendation.priority || 'info'} + ${escapeHtml(recommendation.title)} + ${escapeHtml(recommendation.priority || 'info')}
    -

    ${recommendation.rationale}

    +

    ${escapeHtml(recommendation.rationale)}

    Signals: ${recommendation.signal_count}

    ${typeof recommendation.weighted_signal_score === 'number' && recommendation.weighted_signal_score > 0 ? `

    Weighted score: ${recommendation.weighted_signal_score.toFixed(1)}

    ` : ''} ${recommendation.source_count ? `

    Sources: ${recommendation.source_count}

    ` : ''} - ${recommendation.suggested_paths?.length ? `

    Suggested paths: ${recommendation.suggested_paths.join(', ')}

    ` : ''} + ${recommendation.suggested_paths?.length ? `

    Suggested paths: ${recommendation.suggested_paths.map(escapeHtml).join(', ')}

    ` : ''} ${renderSuggestedPathsBySource(recommendation)} - ${recommendation.supporting_source_types?.length ? `

    Backed by ${recommendation.supporting_source_types.join(', ')}.

    ` : ''} + ${recommendation.supporting_source_types?.length ? `

    Backed by ${recommendation.supporting_source_types.map(escapeHtml).join(', ')}.

    ` : ''} ${recommendationSupportCount(recommendation) ? `

    Linked items: ${recommendationSupportCount(recommendation)}

    ` : ''}
    • `).join('')} @@ -367,7 +389,7 @@ function renderImplementationLane(lane) { `).join('')} - ${lane?.generated_at ? `

    Generated ${lane.generated_at}

    ` : ''} + ${lane?.generated_at ? `

    Generated ${escapeHtml(lane.generated_at)}

    ` : ''} ${recommendations.length ? renderRecommendationCards(recommendations) : `

    No merged implementation recommendations are available yet. Run bash apps/exposure-studio/run.sh implementation-lane.

    @@ -390,7 +412,7 @@ function renderWebsiteLanePages(lane) { ${renderRestoreNotice(lane)} ${renderRankingLegend(lane)}

    - ${lane?.generated_at ? `Generated ${lane.generated_at} · ` : ''} + ${lane?.generated_at ? `Generated ${escapeHtml(lane.generated_at)} · ` : ''} ${typeof lane?.target_count === 'number' ? `${lane.target_count} targets` : ''} ${typeof lane?.page_count === 'number' ? ` · ${lane.page_count} pages` : ''} ${typeof lane?.discovered_page_count === 'number' ? ` · ${lane.discovered_page_count} discovered` : ''} @@ -399,21 +421,21 @@ function renderWebsiteLanePages(lane) { ${pages.slice(0, 12).map((page) => `

  • - ${page.title} - ${page.fetch_status} + ${escapeHtml(page.title)} + ${escapeHtml(page.fetch_status)}
    -

    ${page.category || 'website'}${page.http_status ? ` · HTTP ${page.http_status}` : ''}${page.text_word_count ? ` · ${page.text_word_count} words` : ''}${typeof page.crawl_depth === 'number' ? ` · depth ${page.crawl_depth}` : ''}

    +

    ${escapeHtml(page.category || 'website')}${page.http_status ? ` · HTTP ${escapeHtml(page.http_status)}` : ''}${page.text_word_count ? ` · ${page.text_word_count} words` : ''}${typeof page.crawl_depth === 'number' ? ` · depth ${page.crawl_depth}` : ''}

    ${typeof page.importance_score === 'number' && page.importance_score > 0 ? `

    Importance: ${page.importance_score.toFixed(1)}

    ` : ''} - ${page.discovered_from_page_id ? `

    Discovered from: ${page.discovered_from_page_id}

    ` : `

    Seed target: ${page.seed_target_id || page.page_id}

    `} - ${page.implementation_notes?.length ? `

    Implementation notes: ${page.implementation_notes.join(' | ')}

    ` : ''} - ${page.evidence_snippets?.length ? `

    ${page.evidence_snippets.join(' | ')}

    ` : ''} - ${page.error_summary ? `

    ${page.error_summary}

    ` : ''} + ${page.discovered_from_page_id ? `

    Discovered from: ${escapeHtml(page.discovered_from_page_id)}

    ` : `

    Seed target: ${escapeHtml(page.seed_target_id || page.page_id)}

    `} + ${page.implementation_notes?.length ? `

    Implementation notes: ${page.implementation_notes.map(escapeHtml).join(' | ')}

    ` : ''} + ${page.evidence_snippets?.length ? `

    ${page.evidence_snippets.map(escapeHtml).join(' | ')}

    ` : ''} + ${page.error_summary ? `

    ${escapeHtml(page.error_summary)}

    ` : ''} ${page.implementation_signals?.length ? `
    - ${page.implementation_signals.map((signal) => `${signal}`).join('')} + ${page.implementation_signals.map((signal) => `${escapeHtml(signal)}`).join('')}
    ` : ''} - ${page.url ? `Open page` : ''} + ${page.url ? `Open page` : ''}
    • `).join('')} @@ -446,8 +468,8 @@ function renderYouTubeLaneRecommendations(lane) { } const meta = [ - lane?.generated_at ? `Generated ${lane.generated_at}` : '', - lane?.model_name ? `Model ${lane.model_name}` : '', + lane?.generated_at ? `Generated ${escapeHtml(lane.generated_at)}` : '', + lane?.model_name ? `Model ${escapeHtml(lane.model_name)}` : '', typeof lane?.transcribed_video_count === 'number' ? `${lane.transcribed_video_count} transcribed` : '', ].filter(Boolean); @@ -474,21 +496,21 @@ function renderYouTubeLaneVideos(lane) { ${videos.slice(0, 12).map((video) => `
  • - ${video.title} - ${video.transcript_status} + ${escapeHtml(video.title)} + ${escapeHtml(video.transcript_status)}
    -

    ${video.published_at || 'unknown publish date'}${video.duration_text ? ` · ${video.duration_text}` : ''}${video.transcript_language ? ` · ${video.transcript_language}` : ''}${video.transcript_source ? ` · ${video.transcript_source}` : ''}

    +

    ${escapeHtml(video.published_at || 'unknown publish date')}${video.duration_text ? ` · ${escapeHtml(video.duration_text)}` : ''}${video.transcript_language ? ` · ${escapeHtml(video.transcript_language)}` : ''}${video.transcript_source ? ` · ${escapeHtml(video.transcript_source)}` : ''}

    Transcript words: ${video.transcript_word_count}

    ${renderYoutubeAcquisitionDetail(video) ? `

    Acquisition: ${renderYoutubeAcquisitionDetail(video)}

    ` : ''} - ${video.implementation_notes?.length ? `

    Implementation notes: ${video.implementation_notes.join(' | ')}

    ` : ''} - ${video.evidence_snippets?.length ? `

    ${video.evidence_snippets.join(' | ')}

    ` : ''} - ${video.error_summary ? `

    ${video.error_summary}

    ` : ''} + ${video.implementation_notes?.length ? `

    Implementation notes: ${video.implementation_notes.map(escapeHtml).join(' | ')}

    ` : ''} + ${video.evidence_snippets?.length ? `

    ${video.evidence_snippets.map(escapeHtml).join(' | ')}

    ` : ''} + ${video.error_summary ? `

    ${escapeHtml(video.error_summary)}

    ` : ''} ${video.implementation_signals?.length ? `
    - ${video.implementation_signals.map((signal) => `${signal}`).join('')} + ${video.implementation_signals.map((signal) => `${escapeHtml(signal)}`).join('')}
    ` : ''} - ${video.video_url ? `Open video` : ''} + ${video.video_url ? `Open video` : ''}
    • `).join('')} @@ -598,6 +620,6 @@ async function boot() { boot().catch((error) => { document.body.insertAdjacentHTML( 'beforeend', - `
    Failed to load SurfaceScope demo data: ${error instanceof Error ? error.message : String(error)}
    `, + `
    Failed to load SurfaceScope demo data: ${escapeHtml(error instanceof Error ? error.message : String(error))}
    `, ); }); diff --git a/apps/exposure-studio/src/main.rs b/apps/exposure-studio/src/main.rs index fefa2d8..58d4d9f 100644 --- a/apps/exposure-studio/src/main.rs +++ b/apps/exposure-studio/src/main.rs @@ -304,7 +304,13 @@ async fn resolve_resource_ips(resource: &ApprovedResource) -> Vec { } fn extract_html_title(body: &str) -> String { - let lower = body.to_lowercase(); + // Use ASCII-only byte-level lowercasing so byte indices remain valid for non-ASCII input. + // `to_lowercase()` can change UTF-8 byte lengths for non-ASCII chars (e.g. Turkish İ), + // making indices derived from the lowercased string invalid on the original. Since HTML + // tags only contain ASCII characters, lowercasing bytes is both correct and efficient. + let lower = String::from_utf8( + body.bytes().map(|b| b.to_ascii_lowercase()).collect() + ).unwrap_or_default(); let open = lower.find(""); let close = lower.find(""); match (open, close) { @@ -390,8 +396,9 @@ async fn scan_resource(client: &Client, asset: &Asset, resource: &ApprovedResour for url in candidate_urls_for_resource(resource) { match client.get(url.clone()).send().await { Ok(response) => { + let status = response.status(); final_url = response.url().to_string(); - http_status = Some(response.status().as_u16()); + http_status = Some(status.as_u16()); server_header = response .headers() .get(header::SERVER) @@ -400,7 +407,7 @@ async fn scan_resource(client: &Client, asset: &Asset, resource: &ApprovedResour .to_string(); let body = response.text().await.unwrap_or_default(); page_title = extract_html_title(&body); - reachability = if http_status == Some(200) { + reachability = if status.is_success() { "reachable".to_string() } else { "reachable-with-error".to_string() @@ -408,7 +415,11 @@ async fn scan_resource(client: &Client, asset: &Asset, resource: &ApprovedResour if page_title.is_empty() { notes.push("HTTP response did not include an HTML title tag.".to_string()); } - break; + // Only stop on a successful response; continue to the next candidate + // (e.g. http fallback after an https 404) otherwise. + if status.is_success() { + break; + } } Err(error) => { notes.push(format!("HTTP probe failed for {url}: {error}"));