diff --git a/.changeset/ollama-instrumentation.md b/.changeset/ollama-instrumentation.md new file mode 100644 index 000000000..52443f3be --- /dev/null +++ b/.changeset/ollama-instrumentation.md @@ -0,0 +1,5 @@ +--- +"braintrust": minor +--- + +feat: Add `ollama` instrumentation diff --git a/AGENTS.md b/AGENTS.md index c30331676..8bb1399b7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -55,6 +55,8 @@ pnpm run test:e2e:record # Re-record provider cassettes and update snap When adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output. +New instrumentation e2e coverage must test both a pinned SDK dependency and a separately named latest dependency alias for every supported version line. Only the latest alias should participate in `test:e2e:bump`; add pinned and latest variants to the CI e2e summary. + Span-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair. **From repo root:** diff --git a/e2e/README.md b/e2e/README.md index ebe595db9..5d2aa5912 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -105,6 +105,7 @@ Provider credentials are only required when recording or explicitly running live - `CURSOR_API_KEY` - `OPENROUTER_API_KEY` - `MISTRAL_API_KEY` +- `OLLAMA_API_KEY` - `HUGGINGFACE_API_KEY` - `COHERE_API_KEY` - `GROQ_API_KEY` @@ -165,7 +166,7 @@ pnpm --filter=@braintrust/js-e2e-tests run test:e2e:record -- ANTHROPIC_API_KEY=... AWS_BEARER_TOKEN_BEDROCK=... \ OPENAI_API_KEY=... GEMINI_API_KEY=... \ COHERE_API_KEY=... GROQ_API_KEY=... HUGGINGFACE_API_KEY=... \ -MISTRAL_API_KEY=... OPENROUTER_API_KEY=... \ +MISTRAL_API_KEY=... OLLAMA_API_KEY=... OPENROUTER_API_KEY=... \ CURSOR_API_KEY=... \ pnpm --filter=@braintrust/js-e2e-tests run test:e2e:record ``` @@ -173,7 +174,7 @@ CURSOR_API_KEY=... \ After recording, run again **without any provider keys** to confirm the cassette is sufficient: ```bash -unset ANTHROPIC_API_KEY AWS_BEARER_TOKEN_BEDROCK OPENAI_API_KEY GEMINI_API_KEY GOOGLE_API_KEY GOOGLE_GENAI_API_KEY COHERE_API_KEY GROQ_API_KEY HUGGINGFACE_API_KEY MISTRAL_API_KEY OPENROUTER_API_KEY CURSOR_API_KEY +unset ANTHROPIC_API_KEY AWS_BEARER_TOKEN_BEDROCK OPENAI_API_KEY GEMINI_API_KEY GOOGLE_API_KEY GOOGLE_GENAI_API_KEY COHERE_API_KEY GROQ_API_KEY HUGGINGFACE_API_KEY MISTRAL_API_KEY OLLAMA_API_KEY OPENROUTER_API_KEY CURSOR_API_KEY pnpm --filter=@braintrust/js-e2e-tests run test:e2e ``` @@ -185,7 +186,7 @@ After any successful record run, stale cassette variants are cleaned only inside These scenarios have cassette wiring in place and will use cassettes once they're recorded: -`anthropic-bedrock-instrumentation`, `anthropic-instrumentation`, `openai-instrumentation`, `openai-codex-instrumentation`, `ai-sdk-instrumentation`, `ai-sdk-otel-export`, `claude-agent-sdk-instrumentation`, `cohere-instrumentation`, `cursor-sdk-instrumentation`, `github-copilot-instrumentation`, `google-adk-instrumentation`, `google-genai-instrumentation`, `groq-instrumentation`, `huggingface-instrumentation`, `mistral-instrumentation`, `openrouter-agent-instrumentation`, `openrouter-instrumentation`, `wrap-langchain-js-traces`. +`anthropic-bedrock-instrumentation`, `anthropic-instrumentation`, `openai-instrumentation`, `openai-codex-instrumentation`, `ai-sdk-instrumentation`, `ai-sdk-otel-export`, `claude-agent-sdk-instrumentation`, `cohere-instrumentation`, `cursor-sdk-instrumentation`, `github-copilot-instrumentation`, `google-adk-instrumentation`, `google-genai-instrumentation`, `groq-instrumentation`, `huggingface-instrumentation`, `mistral-instrumentation`, `ollama-instrumentation`, `openrouter-agent-instrumentation`, `openrouter-instrumentation`, `wrap-langchain-js-traces`. ### Cassette format diff --git a/e2e/config/pr-comment-scenarios.json b/e2e/config/pr-comment-scenarios.json index 65a85ab6e..a5470a3ab 100644 --- a/e2e/config/pr-comment-scenarios.json +++ b/e2e/config/pr-comment-scenarios.json @@ -397,6 +397,29 @@ } ] }, + { + "scenarioDirName": "ollama-instrumentation", + "label": "Ollama Instrumentation", + "metadataScenario": "ollama-instrumentation", + "variants": [ + { + "variantKey": "ollama-v0.6-wrapped", + "label": "v0.6 pinned wrapped" + }, + { + "variantKey": "ollama-v0.6-auto", + "label": "v0.6 pinned auto-hook" + }, + { + "variantKey": "ollama-v0.6-latest-wrapped", + "label": "v0.6 latest wrapped" + }, + { + "variantKey": "ollama-v0.6-latest-auto", + "label": "v0.6 latest auto-hook" + } + ] + }, { "scenarioDirName": "cohere-instrumentation", "label": "Cohere Instrumentation", diff --git a/e2e/helpers/scenario-harness.ts b/e2e/helpers/scenario-harness.ts index b41e3a8be..8390f5a87 100644 --- a/e2e/helpers/scenario-harness.ts +++ b/e2e/helpers/scenario-harness.ts @@ -315,6 +315,7 @@ function getCassetteServerRoutes(): CassetteServerRoute[] { upstreamOrigin: "https://router.huggingface.co", }, { prefix: "/mistral", upstreamOrigin: "https://api.mistral.ai" }, + { prefix: "/ollama", upstreamOrigin: "https://ollama.com" }, { prefix: "/openai", upstreamOrigin: "https://api.openai.com" }, { prefix: "/openrouter", upstreamOrigin: "https://openrouter.ai" }, ]; @@ -349,6 +350,7 @@ function getCassetteEnv(wiring: ActiveCassetteWiring): Record { HUGGINGFACE_ROUTER_BASE_URL: `${serverUrl}/huggingface-router`, MISTRAL_API_URL: `${serverUrl}/mistral`, MISTRAL_BASE_URL: `${serverUrl}/mistral`, + OLLAMA_HOST: `${serverUrl}/ollama`, OPENAI_BASE_URL: `${serverUrl}/openai/v1`, OPENROUTER_BASE_URL: `${serverUrl}/openrouter/api/v1`, }; @@ -398,6 +400,7 @@ const CASSETTE_PROVIDER_KEYS: Array<{ { envVars: ["GROQ_API_KEY"], placeholder: "gsk_cassette-placeholder" }, { envVars: ["HUGGINGFACE_API_KEY"], placeholder: "hf_cassette-placeholder" }, { envVars: ["MISTRAL_API_KEY"], placeholder: "cassette-placeholder" }, + { envVars: ["OLLAMA_API_KEY"], placeholder: "cassette-placeholder" }, { envVars: ["OPENAI_API_KEY"], placeholder: "sk-cassette-placeholder" }, { envVars: ["OPENROUTER_API_KEY"], diff --git a/e2e/scenarios/ollama-instrumentation/__cassettes__/ollama-v0.6-latest.cassette.json b/e2e/scenarios/ollama-instrumentation/__cassettes__/ollama-v0.6-latest.cassette.json new file mode 100644 index 000000000..f6c999c5e --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__cassettes__/ollama-v0.6-latest.cassette.json @@ -0,0 +1,228 @@ +{ + "entries": [ + { + "callIndex": 0, + "id": "2a86bcda6598aa82", + "matchKey": "POST ollama.com/api/chat", + "recordedAt": "2026-07-30T14:17:27.786Z", + "request": { + "body": { + "kind": "json", + "value": { + "messages": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "model": "gpt-oss:20b", + "options": { + "num_predict": 128, + "temperature": 0 + }, + "stream": false, + "think": false + } + }, + "headers": {}, + "method": "POST", + "url": "https://ollama.com/api/chat" + }, + "response": { + "body": { + "kind": "json", + "value": { + "created_at": "2026-07-30T14:17:27.836768072Z", + "done": true, + "done_reason": "stop", + "eval_count": 40, + "message": { + "content": "OK", + "role": "assistant", + "thinking": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK." + }, + "model": "gpt-oss:20b", + "prompt_eval_count": 67, + "total_duration": 820667686 + } + }, + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000", + "content-length": "349", + "content-type": "application/json", + "date": "Thu, 30 Jul 2026 14:17:27 GMT", + "server": "Google Frontend", + "set-cookie": "[REDACTED]", + "traceparent": "00-c66f99321115f5f107758a3fad47a00f-bcbe1f3d1e4384ae-00", + "via": "1.1 google", + "x-build-commit": "a8a32868d6b98462cb27eaaeaf00b4aaf8110e7b", + "x-build-time": "2026-07-29T17:02:39-07:00", + "x-cloud-trace-context": "c66f99321115f5f107758a3fad47a00f/13600342272066684078", + "x-frame-options": "DENY", + "x-request-id": "127f9275-9630-4810-9a7c-6b72c370cb3c" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 1, + "id": "923a3e6055148564", + "matchKey": "POST ollama.com/api/chat", + "recordedAt": "2026-07-30T14:17:29.278Z", + "request": { + "body": { + "kind": "json", + "value": { + "messages": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "model": "gpt-oss:20b", + "options": { + "num_predict": 128, + "temperature": 0 + }, + "stream": true, + "think": false + } + }, + "headers": {}, + "method": "POST", + "url": "https://ollama.com/api/chat" + }, + "response": { + "body": { + "kind": "text", + "value": "{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.34967424Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"The\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.359378568Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" user\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.390259604Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" says\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.396460009Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\":\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.396601569Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" \\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.3975552Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"Reply\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.405247206Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" with\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.407624928Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" exactly\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.418610508Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.464847596Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\".\\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.465359817Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" So\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.465435117Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" we\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.465480977Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" must\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.478837208Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" output\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.478898448Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" exactly\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.488134706Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" the\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.496465303Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" word\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.532802133Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" \\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.532881813Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.537893028Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"\\\".\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.547788406Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" No\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.555467652Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" extra\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.571374226Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" spaces\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.571476426Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\",\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.587682829Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" no\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.592180173Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" newline\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.593067434Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"?\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.606547985Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" They\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.615570543Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" said\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.645977628Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" \\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.646838859Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"Reply\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.655935567Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" with\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.665154714Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" exactly\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.673848022Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.689652135Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\".\\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.689813995Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" Usually\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.702106365Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" we\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.710015642Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" output\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.717246658Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" the\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.726397266Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" word\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.735010853Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\".\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.789715719Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" Probably\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.790339829Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" just\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.790963009Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" \\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.79104392Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.79129428Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"\\\".\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.79150011Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" No\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.800693548Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" punctuation\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.807873884Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\".\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.808396504Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" So\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.819012393Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" output\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.840200251Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\":\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.840856441Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:28.843193773Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\".\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:29.248242423Z\",\"message\":{\"role\":\"assistant\",\"content\":\"STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:29.255514869Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:29.330188961Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\"},\"done\":true,\"done_reason\":\"stop\",\"total_duration\":1363219361,\"prompt_eval_count\":67,\"eval_count\":65}\n" + }, + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000", + "content-type": "application/json", + "date": "Thu, 30 Jul 2026 14:17:28 GMT", + "server": "Google Frontend", + "set-cookie": "[REDACTED]", + "traceparent": "00-4455c46106092326278890284531f2ca-df664b0c9e5442e6-00", + "transfer-encoding": "chunked", + "via": "1.1 google", + "x-build-commit": "a8a32868d6b98462cb27eaaeaf00b4aaf8110e7b", + "x-build-time": "2026-07-29T17:02:39-07:00", + "x-cloud-trace-context": "4455c46106092326278890284531f2ca/16097636435650429670", + "x-frame-options": "DENY", + "x-request-id": "55fbd7c8-7512-4361-a3ae-8f2810927893" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 0, + "id": "99f1d8c4b0c43219", + "matchKey": "POST ollama.com/api/generate", + "recordedAt": "2026-07-30T14:17:33.894Z", + "request": { + "body": { + "kind": "json", + "value": { + "model": "gpt-oss:20b", + "options": { + "num_predict": 128, + "temperature": 0 + }, + "prompt": "Reply with exactly OK.", + "stream": false, + "think": false + } + }, + "headers": {}, + "method": "POST", + "url": "https://ollama.com/api/generate" + }, + "response": { + "body": { + "kind": "json", + "value": { + "created_at": "2026-07-30T14:17:33.941708086Z", + "done": true, + "done_reason": "stop", + "eval_count": 40, + "model": "gpt-oss:20b", + "prompt_eval_count": 67, + "response": "OK", + "thinking": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "total_duration": 4489076853 + } + }, + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000", + "content-length": "320", + "content-type": "application/json", + "date": "Thu, 30 Jul 2026 14:17:33 GMT", + "server": "Google Frontend", + "set-cookie": "[REDACTED]", + "traceparent": "00-106b492de1a5a0ce278890284531f004-f53ba4690e6cf0bc-00", + "via": "1.1 google", + "x-build-commit": "a8a32868d6b98462cb27eaaeaf00b4aaf8110e7b", + "x-build-time": "2026-07-29T17:02:39-07:00", + "x-cloud-trace-context": "106b492de1a5a0ce278890284531f004/17670898334038814908", + "x-frame-options": "DENY", + "x-request-id": "203de0b5-0587-4e63-88c6-e083c1074a17" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 1, + "id": "54471dd63ef97520", + "matchKey": "POST ollama.com/api/generate", + "recordedAt": "2026-07-30T14:17:34.958Z", + "request": { + "body": { + "kind": "json", + "value": { + "model": "gpt-oss:20b", + "options": { + "num_predict": 128, + "temperature": 0 + }, + "prompt": "Reply with exactly STREAM.", + "stream": true, + "think": false + } + }, + "headers": {}, + "method": "POST", + "url": "https://ollama.com/api/generate" + }, + "response": { + "body": { + "kind": "text", + "value": "{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.389627377Z\",\"response\":\"\",\"thinking\":\"The\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.395533242Z\",\"response\":\"\",\"thinking\":\" user\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.403797509Z\",\"response\":\"\",\"thinking\":\" says\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.412127386Z\",\"response\":\"\",\"thinking\":\":\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.420263983Z\",\"response\":\"\",\"thinking\":\" \\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.431670083Z\",\"response\":\"\",\"thinking\":\"Reply\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.439038379Z\",\"response\":\"\",\"thinking\":\" with\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.445798215Z\",\"response\":\"\",\"thinking\":\" exactly\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.453676261Z\",\"response\":\"\",\"thinking\":\" STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.462879529Z\",\"response\":\"\",\"thinking\":\".\\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.48754039Z\",\"response\":\"\",\"thinking\":\" So\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.497012667Z\",\"response\":\"\",\"thinking\":\" we\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.503352273Z\",\"response\":\"\",\"thinking\":\" must\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.508533877Z\",\"response\":\"\",\"thinking\":\" output\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.518284895Z\",\"response\":\"\",\"thinking\":\" exactly\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.524960831Z\",\"response\":\"\",\"thinking\":\" the\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.532656867Z\",\"response\":\"\",\"thinking\":\" word\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.555859397Z\",\"response\":\"\",\"thinking\":\" \\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.563573723Z\",\"response\":\"\",\"thinking\":\"STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.564276844Z\",\"response\":\"\",\"thinking\":\"\\\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.566337645Z\",\"response\":\"\",\"thinking\":\" No\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.573933902Z\",\"response\":\"\",\"thinking\":\" extra\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.581418518Z\",\"response\":\"\",\"thinking\":\" spaces\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.589888165Z\",\"response\":\"\",\"thinking\":\",\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.601848775Z\",\"response\":\"\",\"thinking\":\" no\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.610821713Z\",\"response\":\"\",\"thinking\":\" newline\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.615298726Z\",\"response\":\"\",\"thinking\":\"?\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.624505524Z\",\"response\":\"\",\"thinking\":\" They\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.632406001Z\",\"response\":\"\",\"thinking\":\" said\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.642595159Z\",\"response\":\"\",\"thinking\":\" \\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.650027446Z\",\"response\":\"\",\"thinking\":\"Reply\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.661598875Z\",\"response\":\"\",\"thinking\":\" with\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.669144222Z\",\"response\":\"\",\"thinking\":\" exactly\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.675096107Z\",\"response\":\"\",\"thinking\":\" STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.686189736Z\",\"response\":\"\",\"thinking\":\".\\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.692248381Z\",\"response\":\"\",\"thinking\":\" Usually\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.705089872Z\",\"response\":\"\",\"thinking\":\" we\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.709504965Z\",\"response\":\"\",\"thinking\":\" output\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.720336364Z\",\"response\":\"\",\"thinking\":\" the\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.72721519Z\",\"response\":\"\",\"thinking\":\" word\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.735638087Z\",\"response\":\"\",\"thinking\":\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.747191607Z\",\"response\":\"\",\"thinking\":\" Probably\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.760642398Z\",\"response\":\"\",\"thinking\":\" just\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.778758373Z\",\"response\":\"\",\"thinking\":\" \\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.779113164Z\",\"response\":\"\",\"thinking\":\"STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.798139999Z\",\"response\":\"\",\"thinking\":\"\\\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.799441891Z\",\"response\":\"\",\"thinking\":\" No\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.799505571Z\",\"response\":\"\",\"thinking\":\" punctuation\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.808817408Z\",\"response\":\"\",\"thinking\":\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.827693204Z\",\"response\":\"\",\"thinking\":\" So\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.837738383Z\",\"response\":\"\",\"thinking\":\" final\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.849834023Z\",\"response\":\"\",\"thinking\":\" answer\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.856697219Z\",\"response\":\"\",\"thinking\":\":\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.863521814Z\",\"response\":\"\",\"thinking\":\" STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.865494336Z\",\"response\":\"\",\"thinking\":\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.927466028Z\",\"response\":\"STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:34.937183716Z\",\"response\":\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:35.009953287Z\",\"response\":\"\",\"done\":true,\"done_reason\":\"stop\",\"total_duration\":946827533,\"prompt_eval_count\":67,\"eval_count\":66}\n" + }, + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000", + "content-type": "application/json", + "date": "Thu, 30 Jul 2026 14:17:34 GMT", + "server": "Google Frontend", + "set-cookie": "[REDACTED]", + "traceparent": "00-e8e48c6fbc8884675481ff7328ae2d0b-7a094f57d5536a16-00", + "transfer-encoding": "chunked", + "via": "1.1 google", + "x-build-commit": "a8a32868d6b98462cb27eaaeaf00b4aaf8110e7b", + "x-build-time": "2026-07-29T17:02:39-07:00", + "x-cloud-trace-context": "e8e48c6fbc8884675481ff7328ae2d0b/8793646986077366806", + "x-frame-options": "DENY", + "x-request-id": "97ff230e-37fe-4bb0-a698-396b552dcdcb" + }, + "status": 200, + "statusText": "OK" + } + } + ], + "meta": { + "createdAt": "2026-07-30T14:15:13.600Z" + } +} diff --git a/e2e/scenarios/ollama-instrumentation/__cassettes__/ollama-v0.6.cassette.json b/e2e/scenarios/ollama-instrumentation/__cassettes__/ollama-v0.6.cassette.json new file mode 100644 index 000000000..e092efa99 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__cassettes__/ollama-v0.6.cassette.json @@ -0,0 +1,228 @@ +{ + "entries": [ + { + "callIndex": 0, + "id": "2a86bcda6598aa82", + "matchKey": "POST ollama.com/api/chat", + "recordedAt": "2026-07-30T14:17:15.919Z", + "request": { + "body": { + "kind": "json", + "value": { + "messages": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "model": "gpt-oss:20b", + "options": { + "num_predict": 128, + "temperature": 0 + }, + "stream": false, + "think": false + } + }, + "headers": {}, + "method": "POST", + "url": "https://ollama.com/api/chat" + }, + "response": { + "body": { + "kind": "json", + "value": { + "created_at": "2026-07-30T14:17:15.968759871Z", + "done": true, + "done_reason": "stop", + "eval_count": 40, + "message": { + "content": "OK", + "role": "assistant", + "thinking": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK." + }, + "model": "gpt-oss:20b", + "prompt_eval_count": 67, + "total_duration": 2759051201 + } + }, + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000", + "content-length": "350", + "content-type": "application/json", + "date": "Thu, 30 Jul 2026 14:17:15 GMT", + "server": "Google Frontend", + "set-cookie": "[REDACTED]", + "traceparent": "00-499b9cbcd8010b751b766ba772b23e2e-45c927d9e2a30383-00", + "via": "1.1 google", + "x-build-commit": "a8a32868d6b98462cb27eaaeaf00b4aaf8110e7b", + "x-build-time": "2026-07-29T17:02:39-07:00", + "x-cloud-trace-context": "499b9cbcd8010b751b766ba772b23e2e/5028594275699590019", + "x-frame-options": "DENY", + "x-request-id": "69feef07-e84b-461c-b965-82ca63d5cd59" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 1, + "id": "923a3e6055148564", + "matchKey": "POST ollama.com/api/chat", + "recordedAt": "2026-07-30T14:17:16.947Z", + "request": { + "body": { + "kind": "json", + "value": { + "messages": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "model": "gpt-oss:20b", + "options": { + "num_predict": 128, + "temperature": 0 + }, + "stream": true, + "think": false + } + }, + "headers": {}, + "method": "POST", + "url": "https://ollama.com/api/chat" + }, + "response": { + "body": { + "kind": "text", + "value": "{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.420595067Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"The\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.428180554Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" user\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.434645149Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" says\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.443487256Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\":\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.454888486Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" \\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.464434994Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"Reply\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.468907788Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" with\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.475863774Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" exactly\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.486244792Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.494834779Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\".\\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.501858345Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" So\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.513830845Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" the\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.522514833Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" assistant\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.528664688Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" should\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.536021444Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" reply\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.54387051Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" with\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.553723509Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" exactly\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.561350065Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" the\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.588566508Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" word\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.598017396Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" \\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.60322265Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.609424945Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"\\\".\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.619468064Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" No\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.643468554Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" other\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.644496975Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" characters\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.65059612Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\",\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.665049332Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" no\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.666145933Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" punctuation\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.675696741Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\",\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.685230109Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" no\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.68602731Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" whitespace\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.702263893Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\".\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.718099466Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" Just\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.764431305Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\" \\\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.794734681Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.843230121Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\",\"thinking\":\"\\\".\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.895195035Z\",\"message\":{\"role\":\"assistant\",\"content\":\"STREAM\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.897310446Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\"},\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:16.996437479Z\",\"message\":{\"role\":\"assistant\",\"content\":\"\"},\"done\":true,\"done_reason\":\"stop\",\"total_duration\":913140194,\"prompt_eval_count\":67,\"eval_count\":47}\n" + }, + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000", + "content-type": "application/json", + "date": "Thu, 30 Jul 2026 14:17:16 GMT", + "server": "Google Frontend", + "set-cookie": "[REDACTED]", + "traceparent": "00-222faed9a3a0e3c91b766ba772b23075-d9f7f454c56193bb-00", + "transfer-encoding": "chunked", + "via": "1.1 google", + "x-build-commit": "a8a32868d6b98462cb27eaaeaf00b4aaf8110e7b", + "x-build-time": "2026-07-29T17:02:39-07:00", + "x-cloud-trace-context": "222faed9a3a0e3c91b766ba772b23075/15706290870403830715", + "x-frame-options": "DENY", + "x-request-id": "56b16227-d494-42bb-abf8-9db1b40d9f6d" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 0, + "id": "99f1d8c4b0c43219", + "matchKey": "POST ollama.com/api/generate", + "recordedAt": "2026-07-30T14:17:18.001Z", + "request": { + "body": { + "kind": "json", + "value": { + "model": "gpt-oss:20b", + "options": { + "num_predict": 128, + "temperature": 0 + }, + "prompt": "Reply with exactly OK.", + "stream": false, + "think": false + } + }, + "headers": {}, + "method": "POST", + "url": "https://ollama.com/api/generate" + }, + "response": { + "body": { + "kind": "json", + "value": { + "created_at": "2026-07-30T14:17:18.050054605Z", + "done": true, + "done_reason": "stop", + "eval_count": 40, + "model": "gpt-oss:20b", + "prompt_eval_count": 67, + "response": "OK", + "thinking": "The user says: \"Reply with exactly OK.\" So we must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "total_duration": 947970164 + } + }, + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000", + "content-length": "320", + "content-type": "application/json", + "date": "Thu, 30 Jul 2026 14:17:18 GMT", + "server": "Google Frontend", + "set-cookie": "[REDACTED]", + "traceparent": "00-163ade5657ad33301b766ba772b2374e-25ddb75508a07d45-00", + "via": "1.1 google", + "x-build-commit": "a8a32868d6b98462cb27eaaeaf00b4aaf8110e7b", + "x-build-time": "2026-07-29T17:02:39-07:00", + "x-cloud-trace-context": "163ade5657ad33301b766ba772b2374e/2728538525101227333", + "x-frame-options": "DENY", + "x-request-id": "ccf7a2c3-d525-44ad-b0fa-cfa55a512239" + }, + "status": 200, + "statusText": "OK" + } + }, + { + "callIndex": 1, + "id": "54471dd63ef97520", + "matchKey": "POST ollama.com/api/generate", + "recordedAt": "2026-07-30T14:17:19.889Z", + "request": { + "body": { + "kind": "json", + "value": { + "model": "gpt-oss:20b", + "options": { + "num_predict": 128, + "temperature": 0 + }, + "prompt": "Reply with exactly STREAM.", + "stream": true, + "think": false + } + }, + "headers": {}, + "method": "POST", + "url": "https://ollama.com/api/generate" + }, + "response": { + "body": { + "kind": "text", + "value": "{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.599724432Z\",\"response\":\"\",\"thinking\":\"The\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.601082173Z\",\"response\":\"\",\"thinking\":\" user\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.601825504Z\",\"response\":\"\",\"thinking\":\" says\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.603175275Z\",\"response\":\"\",\"thinking\":\":\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.604566096Z\",\"response\":\"\",\"thinking\":\" \\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.605298347Z\",\"response\":\"\",\"thinking\":\"Reply\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.605716007Z\",\"response\":\"\",\"thinking\":\" with\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657090971Z\",\"response\":\"\",\"thinking\":\" exactly\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657144751Z\",\"response\":\"\",\"thinking\":\" STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.65718348Z\",\"response\":\"\",\"thinking\":\".\\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657229391Z\",\"response\":\"\",\"thinking\":\" So\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657271011Z\",\"response\":\"\",\"thinking\":\" the\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657398341Z\",\"response\":\"\",\"thinking\":\" assistant\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657473381Z\",\"response\":\"\",\"thinking\":\" should\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657507981Z\",\"response\":\"\",\"thinking\":\" reply\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657562171Z\",\"response\":\"\",\"thinking\":\" with\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657608451Z\",\"response\":\"\",\"thinking\":\" exactly\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657649941Z\",\"response\":\"\",\"thinking\":\" the\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.657975201Z\",\"response\":\"\",\"thinking\":\" word\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658022951Z\",\"response\":\"\",\"thinking\":\" \\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658079621Z\",\"response\":\"\",\"thinking\":\"STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658124381Z\",\"response\":\"\",\"thinking\":\"\\\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658174071Z\",\"response\":\"\",\"thinking\":\" No\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658224241Z\",\"response\":\"\",\"thinking\":\" other\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658275511Z\",\"response\":\"\",\"thinking\":\" characters\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658624692Z\",\"response\":\"\",\"thinking\":\",\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658671932Z\",\"response\":\"\",\"thinking\":\" no\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658725682Z\",\"response\":\"\",\"thinking\":\" punctuation\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.658773082Z\",\"response\":\"\",\"thinking\":\",\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.720832694Z\",\"response\":\"\",\"thinking\":\" no\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.721380494Z\",\"response\":\"\",\"thinking\":\" whitespace\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.721458554Z\",\"response\":\"\",\"thinking\":\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.721841315Z\",\"response\":\"\",\"thinking\":\" Just\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.721914635Z\",\"response\":\"\",\"thinking\":\" \\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.721965435Z\",\"response\":\"\",\"thinking\":\"STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722044485Z\",\"response\":\"\",\"thinking\":\"\\\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722130975Z\",\"response\":\"\",\"thinking\":\" Ensure\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722193605Z\",\"response\":\"\",\"thinking\":\" no\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722236845Z\",\"response\":\"\",\"thinking\":\" trailing\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722295275Z\",\"response\":\"\",\"thinking\":\" spaces\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722455815Z\",\"response\":\"\",\"thinking\":\" or\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722508865Z\",\"response\":\"\",\"thinking\":\" new\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722556475Z\",\"response\":\"\",\"thinking\":\"lines\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722604435Z\",\"response\":\"\",\"thinking\":\"?\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.722650915Z\",\"response\":\"\",\"thinking\":\" The\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.723595826Z\",\"response\":\"\",\"thinking\":\" instruction\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.723652066Z\",\"response\":\"\",\"thinking\":\":\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.723690436Z\",\"response\":\"\",\"thinking\":\" \\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.723731326Z\",\"response\":\"\",\"thinking\":\"Reply\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.723800286Z\",\"response\":\"\",\"thinking\":\" with\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.723847616Z\",\"response\":\"\",\"thinking\":\" exactly\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.724054236Z\",\"response\":\"\",\"thinking\":\" STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.840079094Z\",\"response\":\"\",\"thinking\":\".\\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.840384084Z\",\"response\":\"\",\"thinking\":\" Usually\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.840527404Z\",\"response\":\"\",\"thinking\":\" means\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.841446975Z\",\"response\":\"\",\"thinking\":\" output\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.842998126Z\",\"response\":\"\",\"thinking\":\" exactly\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.843058326Z\",\"response\":\"\",\"thinking\":\" the\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.843340866Z\",\"response\":\"\",\"thinking\":\" word\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.843399236Z\",\"response\":\"\",\"thinking\":\" STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.844343077Z\",\"response\":\"\",\"thinking\":\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.844413087Z\",\"response\":\"\",\"thinking\":\" So\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.846992499Z\",\"response\":\"\",\"thinking\":\" I\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.84708117Z\",\"response\":\"\",\"thinking\":\" will\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.848529051Z\",\"response\":\"\",\"thinking\":\" output\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.849822492Z\",\"response\":\"\",\"thinking\":\" \\\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.851341573Z\",\"response\":\"\",\"thinking\":\"STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.851916643Z\",\"response\":\"\",\"thinking\":\"\\\".\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.85971428Z\",\"response\":\"STREAM\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.85989297Z\",\"response\":\"\",\"done\":false}\n{\"model\":\"gpt-oss:20b\",\"created_at\":\"2026-07-30T14:17:19.937042165Z\",\"response\":\"\",\"done\":true,\"done_reason\":\"stop\",\"total_duration\":1721668042,\"prompt_eval_count\":67,\"eval_count\":79}\n" + }, + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000", + "content-type": "application/json", + "date": "Thu, 30 Jul 2026 14:17:19 GMT", + "server": "Google Frontend", + "set-cookie": "[REDACTED]", + "traceparent": "00-55d003c552c6028927ca4a55359c9d81-3b8fecdadd2af271-00", + "transfer-encoding": "chunked", + "via": "1.1 google", + "x-build-commit": "a8a32868d6b98462cb27eaaeaf00b4aaf8110e7b", + "x-build-time": "2026-07-29T17:02:39-07:00", + "x-cloud-trace-context": "55d003c552c6028927ca4a55359c9d81/4291909394664977009", + "x-frame-options": "DENY", + "x-request-id": "41903295-2fcb-4eb9-a612-8789e04d7cd7" + }, + "status": 200, + "statusText": "OK" + } + } + ], + "meta": { + "createdAt": "2026-07-30T13:54:11.198Z" + } +} diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.json b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.json new file mode 100644 index 000000000..1fe6de648 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.json @@ -0,0 +1,329 @@ +{ + "span_tree": [ + { + "name": "ollama-instrumentation-root", + "type": "task", + "children": [ + { + "name": "ollama-chat-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK", + "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 40, + "prompt_tokens": 67, + "tokens": 107 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "ollama-chat-stream-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "STREAM", + "reasoning": "The user says: \"Reply with exactly STREAM.\" So the assistant should reply with exactly the word \"STREAM\". No other characters, no punctuation, no whitespace. Just \"STREAM\".", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 47, + "prompt_tokens": 67, + "time_to_first_token": 0, + "tokens": 114 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "ollama-generate-operation", + "children": [ + { + "name": "ollama.generate", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK", + "reasoning": "The user says: \"Reply with exactly OK.\" So we must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 40, + "prompt_tokens": 67, + "tokens": 107 + } + } + ], + "metadata": { + "operation": "generate", + "testRunId": "" + } + }, + { + "name": "ollama-generate-stream-operation", + "children": [ + { + "name": "ollama.generate", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "STREAM", + "reasoning": "The user says: \"Reply with exactly STREAM.\" So the assistant should reply with exactly the word \"STREAM\". No other characters, no punctuation, no whitespace. Just \"STREAM\". Ensure no trailing spaces or newlines? The instruction: \"Reply with exactly STREAM.\" Usually means output exactly the word STREAM. So I will output \"STREAM\".", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 79, + "prompt_tokens": 67, + "time_to_first_token": 0, + "tokens": 146 + } + } + ], + "metadata": { + "operation": "generate-stream", + "testRunId": "" + } + }, + { + "name": "ollama-tool-call-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use get_temperature for Paris.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"city\":\"Paris\"}", + "name": "get_temperature" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "metadata": { + "model": "gpt-oss:20b", + "provider": "ollama", + "tools": [ + { + "function": { + "description": "Get the temperature for a city", + "name": "get_temperature", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_tokens": 5, + "prompt_tokens": 12, + "tokens": 17 + } + } + ], + "metadata": { + "operation": "tool-call", + "testRunId": "" + } + }, + { + "name": "ollama-embed-operation", + "children": [ + { + "name": "ollama.embed", + "type": "llm", + "children": [], + "input": [ + "braintrust tracing", + "ollama instrumentation" + ], + "output": { + "embedding_length": 4 + }, + "metadata": { + "model": "embeddinggemma", + "provider": "ollama" + }, + "metrics": { + "prompt_tokens": 4, + "tokens": 4 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "ollama-embeddings-operation", + "children": [ + { + "name": "ollama.embeddings", + "type": "llm", + "children": [], + "input": "braintrust tracing", + "output": { + "embedding_length": 3 + }, + "metadata": { + "model": "all-minilm", + "provider": "ollama" + } + } + ], + "metadata": { + "operation": "embeddings", + "testRunId": "" + } + }, + { + "name": "ollama-error-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Trigger an error.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-oss:20b", + "provider": "ollama" + }, + "error": "synthetic Ollama failure" + } + ], + "metadata": { + "operation": "error", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "ollama-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.txt b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.txt new file mode 100644 index 000000000..efdb2be28 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-auto.span-tree.txt @@ -0,0 +1,257 @@ +span_tree: +└── ollama-instrumentation-root [task] + metadata: { + "scenario": "ollama-instrumentation", + "testRunId": "" + } + ├── ollama-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK", + │ "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 40, + │ "prompt_tokens": 67, + │ "tokens": 107 + │ } + ├── ollama-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "STREAM", + │ "reasoning": "The user says: \"Reply with exactly STREAM.\" So the assistant should reply with exactly the word \"STREAM\". No other characters, no punctuation, no whitespace. Just \"STREAM\".", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 47, + │ "prompt_tokens": 67, + │ "time_to_first_token": 0, + │ "tokens": 114 + │ } + ├── ollama-generate-operation + │ metadata: { + │ "operation": "generate", + │ "testRunId": "" + │ } + │ └── ollama.generate [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK", + │ "reasoning": "The user says: \"Reply with exactly OK.\" So we must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 40, + │ "prompt_tokens": 67, + │ "tokens": 107 + │ } + ├── ollama-generate-stream-operation + │ metadata: { + │ "operation": "generate-stream", + │ "testRunId": "" + │ } + │ └── ollama.generate [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "STREAM", + │ "reasoning": "The user says: \"Reply with exactly STREAM.\" So the assistant should reply with exactly the word \"STREAM\". No other characters, no punctuation, no whitespace. Just \"STREAM\". Ensure no trailing spaces or newlines? The instruction: \"Reply with exactly STREAM.\" Usually means output exactly the word STREAM. So I will output \"STREAM\".", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 79, + │ "prompt_tokens": 67, + │ "time_to_first_token": 0, + │ "tokens": 146 + │ } + ├── ollama-tool-call-operation + │ metadata: { + │ "operation": "tool-call", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Use get_temperature for Paris.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "tool_calls", + │ "index": 0, + │ "message": { + │ "content": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"city\":\"Paris\"}", + │ "name": "get_temperature" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } + │ } + │ ] + │ metadata: { + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the temperature for a city", + │ "name": "get_temperature", + │ "parameters": { + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "prompt_tokens": 12, + │ "tokens": 17 + │ } + ├── ollama-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── ollama.embed [llm] + │ input: [ + │ "braintrust tracing", + │ "ollama instrumentation" + │ ] + │ output: { + │ "embedding_length": 4 + │ } + │ metadata: { + │ "model": "embeddinggemma", + │ "provider": "ollama" + │ } + │ metrics: { + │ "prompt_tokens": 4, + │ "tokens": 4 + │ } + ├── ollama-embeddings-operation + │ metadata: { + │ "operation": "embeddings", + │ "testRunId": "" + │ } + │ └── ollama.embeddings [llm] + │ input: "braintrust tracing" + │ output: { + │ "embedding_length": 3 + │ } + │ metadata: { + │ "model": "all-minilm", + │ "provider": "ollama" + │ } + └── ollama-error-operation + metadata: { + "operation": "error", + "testRunId": "" + } + └── ollama.chat [llm] + input: [ + { + "content": "Trigger an error.", + "role": "user" + } + ] + metadata: { + "model": "gpt-oss:20b", + "provider": "ollama" + } + error: "synthetic Ollama failure" diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.json b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.json new file mode 100644 index 000000000..e6959f455 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.json @@ -0,0 +1,329 @@ +{ + "span_tree": [ + { + "name": "ollama-instrumentation-root", + "type": "task", + "children": [ + { + "name": "ollama-chat-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK", + "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 40, + "prompt_tokens": 67, + "tokens": 107 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "ollama-chat-stream-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "STREAM", + "reasoning": "The user says: \"Reply with exactly STREAM.\" So we must output exactly the word \"STREAM\". No extra spaces, no newline? They said \"Reply with exactly STREAM.\" Usually we output the word. Probably just \"STREAM\". No punctuation. So output: STREAM.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 65, + "prompt_tokens": 67, + "time_to_first_token": 0, + "tokens": 132 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "ollama-generate-operation", + "children": [ + { + "name": "ollama.generate", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK", + "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 40, + "prompt_tokens": 67, + "tokens": 107 + } + } + ], + "metadata": { + "operation": "generate", + "testRunId": "" + } + }, + { + "name": "ollama-generate-stream-operation", + "children": [ + { + "name": "ollama.generate", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "STREAM", + "reasoning": "The user says: \"Reply with exactly STREAM.\" So we must output exactly the word \"STREAM\". No extra spaces, no newline? They said \"Reply with exactly STREAM.\" Usually we output the word. Probably just \"STREAM\". No punctuation. So final answer: STREAM.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 66, + "prompt_tokens": 67, + "time_to_first_token": 0, + "tokens": 133 + } + } + ], + "metadata": { + "operation": "generate-stream", + "testRunId": "" + } + }, + { + "name": "ollama-tool-call-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use get_temperature for Paris.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"city\":\"Paris\"}", + "name": "get_temperature" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "metadata": { + "model": "gpt-oss:20b", + "provider": "ollama", + "tools": [ + { + "function": { + "description": "Get the temperature for a city", + "name": "get_temperature", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_tokens": 5, + "prompt_tokens": 12, + "tokens": 17 + } + } + ], + "metadata": { + "operation": "tool-call", + "testRunId": "" + } + }, + { + "name": "ollama-embed-operation", + "children": [ + { + "name": "ollama.embed", + "type": "llm", + "children": [], + "input": [ + "braintrust tracing", + "ollama instrumentation" + ], + "output": { + "embedding_length": 4 + }, + "metadata": { + "model": "embeddinggemma", + "provider": "ollama" + }, + "metrics": { + "prompt_tokens": 4, + "tokens": 4 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "ollama-embeddings-operation", + "children": [ + { + "name": "ollama.embeddings", + "type": "llm", + "children": [], + "input": "braintrust tracing", + "output": { + "embedding_length": 3 + }, + "metadata": { + "model": "all-minilm", + "provider": "ollama" + } + } + ], + "metadata": { + "operation": "embeddings", + "testRunId": "" + } + }, + { + "name": "ollama-error-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Trigger an error.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-oss:20b", + "provider": "ollama" + }, + "error": "synthetic Ollama failure" + } + ], + "metadata": { + "operation": "error", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "ollama-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.txt b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.txt new file mode 100644 index 000000000..b75077038 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-auto.span-tree.txt @@ -0,0 +1,257 @@ +span_tree: +└── ollama-instrumentation-root [task] + metadata: { + "scenario": "ollama-instrumentation", + "testRunId": "" + } + ├── ollama-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK", + │ "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 40, + │ "prompt_tokens": 67, + │ "tokens": 107 + │ } + ├── ollama-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "STREAM", + │ "reasoning": "The user says: \"Reply with exactly STREAM.\" So we must output exactly the word \"STREAM\". No extra spaces, no newline? They said \"Reply with exactly STREAM.\" Usually we output the word. Probably just \"STREAM\". No punctuation. So output: STREAM.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 65, + │ "prompt_tokens": 67, + │ "time_to_first_token": 0, + │ "tokens": 132 + │ } + ├── ollama-generate-operation + │ metadata: { + │ "operation": "generate", + │ "testRunId": "" + │ } + │ └── ollama.generate [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK", + │ "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 40, + │ "prompt_tokens": 67, + │ "tokens": 107 + │ } + ├── ollama-generate-stream-operation + │ metadata: { + │ "operation": "generate-stream", + │ "testRunId": "" + │ } + │ └── ollama.generate [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "STREAM", + │ "reasoning": "The user says: \"Reply with exactly STREAM.\" So we must output exactly the word \"STREAM\". No extra spaces, no newline? They said \"Reply with exactly STREAM.\" Usually we output the word. Probably just \"STREAM\". No punctuation. So final answer: STREAM.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 66, + │ "prompt_tokens": 67, + │ "time_to_first_token": 0, + │ "tokens": 133 + │ } + ├── ollama-tool-call-operation + │ metadata: { + │ "operation": "tool-call", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Use get_temperature for Paris.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "tool_calls", + │ "index": 0, + │ "message": { + │ "content": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"city\":\"Paris\"}", + │ "name": "get_temperature" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } + │ } + │ ] + │ metadata: { + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the temperature for a city", + │ "name": "get_temperature", + │ "parameters": { + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "prompt_tokens": 12, + │ "tokens": 17 + │ } + ├── ollama-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── ollama.embed [llm] + │ input: [ + │ "braintrust tracing", + │ "ollama instrumentation" + │ ] + │ output: { + │ "embedding_length": 4 + │ } + │ metadata: { + │ "model": "embeddinggemma", + │ "provider": "ollama" + │ } + │ metrics: { + │ "prompt_tokens": 4, + │ "tokens": 4 + │ } + ├── ollama-embeddings-operation + │ metadata: { + │ "operation": "embeddings", + │ "testRunId": "" + │ } + │ └── ollama.embeddings [llm] + │ input: "braintrust tracing" + │ output: { + │ "embedding_length": 3 + │ } + │ metadata: { + │ "model": "all-minilm", + │ "provider": "ollama" + │ } + └── ollama-error-operation + metadata: { + "operation": "error", + "testRunId": "" + } + └── ollama.chat [llm] + input: [ + { + "content": "Trigger an error.", + "role": "user" + } + ] + metadata: { + "model": "gpt-oss:20b", + "provider": "ollama" + } + error: "synthetic Ollama failure" diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.json b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.json new file mode 100644 index 000000000..e6959f455 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.json @@ -0,0 +1,329 @@ +{ + "span_tree": [ + { + "name": "ollama-instrumentation-root", + "type": "task", + "children": [ + { + "name": "ollama-chat-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK", + "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 40, + "prompt_tokens": 67, + "tokens": 107 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "ollama-chat-stream-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "STREAM", + "reasoning": "The user says: \"Reply with exactly STREAM.\" So we must output exactly the word \"STREAM\". No extra spaces, no newline? They said \"Reply with exactly STREAM.\" Usually we output the word. Probably just \"STREAM\". No punctuation. So output: STREAM.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 65, + "prompt_tokens": 67, + "time_to_first_token": 0, + "tokens": 132 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "ollama-generate-operation", + "children": [ + { + "name": "ollama.generate", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK", + "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 40, + "prompt_tokens": 67, + "tokens": 107 + } + } + ], + "metadata": { + "operation": "generate", + "testRunId": "" + } + }, + { + "name": "ollama-generate-stream-operation", + "children": [ + { + "name": "ollama.generate", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "STREAM", + "reasoning": "The user says: \"Reply with exactly STREAM.\" So we must output exactly the word \"STREAM\". No extra spaces, no newline? They said \"Reply with exactly STREAM.\" Usually we output the word. Probably just \"STREAM\". No punctuation. So final answer: STREAM.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 66, + "prompt_tokens": 67, + "time_to_first_token": 0, + "tokens": 133 + } + } + ], + "metadata": { + "operation": "generate-stream", + "testRunId": "" + } + }, + { + "name": "ollama-tool-call-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use get_temperature for Paris.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"city\":\"Paris\"}", + "name": "get_temperature" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "metadata": { + "model": "gpt-oss:20b", + "provider": "ollama", + "tools": [ + { + "function": { + "description": "Get the temperature for a city", + "name": "get_temperature", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_tokens": 5, + "prompt_tokens": 12, + "tokens": 17 + } + } + ], + "metadata": { + "operation": "tool-call", + "testRunId": "" + } + }, + { + "name": "ollama-embed-operation", + "children": [ + { + "name": "ollama.embed", + "type": "llm", + "children": [], + "input": [ + "braintrust tracing", + "ollama instrumentation" + ], + "output": { + "embedding_length": 4 + }, + "metadata": { + "model": "embeddinggemma", + "provider": "ollama" + }, + "metrics": { + "prompt_tokens": 4, + "tokens": 4 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "ollama-embeddings-operation", + "children": [ + { + "name": "ollama.embeddings", + "type": "llm", + "children": [], + "input": "braintrust tracing", + "output": { + "embedding_length": 3 + }, + "metadata": { + "model": "all-minilm", + "provider": "ollama" + } + } + ], + "metadata": { + "operation": "embeddings", + "testRunId": "" + } + }, + { + "name": "ollama-error-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Trigger an error.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-oss:20b", + "provider": "ollama" + }, + "error": "synthetic Ollama failure" + } + ], + "metadata": { + "operation": "error", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "ollama-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.txt b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.txt new file mode 100644 index 000000000..b75077038 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-latest-wrapped.span-tree.txt @@ -0,0 +1,257 @@ +span_tree: +└── ollama-instrumentation-root [task] + metadata: { + "scenario": "ollama-instrumentation", + "testRunId": "" + } + ├── ollama-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK", + │ "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 40, + │ "prompt_tokens": 67, + │ "tokens": 107 + │ } + ├── ollama-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "STREAM", + │ "reasoning": "The user says: \"Reply with exactly STREAM.\" So we must output exactly the word \"STREAM\". No extra spaces, no newline? They said \"Reply with exactly STREAM.\" Usually we output the word. Probably just \"STREAM\". No punctuation. So output: STREAM.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 65, + │ "prompt_tokens": 67, + │ "time_to_first_token": 0, + │ "tokens": 132 + │ } + ├── ollama-generate-operation + │ metadata: { + │ "operation": "generate", + │ "testRunId": "" + │ } + │ └── ollama.generate [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK", + │ "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 40, + │ "prompt_tokens": 67, + │ "tokens": 107 + │ } + ├── ollama-generate-stream-operation + │ metadata: { + │ "operation": "generate-stream", + │ "testRunId": "" + │ } + │ └── ollama.generate [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "STREAM", + │ "reasoning": "The user says: \"Reply with exactly STREAM.\" So we must output exactly the word \"STREAM\". No extra spaces, no newline? They said \"Reply with exactly STREAM.\" Usually we output the word. Probably just \"STREAM\". No punctuation. So final answer: STREAM.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 66, + │ "prompt_tokens": 67, + │ "time_to_first_token": 0, + │ "tokens": 133 + │ } + ├── ollama-tool-call-operation + │ metadata: { + │ "operation": "tool-call", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Use get_temperature for Paris.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "tool_calls", + │ "index": 0, + │ "message": { + │ "content": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"city\":\"Paris\"}", + │ "name": "get_temperature" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } + │ } + │ ] + │ metadata: { + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the temperature for a city", + │ "name": "get_temperature", + │ "parameters": { + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "prompt_tokens": 12, + │ "tokens": 17 + │ } + ├── ollama-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── ollama.embed [llm] + │ input: [ + │ "braintrust tracing", + │ "ollama instrumentation" + │ ] + │ output: { + │ "embedding_length": 4 + │ } + │ metadata: { + │ "model": "embeddinggemma", + │ "provider": "ollama" + │ } + │ metrics: { + │ "prompt_tokens": 4, + │ "tokens": 4 + │ } + ├── ollama-embeddings-operation + │ metadata: { + │ "operation": "embeddings", + │ "testRunId": "" + │ } + │ └── ollama.embeddings [llm] + │ input: "braintrust tracing" + │ output: { + │ "embedding_length": 3 + │ } + │ metadata: { + │ "model": "all-minilm", + │ "provider": "ollama" + │ } + └── ollama-error-operation + metadata: { + "operation": "error", + "testRunId": "" + } + └── ollama.chat [llm] + input: [ + { + "content": "Trigger an error.", + "role": "user" + } + ] + metadata: { + "model": "gpt-oss:20b", + "provider": "ollama" + } + error: "synthetic Ollama failure" diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.json b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.json new file mode 100644 index 000000000..1fe6de648 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.json @@ -0,0 +1,329 @@ +{ + "span_tree": [ + { + "name": "ollama-instrumentation-root", + "type": "task", + "children": [ + { + "name": "ollama-chat-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK", + "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 40, + "prompt_tokens": 67, + "tokens": 107 + } + } + ], + "metadata": { + "operation": "chat", + "testRunId": "" + } + }, + { + "name": "ollama-chat-stream-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "STREAM", + "reasoning": "The user says: \"Reply with exactly STREAM.\" So the assistant should reply with exactly the word \"STREAM\". No other characters, no punctuation, no whitespace. Just \"STREAM\".", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 47, + "prompt_tokens": 67, + "time_to_first_token": 0, + "tokens": 114 + } + } + ], + "metadata": { + "operation": "chat-stream", + "testRunId": "" + } + }, + { + "name": "ollama-generate-operation", + "children": [ + { + "name": "ollama.generate", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "OK", + "reasoning": "The user says: \"Reply with exactly OK.\" So we must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 40, + "prompt_tokens": 67, + "tokens": 107 + } + } + ], + "metadata": { + "operation": "generate", + "testRunId": "" + } + }, + { + "name": "ollama-generate-stream-operation", + "children": [ + { + "name": "ollama.generate", + "type": "llm", + "children": [], + "input": [ + { + "content": "Reply with exactly STREAM.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "STREAM", + "reasoning": "The user says: \"Reply with exactly STREAM.\" So the assistant should reply with exactly the word \"STREAM\". No other characters, no punctuation, no whitespace. Just \"STREAM\". Ensure no trailing spaces or newlines? The instruction: \"Reply with exactly STREAM.\" Usually means output exactly the word STREAM. So I will output \"STREAM\".", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 128, + "model": "gpt-oss:20b", + "provider": "ollama", + "temperature": 0 + }, + "metrics": { + "completion_tokens": 79, + "prompt_tokens": 67, + "time_to_first_token": 0, + "tokens": 146 + } + } + ], + "metadata": { + "operation": "generate-stream", + "testRunId": "" + } + }, + { + "name": "ollama-tool-call-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Use get_temperature for Paris.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"city\":\"Paris\"}", + "name": "get_temperature" + }, + "id": "", + "type": "function" + } + ] + } + } + ], + "metadata": { + "model": "gpt-oss:20b", + "provider": "ollama", + "tools": [ + { + "function": { + "description": "Get the temperature for a city", + "name": "get_temperature", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] + }, + "metrics": { + "completion_tokens": 5, + "prompt_tokens": 12, + "tokens": 17 + } + } + ], + "metadata": { + "operation": "tool-call", + "testRunId": "" + } + }, + { + "name": "ollama-embed-operation", + "children": [ + { + "name": "ollama.embed", + "type": "llm", + "children": [], + "input": [ + "braintrust tracing", + "ollama instrumentation" + ], + "output": { + "embedding_length": 4 + }, + "metadata": { + "model": "embeddinggemma", + "provider": "ollama" + }, + "metrics": { + "prompt_tokens": 4, + "tokens": 4 + } + } + ], + "metadata": { + "operation": "embed", + "testRunId": "" + } + }, + { + "name": "ollama-embeddings-operation", + "children": [ + { + "name": "ollama.embeddings", + "type": "llm", + "children": [], + "input": "braintrust tracing", + "output": { + "embedding_length": 3 + }, + "metadata": { + "model": "all-minilm", + "provider": "ollama" + } + } + ], + "metadata": { + "operation": "embeddings", + "testRunId": "" + } + }, + { + "name": "ollama-error-operation", + "children": [ + { + "name": "ollama.chat", + "type": "llm", + "children": [], + "input": [ + { + "content": "Trigger an error.", + "role": "user" + } + ], + "metadata": { + "model": "gpt-oss:20b", + "provider": "ollama" + }, + "error": "synthetic Ollama failure" + } + ], + "metadata": { + "operation": "error", + "testRunId": "" + } + } + ], + "metadata": { + "scenario": "ollama-instrumentation", + "testRunId": "" + } + } + ] +} diff --git a/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.txt b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.txt new file mode 100644 index 000000000..efdb2be28 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/__snapshots__/ollama-v0.6-wrapped.span-tree.txt @@ -0,0 +1,257 @@ +span_tree: +└── ollama-instrumentation-root [task] + metadata: { + "scenario": "ollama-instrumentation", + "testRunId": "" + } + ├── ollama-chat-operation + │ metadata: { + │ "operation": "chat", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK", + │ "reasoning": "The user says: \"Reply with exactly OK.\" So I must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 40, + │ "prompt_tokens": 67, + │ "tokens": 107 + │ } + ├── ollama-chat-stream-operation + │ metadata: { + │ "operation": "chat-stream", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "STREAM", + │ "reasoning": "The user says: \"Reply with exactly STREAM.\" So the assistant should reply with exactly the word \"STREAM\". No other characters, no punctuation, no whitespace. Just \"STREAM\".", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 47, + │ "prompt_tokens": 67, + │ "time_to_first_token": 0, + │ "tokens": 114 + │ } + ├── ollama-generate-operation + │ metadata: { + │ "operation": "generate", + │ "testRunId": "" + │ } + │ └── ollama.generate [llm] + │ input: [ + │ { + │ "content": "Reply with exactly OK.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "OK", + │ "reasoning": "The user says: \"Reply with exactly OK.\" So we must reply with exactly \"OK\". No extra spaces, no punctuation. Just OK.", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 40, + │ "prompt_tokens": 67, + │ "tokens": 107 + │ } + ├── ollama-generate-stream-operation + │ metadata: { + │ "operation": "generate-stream", + │ "testRunId": "" + │ } + │ └── ollama.generate [llm] + │ input: [ + │ { + │ "content": "Reply with exactly STREAM.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "STREAM", + │ "reasoning": "The user says: \"Reply with exactly STREAM.\" So the assistant should reply with exactly the word \"STREAM\". No other characters, no punctuation, no whitespace. Just \"STREAM\". Ensure no trailing spaces or newlines? The instruction: \"Reply with exactly STREAM.\" Usually means output exactly the word STREAM. So I will output \"STREAM\".", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "max_tokens": 128, + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "temperature": 0 + │ } + │ metrics: { + │ "completion_tokens": 79, + │ "prompt_tokens": 67, + │ "time_to_first_token": 0, + │ "tokens": 146 + │ } + ├── ollama-tool-call-operation + │ metadata: { + │ "operation": "tool-call", + │ "testRunId": "" + │ } + │ └── ollama.chat [llm] + │ input: [ + │ { + │ "content": "Use get_temperature for Paris.", + │ "role": "user" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "tool_calls", + │ "index": 0, + │ "message": { + │ "content": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"city\":\"Paris\"}", + │ "name": "get_temperature" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ } + │ } + │ ] + │ metadata: { + │ "model": "gpt-oss:20b", + │ "provider": "ollama", + │ "tools": [ + │ { + │ "function": { + │ "description": "Get the temperature for a city", + │ "name": "get_temperature", + │ "parameters": { + │ "properties": { + │ "city": { + │ "type": "string" + │ } + │ }, + │ "required": [ + │ "city" + │ ], + │ "type": "object" + │ } + │ }, + │ "type": "function" + │ } + │ ] + │ } + │ metrics: { + │ "completion_tokens": 5, + │ "prompt_tokens": 12, + │ "tokens": 17 + │ } + ├── ollama-embed-operation + │ metadata: { + │ "operation": "embed", + │ "testRunId": "" + │ } + │ └── ollama.embed [llm] + │ input: [ + │ "braintrust tracing", + │ "ollama instrumentation" + │ ] + │ output: { + │ "embedding_length": 4 + │ } + │ metadata: { + │ "model": "embeddinggemma", + │ "provider": "ollama" + │ } + │ metrics: { + │ "prompt_tokens": 4, + │ "tokens": 4 + │ } + ├── ollama-embeddings-operation + │ metadata: { + │ "operation": "embeddings", + │ "testRunId": "" + │ } + │ └── ollama.embeddings [llm] + │ input: "braintrust tracing" + │ output: { + │ "embedding_length": 3 + │ } + │ metadata: { + │ "model": "all-minilm", + │ "provider": "ollama" + │ } + └── ollama-error-operation + metadata: { + "operation": "error", + "testRunId": "" + } + └── ollama.chat [llm] + input: [ + { + "content": "Trigger an error.", + "role": "user" + } + ] + metadata: { + "model": "gpt-oss:20b", + "provider": "ollama" + } + error: "synthetic Ollama failure" diff --git a/e2e/scenarios/ollama-instrumentation/assertions.ts b/e2e/scenarios/ollama-instrumentation/assertions.ts new file mode 100644 index 000000000..334d03a00 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/assertions.ts @@ -0,0 +1,227 @@ +import { beforeAll, describe, expect, test } from "vitest"; +import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; +import { + withScenarioHarness, + type ScenarioRunContext, +} from "../../helpers/scenario-harness"; +import { matchSpanTreeSnapshot } from "../../helpers/span-tree"; +import { findChildSpans, findLatestSpan } from "../../helpers/trace-selectors"; +import { + EMBEDDING_MODEL, + GENERATION_MODEL, + LEGACY_EMBEDDING_MODEL, + ROOT_NAME, + SCENARIO_NAME, +} from "./constants.mjs"; + +type RunOllamaScenario = (harness: { + runNodeScenarioDir: (options: { + entry: string; + nodeArgs: string[]; + runContext?: ScenarioRunContext; + scenarioDir: string; + timeoutMs: number; + }) => Promise; + runScenarioDir: (options: { + entry: string; + runContext?: ScenarioRunContext; + scenarioDir: string; + timeoutMs: number; + }) => Promise; +}) => Promise; + +function findProviderSpan( + events: CapturedLogEvent[], + operationName: string, + providerSpanName: string, +): CapturedLogEvent | undefined { + const operation = findLatestSpan(events, operationName); + const spans = findChildSpans(events, providerSpanName, operation?.span.id); + return ( + spans.find( + (candidate) => + candidate.output !== undefined || candidate.row.error !== undefined, + ) ?? spans[0] + ); +} + +function selectedEvents(events: CapturedLogEvent[]): CapturedLogEvent[] { + const operations = [ + ["ollama-chat-operation", "ollama.chat"], + ["ollama-chat-stream-operation", "ollama.chat"], + ["ollama-generate-operation", "ollama.generate"], + ["ollama-generate-stream-operation", "ollama.generate"], + ["ollama-tool-call-operation", "ollama.chat"], + ["ollama-embed-operation", "ollama.embed"], + ["ollama-embeddings-operation", "ollama.embeddings"], + ["ollama-error-operation", "ollama.chat"], + ] as const; + return [ + findLatestSpan(events, ROOT_NAME), + ...operations.flatMap(([operationName, providerSpanName]) => [ + findLatestSpan(events, operationName), + findProviderSpan(events, operationName, providerSpanName), + ]), + ].filter((event): event is CapturedLogEvent => event !== undefined); +} + +function expectTokenMetrics(event: CapturedLogEvent | undefined): void { + expect(event?.metrics).toMatchObject({ + prompt_tokens: expect.any(Number), + completion_tokens: expect.any(Number), + tokens: expect.any(Number), + }); +} + +export function defineOllamaInstrumentationAssertions(options: { + name: string; + runScenario: RunOllamaScenario; + snapshotName: string; + testFileUrl: string; + timeoutMs: number; +}): void { + const snapshotPath = resolveFileSnapshotPath( + options.testFileUrl, + `${options.snapshotName}.span-tree.json`, + ); + + describe(options.name, () => { + let events: CapturedLogEvent[] = []; + + beforeAll(async () => { + await withScenarioHarness(async (harness) => { + await options.runScenario(harness); + events = harness.events(); + }); + }, options.timeoutMs); + + test("captures chat and generate calls with streaming parity", () => { + for (const [operationName, spanName, streaming] of [ + ["ollama-chat-operation", "ollama.chat", false], + ["ollama-chat-stream-operation", "ollama.chat", true], + ["ollama-generate-operation", "ollama.generate", false], + ["ollama-generate-stream-operation", "ollama.generate", true], + ] as const) { + const span = findProviderSpan(events, operationName, spanName); + expect(span?.span.type).toBe("llm"); + expect(span?.row.metadata).toMatchObject({ + model: GENERATION_MODEL, + provider: "ollama", + }); + expect(span?.output).toEqual([ + expect.objectContaining({ + index: 0, + finish_reason: expect.any(String), + message: expect.objectContaining({ + role: "assistant", + content: expect.any(String), + }), + }), + ]); + expectTokenMetrics(span); + if (streaming) { + expect(span?.metrics?.time_to_first_token).toEqual( + expect.any(Number), + ); + } + expect(span?.row.context).toMatchObject({ + span_origin: { + instrumentation: { name: "ollama" }, + }, + }); + } + }); + + test("captures canonical tool definitions and calls", () => { + const span = findProviderSpan( + events, + "ollama-tool-call-operation", + "ollama.chat", + ); + const metadata = span?.row.metadata as + | Record + | undefined; + const output = span?.output as + | Array<{ + finish_reason?: string; + message?: { + tool_calls?: Array<{ + id?: string; + type?: string; + function?: { name?: string; arguments?: string }; + }>; + }; + }> + | undefined; + + expect(metadata?.tools).toEqual([ + { + type: "function", + function: { + name: "get_temperature", + description: "Get the temperature for a city", + parameters: { + type: "object", + properties: { city: { type: "string" } }, + required: ["city"], + }, + }, + }, + ]); + expect(output?.[0]?.finish_reason).toBe("tool_calls"); + expect(output?.[0]?.message?.tool_calls?.[0]).toMatchObject({ + id: "ollama_call_get_temperature_0", + type: "function", + function: { + name: "get_temperature", + arguments: '{"city":"Paris"}', + }, + }); + }); + + test("captures current and legacy embedding calls compactly", () => { + const embed = findProviderSpan( + events, + "ollama-embed-operation", + "ollama.embed", + ); + const embeddings = findProviderSpan( + events, + "ollama-embeddings-operation", + "ollama.embeddings", + ); + + expect(embed?.row.metadata).toMatchObject({ + model: EMBEDDING_MODEL, + provider: "ollama", + }); + expect(embed?.output).toEqual({ embedding_length: 4 }); + expect(embed?.metrics).toMatchObject({ + prompt_tokens: 4, + tokens: 4, + }); + + expect(embeddings?.row.metadata).toMatchObject({ + model: LEGACY_EMBEDDING_MODEL, + provider: "ollama", + }); + expect(embeddings?.output).toEqual({ embedding_length: 3 }); + }); + + test("captures provider errors without changing caller behavior", () => { + const span = findProviderSpan( + events, + "ollama-error-operation", + "ollama.chat", + ); + expect(span?.row.error).toBe("synthetic Ollama failure"); + }); + + test("captures the scenario root and matches the span tree", async () => { + const root = findLatestSpan(events, ROOT_NAME); + expect(root?.row.metadata).toMatchObject({ scenario: SCENARIO_NAME }); + await matchSpanTreeSnapshot(selectedEvents(events), snapshotPath); + }); + }); +} diff --git a/e2e/scenarios/ollama-instrumentation/constants.mjs b/e2e/scenarios/ollama-instrumentation/constants.mjs new file mode 100644 index 000000000..e69d98a71 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/constants.mjs @@ -0,0 +1,5 @@ +export const ROOT_NAME = "ollama-instrumentation-root"; +export const SCENARIO_NAME = "ollama-instrumentation"; +export const GENERATION_MODEL = "gpt-oss:20b"; +export const EMBEDDING_MODEL = "embeddinggemma"; +export const LEGACY_EMBEDDING_MODEL = "all-minilm"; diff --git a/e2e/scenarios/ollama-instrumentation/package.json b/e2e/scenarios/ollama-instrumentation/package.json new file mode 100644 index 000000000..e9a3b6934 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/package.json @@ -0,0 +1,18 @@ +{ + "name": "@braintrust/e2e-ollama-instrumentation", + "private": true, + "braintrustScenario": { + "bump": { + "dependencies": { + "ollama-v0-6-latest": { + "package": "ollama", + "range": "0.6" + } + } + } + }, + "dependencies": { + "ollama-v0-6": "npm:ollama@0.6.3", + "ollama-v0-6-latest": "npm:ollama@0.6.3" + } +} diff --git a/e2e/scenarios/ollama-instrumentation/pnpm-lock.yaml b/e2e/scenarios/ollama-instrumentation/pnpm-lock.yaml new file mode 100644 index 000000000..d90313710 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/pnpm-lock.yaml @@ -0,0 +1,32 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + ollama-v0-6: + specifier: npm:ollama@0.6.3 + version: ollama@0.6.3 + ollama-v0-6-latest: + specifier: npm:ollama@0.6.3 + version: ollama@0.6.3 + +packages: + + ollama@0.6.3: + resolution: {integrity: sha512-KEWEhIqE5wtfzEIZbDCLH51VFZ6Z3ZSa6sIOg/E/tBV8S51flyqBOXi+bRxlOYKDf8i327zG9eSTb8IJxvm3Zg==} + + whatwg-fetch@3.6.20: + resolution: {integrity: sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==} + +snapshots: + + ollama@0.6.3: + dependencies: + whatwg-fetch: 3.6.20 + + whatwg-fetch@3.6.20: {} diff --git a/e2e/scenarios/ollama-instrumentation/scenario.impl.mjs b/e2e/scenarios/ollama-instrumentation/scenario.impl.mjs new file mode 100644 index 000000000..4a0718764 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/scenario.impl.mjs @@ -0,0 +1,254 @@ +import { wrapOllama } from "braintrust"; +import { + collectAsync, + runOperation, + runTracedScenario, +} from "../../helpers/provider-runtime.mjs"; +import { + EMBEDDING_MODEL, + GENERATION_MODEL, + LEGACY_EMBEDDING_MODEL, + ROOT_NAME, + SCENARIO_NAME, +} from "./constants.mjs"; + +export const OLLAMA_SCENARIO_TIMEOUT_MS = 240_000; + +function requireOllamaConfig() { + const apiKey = process.env.OLLAMA_API_KEY; + const host = process.env.OLLAMA_HOST; + if (!apiKey) { + throw new Error("Expected OLLAMA_API_KEY to be set for e2e"); + } + if (!host) { + throw new Error("Expected OLLAMA_HOST to be set for e2e"); + } + return { apiKey, host }; +} + +function jsonResponse(body, status = 200) { + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); +} + +async function syntheticFetch(input, init) { + const url = String(input); + const body = + typeof init?.body === "string" && init.body.length > 0 + ? JSON.parse(init.body) + : {}; + + if (url.endsWith("/api/embed")) { + return jsonResponse({ + model: body.model, + embeddings: [ + [0.1, 0.2, 0.3, 0.4], + [0.5, 0.6, 0.7, 0.8], + ], + prompt_eval_count: 4, + total_duration: 100, + load_duration: 10, + }); + } + + if (url.endsWith("/api/embeddings")) { + return jsonResponse({ embedding: [0.1, 0.2, 0.3] }); + } + + if (url.endsWith("/api/chat") && Array.isArray(body.tools)) { + return jsonResponse({ + model: body.model, + message: { + role: "assistant", + content: "", + tool_calls: [ + { + function: { + name: "get_temperature", + arguments: { city: "Paris" }, + }, + }, + ], + }, + done: true, + done_reason: "stop", + prompt_eval_count: 12, + eval_count: 5, + }); + } + + return jsonResponse({ error: "synthetic Ollama failure" }, 500); +} + +function generationOptions() { + return { + temperature: 0, + num_predict: 128, + }; +} + +async function runOllamaInstrumentationScenario( + Ollama, + { decorateClient } = {}, +) { + const { apiKey, host } = requireOllamaConfig(); + const headers = { Authorization: `Bearer ${apiKey}` }; + const liveBaseClient = new Ollama({ host, headers }); + const syntheticBaseClient = new Ollama({ + host: "http://synthetic.ollama.test", + headers, + fetch: syntheticFetch, + }); + const liveClient = decorateClient + ? decorateClient(liveBaseClient) + : liveBaseClient; + const syntheticClient = decorateClient + ? decorateClient(syntheticBaseClient) + : syntheticBaseClient; + + await runTracedScenario({ + callback: async () => { + await runOperation("ollama-chat-operation", "chat", async () => { + const response = await liveClient.chat({ + model: GENERATION_MODEL, + messages: [{ role: "user", content: "Reply with exactly OK." }], + options: generationOptions(), + think: false, + }); + if (!response?.message?.content) { + throw new Error("Expected Ollama chat response content"); + } + }); + + await runOperation( + "ollama-chat-stream-operation", + "chat-stream", + async () => { + const stream = await liveClient.chat({ + model: GENERATION_MODEL, + messages: [{ role: "user", content: "Reply with exactly STREAM." }], + options: generationOptions(), + stream: true, + think: false, + }); + const chunks = await collectAsync(stream); + if (chunks.length === 0) { + throw new Error("Expected Ollama chat stream chunks"); + } + }, + ); + + await runOperation("ollama-generate-operation", "generate", async () => { + const response = await liveClient.generate({ + model: GENERATION_MODEL, + prompt: "Reply with exactly OK.", + options: generationOptions(), + think: false, + }); + if (!response?.response) { + throw new Error("Expected Ollama generate response content"); + } + }); + + await runOperation( + "ollama-generate-stream-operation", + "generate-stream", + async () => { + const stream = await liveClient.generate({ + model: GENERATION_MODEL, + prompt: "Reply with exactly STREAM.", + options: generationOptions(), + stream: true, + think: false, + }); + const chunks = await collectAsync(stream); + if (chunks.length === 0) { + throw new Error("Expected Ollama generate stream chunks"); + } + }, + ); + + await runOperation( + "ollama-tool-call-operation", + "tool-call", + async () => { + await syntheticClient.chat({ + model: GENERATION_MODEL, + messages: [ + { + role: "user", + content: "Use get_temperature for Paris.", + }, + ], + tools: [ + { + type: "function", + function: { + name: "get_temperature", + description: "Get the temperature for a city", + parameters: { + type: "object", + properties: { + city: { type: "string" }, + }, + required: ["city"], + }, + }, + }, + ], + }); + }, + ); + + await runOperation("ollama-embed-operation", "embed", async () => { + await syntheticClient.embed({ + model: EMBEDDING_MODEL, + input: ["braintrust tracing", "ollama instrumentation"], + }); + }); + + await runOperation( + "ollama-embeddings-operation", + "embeddings", + async () => { + await syntheticClient.embeddings({ + model: LEGACY_EMBEDDING_MODEL, + prompt: "braintrust tracing", + }); + }, + ); + + await runOperation("ollama-error-operation", "error", async () => { + try { + await syntheticClient.chat({ + model: GENERATION_MODEL, + messages: [{ role: "user", content: "Trigger an error." }], + }); + throw new Error("Expected synthetic Ollama request to fail"); + } catch (error) { + if ( + error instanceof Error && + error.message === "Expected synthetic Ollama request to fail" + ) { + throw error; + } + } + }); + }, + metadata: { scenario: SCENARIO_NAME }, + projectNameBase: "e2e-ollama-instrumentation", + rootName: ROOT_NAME, + }); +} + +export async function runWrappedOllamaInstrumentation(Ollama) { + await runOllamaInstrumentationScenario(Ollama, { + decorateClient: wrapOllama, + }); +} + +export async function runAutoOllamaInstrumentation(Ollama) { + await runOllamaInstrumentationScenario(Ollama); +} diff --git a/e2e/scenarios/ollama-instrumentation/scenario.mjs b/e2e/scenarios/ollama-instrumentation/scenario.mjs new file mode 100644 index 000000000..4d3ef4ae0 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/scenario.mjs @@ -0,0 +1,10 @@ +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoOllamaInstrumentation } from "./scenario.impl.mjs"; + +const ollamaPackageName = + process.env.OLLAMA_PACKAGE_NAME ?? "ollama-v0-6-latest"; + +runMain(async () => { + const { Ollama } = await import(ollamaPackageName); + await runAutoOllamaInstrumentation(Ollama); +}); diff --git a/e2e/scenarios/ollama-instrumentation/scenario.test.ts b/e2e/scenarios/ollama-instrumentation/scenario.test.ts new file mode 100644 index 000000000..525d0f504 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/scenario.test.ts @@ -0,0 +1,76 @@ +import { describe } from "vitest"; +import { + prepareScenarioDir, + readInstalledPackageVersion, + resolveScenarioDir, +} from "../../helpers/scenario-harness"; +import { defineOllamaInstrumentationAssertions } from "./assertions"; +import { OLLAMA_SCENARIO_TIMEOUT_MS } from "./scenario.impl.mjs"; + +const originalScenarioDir = resolveScenarioDir(import.meta.url); +const scenarioDir = await prepareScenarioDir({ + scenarioDir: originalScenarioDir, +}); +const ollamaScenarios = await Promise.all( + [ + { + dependencyName: "ollama-v0-6", + variantKey: "ollama-v0.6", + }, + { + dependencyName: "ollama-v0-6-latest", + variantKey: "ollama-v0.6-latest", + }, + ].map(async (scenario) => ({ + ...scenario, + version: await readInstalledPackageVersion( + scenarioDir, + scenario.dependencyName, + ), + })), +); + +describe.sequential("variants", () => { + for (const scenario of ollamaScenarios) { + describe.sequential(`ollama sdk ${scenario.version}`, () => { + defineOllamaInstrumentationAssertions({ + name: "wrapped instrumentation", + runScenario: async ({ runScenarioDir }) => { + await runScenarioDir({ + entry: "scenario.ts", + env: { OLLAMA_PACKAGE_NAME: scenario.dependencyName }, + runContext: { + variantKey: scenario.variantKey, + originalScenarioDir, + }, + scenarioDir, + timeoutMs: OLLAMA_SCENARIO_TIMEOUT_MS, + }); + }, + snapshotName: `${scenario.variantKey}-wrapped`, + testFileUrl: import.meta.url, + timeoutMs: OLLAMA_SCENARIO_TIMEOUT_MS, + }); + + defineOllamaInstrumentationAssertions({ + name: "auto-hook instrumentation", + runScenario: async ({ runNodeScenarioDir }) => { + await runNodeScenarioDir({ + entry: "scenario.mjs", + env: { OLLAMA_PACKAGE_NAME: scenario.dependencyName }, + nodeArgs: ["--import", "braintrust/hook.mjs"], + runContext: { + variantKey: scenario.variantKey, + originalScenarioDir, + }, + scenarioDir, + timeoutMs: OLLAMA_SCENARIO_TIMEOUT_MS, + }); + }, + snapshotName: `${scenario.variantKey}-auto`, + testFileUrl: import.meta.url, + timeoutMs: OLLAMA_SCENARIO_TIMEOUT_MS, + }); + }); + } +}); diff --git a/e2e/scenarios/ollama-instrumentation/scenario.ts b/e2e/scenarios/ollama-instrumentation/scenario.ts new file mode 100644 index 000000000..6fb1e33d4 --- /dev/null +++ b/e2e/scenarios/ollama-instrumentation/scenario.ts @@ -0,0 +1,10 @@ +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrappedOllamaInstrumentation } from "./scenario.impl.mjs"; + +const ollamaPackageName = + process.env.OLLAMA_PACKAGE_NAME ?? "ollama-v0-6-latest"; + +runMain(async () => { + const { Ollama } = await import(ollamaPackageName); + await runWrappedOllamaInstrumentation(Ollama); +}); diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.json index 448e3137e..04093af29 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.json @@ -17,7 +17,7 @@ "children": [], "input": [ { - "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nNew instrumentation e2e coverage must test both a pinned SDK dependency and a separately named latest dependency alias for every supported version line. Only the latest alias should participate in `test:e2e:bump`; add pinned and latest variants to the CI e2e summary.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", "role": "system" }, { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.txt index 1b306df81..29fdfabcc 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-auto-hook.span-tree.txt @@ -38,7 +38,7 @@ span_tree: ├── anthropic.messages.create [llm] │ input: [ │ { - │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nNew instrumentation e2e coverage must test both a pinned SDK dependency and a separately named latest dependency alias for every supported version line. Only the latest alias should participate in `test:e2e:bump`; add pinned and latest variants to the CI e2e summary.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", │ "role": "system" │ }, │ { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.json index 2e256c94b..dea4dfab8 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.json @@ -17,7 +17,7 @@ "children": [], "input": [ { - "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nNew instrumentation e2e coverage must test both a pinned SDK dependency and a separately named latest dependency alias for every supported version line. Only the latest alias should participate in `test:e2e:bump`; add pinned and latest variants to the CI e2e summary.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", "role": "system" }, { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.txt index ad01a3041..0c93a9b26 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.txt @@ -38,7 +38,7 @@ span_tree: ├── anthropic.messages.create [llm] │ input: [ │ { - │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nNew instrumentation e2e coverage must test both a pinned SDK dependency and a separately named latest dependency alias for every supported version line. Only the latest alias should participate in `test:e2e:bump`; add pinned and latest variants to the CI e2e summary.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", │ "role": "system" │ }, │ { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.json index 2e256c94b..dea4dfab8 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.json @@ -17,7 +17,7 @@ "children": [], "input": [ { - "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nNew instrumentation e2e coverage must test both a pinned SDK dependency and a separately named latest dependency alias for every supported version line. Only the latest alias should participate in `test:e2e:bump`; add pinned and latest variants to the CI e2e summary.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", "role": "system" }, { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.txt index ad01a3041..0c93a9b26 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.txt @@ -38,7 +38,7 @@ span_tree: ├── anthropic.messages.create [llm] │ input: [ │ { - │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nNew instrumentation e2e coverage must test both a pinned SDK dependency and a separately named latest dependency alias for every supported version line. Only the latest alias should participate in `test:e2e:bump`; add pinned and latest variants to the CI e2e summary.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", │ "role": "system" │ }, │ { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.json index 448e3137e..04093af29 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.json @@ -17,7 +17,7 @@ "children": [], "input": [ { - "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nNew instrumentation e2e coverage must test both a pinned SDK dependency and a separately named latest dependency alias for every supported version line. Only the latest alias should participate in `test:e2e:bump`; add pinned and latest variants to the CI e2e summary.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", "role": "system" }, { diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.txt index 1b306df81..29fdfabcc 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-wrapped.span-tree.txt @@ -38,7 +38,7 @@ span_tree: ├── anthropic.messages.create [llm] │ input: [ │ { - │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n/AGENTS.md\">\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\nInstrumentation patches generally do not need to be removed during teardown. Prefer leaving behavior-preserving patches installed when they are idempotent; do not add unpatching machinery by default.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nNew instrumentation e2e coverage must test both a pinned SDK dependency and a separately named latest dependency alias for every supported version line. Only the latest alias should participate in `test:e2e:bump`; add pinned and latest variants to the CI e2e summary.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", │ "role": "system" │ }, │ { diff --git a/js/src/auto-instrumentations/configs/all.ts b/js/src/auto-instrumentations/configs/all.ts index 905715e43..e6886ceb1 100644 --- a/js/src/auto-instrumentations/configs/all.ts +++ b/js/src/auto-instrumentations/configs/all.ts @@ -23,6 +23,7 @@ import { huggingFaceTransformersConfigs } from "./huggingface-transformers"; import { langchainConfigs } from "./langchain"; import { langSmithConfigs } from "./langsmith"; import { mistralConfigs } from "./mistral"; +import { ollamaConfigs } from "./ollama"; import { openAIAgentsCoreConfigs } from "./openai-agents"; import { openaiConfigs } from "./openai"; import { openAICodexConfigs } from "./openai-codex"; @@ -88,6 +89,7 @@ const defaultInstrumentationConfigGroups: readonly InstrumentationConfigGroup[] configs: openRouterAgentConfigs, }, { integrations: ["mistral"], configs: mistralConfigs }, + { integrations: ["ollama"], configs: ollamaConfigs }, { integrations: ["googleADK"], configs: googleADKConfigs }, { integrations: ["cohere"], configs: cohereConfigs }, { integrations: ["groq"], configs: groqConfigs }, diff --git a/js/src/auto-instrumentations/configs/ollama.ts b/js/src/auto-instrumentations/configs/ollama.ts new file mode 100644 index 000000000..1db36b8f0 --- /dev/null +++ b/js/src/auto-instrumentations/configs/ollama.ts @@ -0,0 +1,32 @@ +import type { InstrumentationConfig } from "../orchestrion-js"; +import { ollamaChannels } from "../../instrumentation/plugins/ollama-channels"; + +const methods = [ + ["chat", ollamaChannels.chat.channelName], + ["generate", ollamaChannels.generate.channelName], + ["embed", ollamaChannels.embed.channelName], + ["embeddings", ollamaChannels.embeddings.channelName], +] as const; + +// Ollama's Node entry points in dist/index.{mjs,cjs} subclass the Ollama class +// exported here. These methods are defined on that shared base class, so +// instrumenting these files covers both the Node and browser package exports. +const implementationFiles = ["dist/browser.mjs", "dist/browser.cjs"] as const; + +export const ollamaConfigs: InstrumentationConfig[] = [ + ...implementationFiles.flatMap((filePath) => + methods.map(([methodName, channelName]) => ({ + channelName, + module: { + name: "ollama", + versionRange: ">=0.6.0 <0.7.0", + filePath, + }, + functionQuery: { + className: "Ollama", + methodName, + kind: "Async" as const, + }, + })), + ), +]; diff --git a/js/src/auto-instrumentations/index.ts b/js/src/auto-instrumentations/index.ts index 49f728f82..a92465f4e 100644 --- a/js/src/auto-instrumentations/index.ts +++ b/js/src/auto-instrumentations/index.ts @@ -42,6 +42,7 @@ export { huggingFaceConfigs } from "./configs/huggingface"; export { openRouterAgentConfigs } from "./configs/openrouter-agent"; export { openRouterConfigs } from "./configs/openrouter"; export { mistralConfigs } from "./configs/mistral"; +export { ollamaConfigs } from "./configs/ollama"; export { googleADKConfigs } from "./configs/google-adk"; export { cloudflareAIChatConfigs } from "./configs/cloudflare-ai-chat"; export { cohereConfigs } from "./configs/cohere"; diff --git a/js/src/exports.ts b/js/src/exports.ts index 02104f8c2..9da607b55 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -221,6 +221,7 @@ export { wrapHuggingFaceTransformers } from "./wrappers/huggingface-transformers export { wrapOpenRouterAgent } from "./wrappers/openrouter-agent"; export { wrapOpenRouter } from "./wrappers/openrouter"; export { wrapMistral } from "./wrappers/mistral"; +export { wrapOllama } from "./wrappers/ollama"; export { wrapCohere } from "./wrappers/cohere"; export { wrapGroq } from "./wrappers/groq"; export { wrapBedrockRuntime } from "./wrappers/bedrock-runtime"; diff --git a/js/src/instrumentation/braintrust-plugin.test.ts b/js/src/instrumentation/braintrust-plugin.test.ts index 05c8d4838..cdbe15d5c 100644 --- a/js/src/instrumentation/braintrust-plugin.test.ts +++ b/js/src/instrumentation/braintrust-plugin.test.ts @@ -12,6 +12,7 @@ import { HuggingFaceTransformersPlugin } from "./plugins/huggingface-transformer import { OpenRouterAgentPlugin } from "./plugins/openrouter-agent-plugin"; import { OpenRouterPlugin } from "./plugins/openrouter-plugin"; import { MistralPlugin } from "./plugins/mistral-plugin"; +import { OllamaPlugin } from "./plugins/ollama-plugin"; import { CoherePlugin } from "./plugins/cohere-plugin"; import { GroqPlugin } from "./plugins/groq-plugin"; import { GitHubCopilotPlugin } from "./plugins/github-copilot-plugin"; @@ -90,6 +91,10 @@ vi.mock("./plugins/mistral-plugin", () => ({ MistralPlugin: createPluginClassMock(), })); +vi.mock("./plugins/ollama-plugin", () => ({ + OllamaPlugin: createPluginClassMock(), +})); + vi.mock("./plugins/cohere-plugin", () => ({ CoherePlugin: createPluginClassMock(), })); @@ -543,6 +548,18 @@ describe("BraintrustPlugin", () => { expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); }); + it("should not create Ollama plugin when ollama: false", () => { + const plugin = new BraintrustPlugin({ + integrations: { ollama: false }, + }); + plugin.enable(); + + expect(OllamaPlugin).not.toHaveBeenCalled(); + expect(OpenAIPlugin).toHaveBeenCalledTimes(1); + expect(AnthropicPlugin).toHaveBeenCalledTimes(1); + expect(MistralPlugin).toHaveBeenCalledTimes(1); + }); + it("should not create Cohere plugin when cohere: false", () => { const plugin = new BraintrustPlugin({ integrations: { cohere: false }, @@ -648,6 +665,7 @@ describe("BraintrustPlugin", () => { openrouter: false, openrouterAgent: false, mistral: false, + ollama: false, cohere: false, groq: false, gitHubCopilot: false, @@ -672,6 +690,7 @@ describe("BraintrustPlugin", () => { expect(OpenRouterPlugin).not.toHaveBeenCalled(); expect(OpenRouterAgentPlugin).not.toHaveBeenCalled(); expect(MistralPlugin).not.toHaveBeenCalled(); + expect(OllamaPlugin).not.toHaveBeenCalled(); expect(CoherePlugin).not.toHaveBeenCalled(); expect(GroqPlugin).not.toHaveBeenCalled(); expect(GitHubCopilotPlugin).not.toHaveBeenCalled(); @@ -881,6 +900,7 @@ describe("BraintrustPlugin", () => { const openRouterAgentMock = vi.mocked(OpenRouterAgentPlugin).mock .results[0].value; const mistralMock = vi.mocked(MistralPlugin).mock.results[0].value; + const ollamaMock = vi.mocked(OllamaPlugin).mock.results[0].value; const cohereMock = vi.mocked(CoherePlugin).mock.results[0].value; const groqMock = vi.mocked(GroqPlugin).mock.results[0].value; const piCodingAgentMock = @@ -903,6 +923,7 @@ describe("BraintrustPlugin", () => { expect(openRouterMock.enable).toHaveBeenCalledTimes(1); expect(openRouterAgentMock.enable).toHaveBeenCalledTimes(1); expect(mistralMock.enable).toHaveBeenCalledTimes(1); + expect(ollamaMock.enable).toHaveBeenCalledTimes(1); expect(cohereMock.enable).toHaveBeenCalledTimes(1); expect(groqMock.enable).toHaveBeenCalledTimes(1); expect(piCodingAgentMock.enable).toHaveBeenCalledTimes(1); @@ -935,6 +956,7 @@ describe("BraintrustPlugin", () => { const openRouterAgentMock = vi.mocked(OpenRouterAgentPlugin).mock .results[0].value; const mistralMock = vi.mocked(MistralPlugin).mock.results[0].value; + const ollamaMock = vi.mocked(OllamaPlugin).mock.results[0].value; const cohereMock = vi.mocked(CoherePlugin).mock.results[0].value; const groqMock = vi.mocked(GroqPlugin).mock.results[0].value; const piCodingAgentMock = @@ -959,6 +981,7 @@ describe("BraintrustPlugin", () => { expect(openRouterMock.disable).toHaveBeenCalledTimes(1); expect(openRouterAgentMock.disable).toHaveBeenCalledTimes(1); expect(mistralMock.disable).toHaveBeenCalledTimes(1); + expect(ollamaMock.disable).toHaveBeenCalledTimes(1); expect(cohereMock.disable).toHaveBeenCalledTimes(1); expect(groqMock.disable).toHaveBeenCalledTimes(1); expect(piCodingAgentMock.disable).toHaveBeenCalledTimes(1); @@ -1009,6 +1032,7 @@ describe("BraintrustPlugin", () => { expect(OpenRouterPlugin).not.toHaveBeenCalled(); expect(OpenRouterAgentPlugin).not.toHaveBeenCalled(); expect(MistralPlugin).not.toHaveBeenCalled(); + expect(OllamaPlugin).not.toHaveBeenCalled(); expect(CoherePlugin).not.toHaveBeenCalled(); expect(GroqPlugin).not.toHaveBeenCalled(); expect(PiCodingAgentPlugin).not.toHaveBeenCalled(); @@ -1035,6 +1059,7 @@ describe("BraintrustPlugin", () => { expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); expect(MistralPlugin).toHaveBeenCalledTimes(1); + expect(OllamaPlugin).toHaveBeenCalledTimes(1); expect(CoherePlugin).toHaveBeenCalledTimes(1); expect(GroqPlugin).toHaveBeenCalledTimes(1); expect(PiCodingAgentPlugin).toHaveBeenCalledTimes(1); @@ -1056,6 +1081,7 @@ describe("BraintrustPlugin", () => { openrouter: true, openrouterAgent: true, mistral: false, + ollama: false, cohere: false, groq: true, langchain: true, @@ -1089,6 +1115,7 @@ describe("BraintrustPlugin", () => { expect(groqMock.disable).toHaveBeenCalledTimes(1); expect(langChainMock.disable).toHaveBeenCalledTimes(1); expect(MistralPlugin).not.toHaveBeenCalled(); + expect(OllamaPlugin).not.toHaveBeenCalled(); expect(CoherePlugin).not.toHaveBeenCalled(); }); }); diff --git a/js/src/instrumentation/braintrust-plugin.ts b/js/src/instrumentation/braintrust-plugin.ts index ec88ddec2..201d39970 100644 --- a/js/src/instrumentation/braintrust-plugin.ts +++ b/js/src/instrumentation/braintrust-plugin.ts @@ -12,6 +12,7 @@ import { HuggingFaceTransformersPlugin } from "./plugins/huggingface-transformer import { OpenRouterAgentPlugin } from "./plugins/openrouter-agent-plugin"; import { OpenRouterPlugin } from "./plugins/openrouter-plugin"; import { MistralPlugin } from "./plugins/mistral-plugin"; +import { OllamaPlugin } from "./plugins/ollama-plugin"; import { GoogleADKPlugin } from "./plugins/google-adk-plugin"; import { CoherePlugin } from "./plugins/cohere-plugin"; import { GroqPlugin } from "./plugins/groq-plugin"; @@ -43,6 +44,7 @@ export interface BraintrustPluginConfig { * - HuggingFace Inference SDK * - LangChain.js and LangGraph * - Mistral SDK + * - Ollama SDK * - Cohere SDK * * The plugin is automatically enabled when the Braintrust library is loaded. @@ -64,6 +66,7 @@ export class BraintrustPlugin extends BasePlugin { private openRouterPlugin: OpenRouterPlugin | null = null; private openRouterAgentPlugin: OpenRouterAgentPlugin | null = null; private mistralPlugin: MistralPlugin | null = null; + private ollamaPlugin: OllamaPlugin | null = null; private googleADKPlugin: GoogleADKPlugin | null = null; private coherePlugin: CoherePlugin | null = null; private groqPlugin: GroqPlugin | null = null; @@ -156,6 +159,11 @@ export class BraintrustPlugin extends BasePlugin { this.mistralPlugin.enable(); } + if (integrations.ollama !== false) { + this.ollamaPlugin = new OllamaPlugin(); + this.ollamaPlugin.enable(); + } + // Enable Google ADK integration (default: true) if (integrations.googleADK !== false) { this.googleADKPlugin = new GoogleADKPlugin(); @@ -301,6 +309,11 @@ export class BraintrustPlugin extends BasePlugin { this.mistralPlugin = null; } + if (this.ollamaPlugin) { + this.ollamaPlugin.disable(); + this.ollamaPlugin = null; + } + if (this.googleADKPlugin) { this.googleADKPlugin.disable(); this.googleADKPlugin = null; diff --git a/js/src/instrumentation/config.ts b/js/src/instrumentation/config.ts index c51a5b48d..03ecf52b7 100644 --- a/js/src/instrumentation/config.ts +++ b/js/src/instrumentation/config.ts @@ -17,6 +17,7 @@ export interface InstrumentationIntegrationsConfig { openrouter?: boolean; openrouterAgent?: boolean; mistral?: boolean; + ollama?: boolean; cohere?: boolean; groq?: boolean; bedrock?: boolean; @@ -93,6 +94,7 @@ const envIntegrationAliases: Record< openrouteragent: "openrouterAgent", "openrouter-agent": "openrouterAgent", mistral: "mistral", + ollama: "ollama", googleadk: "googleADK", "google-adk": "googleADK", cohere: "cohere", @@ -140,6 +142,7 @@ export function getDefaultInstrumentationIntegrations(): Record< openrouter: true, openrouterAgent: true, mistral: true, + ollama: true, cohere: true, groq: true, bedrock: true, diff --git a/js/src/instrumentation/plugins/instrumentation-names.test.ts b/js/src/instrumentation/plugins/instrumentation-names.test.ts index 2e342f9b2..fbb0b0561 100644 --- a/js/src/instrumentation/plugins/instrumentation-names.test.ts +++ b/js/src/instrumentation/plugins/instrumentation-names.test.ts @@ -23,6 +23,7 @@ import { huggingFaceChannels } from "./huggingface-channels"; import { langChainChannels } from "./langchain-channels"; import { langSmithChannels } from "./langsmith-channels"; import { mistralChannels } from "./mistral-channels"; +import { ollamaChannels } from "./ollama-channels"; import { openAIAgentsCoreChannels } from "./openai-agents-channels"; import { openAIChannels } from "./openai-channels"; import { openAICodexChannels } from "./openai-codex-channels"; @@ -61,6 +62,7 @@ describe("built-in instrumentation provenance names", () => { [langChainChannels.configure, INSTRUMENTATION_NAMES.LANGCHAIN], [langSmithChannels.createRun, INSTRUMENTATION_NAMES.LANGSMITH], [mistralChannels.chatComplete, INSTRUMENTATION_NAMES.MISTRAL], + [ollamaChannels.chat, INSTRUMENTATION_NAMES.OLLAMA], [ openAIAgentsCoreChannels.onTraceStart, INSTRUMENTATION_NAMES.OPENAI_AGENTS, diff --git a/js/src/instrumentation/plugins/ollama-channels.ts b/js/src/instrumentation/plugins/ollama-channels.ts new file mode 100644 index 000000000..70b7d8533 --- /dev/null +++ b/js/src/instrumentation/plugins/ollama-channels.ts @@ -0,0 +1,47 @@ +import { INSTRUMENTATION_NAMES } from "../../span-origin"; +import type { + OllamaChatRequest, + OllamaChatResponse, + OllamaChatResult, + OllamaEmbedRequest, + OllamaEmbedResponse, + OllamaEmbeddingsRequest, + OllamaEmbeddingsResponse, + OllamaGenerateRequest, + OllamaGenerateResponse, + OllamaGenerateResult, +} from "../../vendor-sdk-types/ollama"; +import { channel, defineChannels } from "../core/channel-definitions"; + +export const ollamaChannels = defineChannels( + "ollama", + { + chat: channel< + [OllamaChatRequest], + OllamaChatResult, + Record, + OllamaChatResponse + >({ + channelName: "chat", + kind: "async", + }), + generate: channel< + [OllamaGenerateRequest], + OllamaGenerateResult, + Record, + OllamaGenerateResponse + >({ + channelName: "generate", + kind: "async", + }), + embed: channel<[OllamaEmbedRequest], OllamaEmbedResponse>({ + channelName: "embed", + kind: "async", + }), + embeddings: channel<[OllamaEmbeddingsRequest], OllamaEmbeddingsResponse>({ + channelName: "embeddings", + kind: "async", + }), + }, + { instrumentationName: INSTRUMENTATION_NAMES.OLLAMA }, +); diff --git a/js/src/instrumentation/plugins/ollama-plugin.test.ts b/js/src/instrumentation/plugins/ollama-plugin.test.ts new file mode 100644 index 000000000..4c9690d63 --- /dev/null +++ b/js/src/instrumentation/plugins/ollama-plugin.test.ts @@ -0,0 +1,321 @@ +import { describe, expect, it } from "vitest"; +import { + aggregateOllamaChatChunks, + aggregateOllamaGenerateChunks, + extractOllamaChatInput, + extractOllamaChatOutput, + extractOllamaEmbedOutput, + extractOllamaEmbeddingsOutput, + extractOllamaGenerateInput, + extractOllamaMetrics, +} from "./ollama-plugin"; + +describe("Ollama instrumentation extraction", () => { + it("normalizes chat inputs, tools, and supported request metadata", () => { + const result = extractOllamaChatInput([ + { + model: "gpt-oss:20b", + messages: [ + { role: "system", content: "Be concise." }, + { role: "user", content: "What is the weather?" }, + { + role: "assistant", + content: "", + tool_calls: [ + { + function: { + name: "get_weather", + arguments: { city: "Paris" }, + }, + }, + ], + }, + { + role: "tool", + tool_name: "get_weather", + content: '{"temperature":18}', + }, + ], + tools: [ + { + type: "function", + function: { + name: "get_weather", + description: "Get weather", + parameters: { + type: "object", + properties: { city: { type: "string" } }, + }, + }, + }, + ], + format: "json", + options: { + temperature: 0, + top_p: 0.8, + num_predict: 32, + frequency_penalty: 0.1, + presence_penalty: 0.2, + stop: ["DONE"], + top_k: 10, + }, + stream: true, + }, + ]); + + expect(result.input).toEqual([ + { role: "system", content: "Be concise." }, + { role: "user", content: "What is the weather?" }, + { + role: "assistant", + content: null, + tool_calls: [ + { + id: "ollama_call_get_weather_0", + type: "function", + function: { + name: "get_weather", + arguments: '{"city":"Paris"}', + }, + }, + ], + }, + { + role: "tool", + tool_call_id: "ollama_call_get_weather_0", + content: '{"temperature":18}', + }, + ]); + expect(result.metadata).toEqual({ + provider: "ollama", + model: "gpt-oss:20b", + temperature: 0, + top_p: 0.8, + max_tokens: 32, + frequency_penalty: 0.1, + presence_penalty: 0.2, + stop: ["DONE"], + response_format: "json", + tools: [ + { + type: "function", + function: { + name: "get_weather", + description: "Get weather", + parameters: { + type: "object", + properties: { city: { type: "string" } }, + }, + }, + }, + ], + }); + }); + + it("converts multimodal chat images into attachments", () => { + const result = extractOllamaChatInput([ + { + model: "llava", + messages: [ + { + role: "user", + content: "Describe this image.", + images: ["data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB"], + }, + ], + }, + ]); + + const content = ( + result.input as Array<{ + content: Array<{ + type: string; + image_url?: { url?: { reference?: unknown } }; + }>; + }> + )[0].content; + expect(content[0]).toEqual({ + type: "text", + text: "Describe this image.", + }); + expect(content[1]).toMatchObject({ + type: "image_url", + image_url: { + url: { + reference: { + type: "braintrust_attachment", + content_type: "image/png", + }, + }, + }, + }); + }); + + it("normalizes chat output and tool calls as OpenAI choices", () => { + expect( + extractOllamaChatOutput({ + model: "gpt-oss:20b", + message: { + role: "assistant", + content: "", + thinking: "I should call the tool.", + tool_calls: [ + { + function: { + name: "get_weather", + arguments: { city: "Paris" }, + }, + }, + ], + }, + done: true, + done_reason: "stop", + }), + ).toEqual([ + { + index: 0, + finish_reason: "tool_calls", + message: { + role: "assistant", + content: null, + reasoning: "I should call the tool.", + tool_calls: [ + { + id: "ollama_call_get_weather_0", + type: "function", + function: { + name: "get_weather", + arguments: '{"city":"Paris"}', + }, + }, + ], + }, + }, + ]); + }); + + it("normalizes raw generation input as canonical messages", () => { + expect( + extractOllamaGenerateInput([ + { + model: "gpt-oss:20b", + system: "Be concise.", + prompt: "Say OK.", + options: { temperature: 0, num_predict: 8 }, + }, + ]), + ).toEqual({ + input: [ + { role: "system", content: "Be concise." }, + { role: "user", content: "Say OK." }, + ], + metadata: { + provider: "ollama", + model: "gpt-oss:20b", + temperature: 0, + max_tokens: 8, + }, + }); + }); + + it("extracts token metrics and ignores invalid counts", () => { + expect( + extractOllamaMetrics({ + prompt_eval_count: 7, + eval_count: 3, + }), + ).toEqual({ + prompt_tokens: 7, + completion_tokens: 3, + tokens: 10, + }); + expect( + extractOllamaMetrics({ + prompt_eval_count: -1, + eval_count: Number.NaN, + }), + ).toEqual({}); + }); + + it("aggregates chat and generation streams into final outputs", () => { + const chat = aggregateOllamaChatChunks( + [ + { + model: "gpt-oss:20b", + message: { role: "assistant", content: "Hello" }, + done: false, + }, + { + model: "gpt-oss:20b", + message: { role: "assistant", content: "!" }, + done: true, + done_reason: "stop", + prompt_eval_count: 5, + eval_count: 2, + }, + ], + undefined, + undefined, + 1, + ); + expect(chat.output).toEqual([ + { + index: 0, + finish_reason: "stop", + message: { role: "assistant", content: "Hello!" }, + }, + ]); + expect(chat.metrics).toMatchObject({ + prompt_tokens: 5, + completion_tokens: 2, + tokens: 7, + }); + expect(chat.metadata).toEqual({ model: "gpt-oss:20b" }); + + const generation = aggregateOllamaGenerateChunks( + [ + { model: "gpt-oss:20b", response: "O", done: false }, + { + model: "gpt-oss:20b", + response: "K", + done: true, + done_reason: "stop", + prompt_eval_count: 4, + eval_count: 2, + }, + ], + undefined, + undefined, + 1, + ); + expect(generation.output).toEqual([ + { + index: 0, + finish_reason: "stop", + message: { role: "assistant", content: "OK" }, + }, + ]); + expect(generation.metrics).toMatchObject({ + prompt_tokens: 4, + completion_tokens: 2, + tokens: 6, + }); + }); + + it("summarizes current and legacy embedding responses", () => { + expect( + extractOllamaEmbedOutput({ + model: "embeddinggemma", + embeddings: [ + [0.1, 0.2, 0.3], + [0.4, 0.5, 0.6], + ], + }), + ).toEqual({ embedding_length: 3 }); + expect( + extractOllamaEmbeddingsOutput({ + embedding: [0.1, 0.2], + }), + ).toEqual({ embedding_length: 2 }); + }); +}); diff --git a/js/src/instrumentation/plugins/ollama-plugin.ts b/js/src/instrumentation/plugins/ollama-plugin.ts new file mode 100644 index 000000000..be3eec9ee --- /dev/null +++ b/js/src/instrumentation/plugins/ollama-plugin.ts @@ -0,0 +1,560 @@ +import { SpanTypeAttribute, isObject } from "../../../util/index"; +import { processInputAttachments } from "../../wrappers/attachment-utils"; +import type { + OllamaChatResponse, + OllamaEmbedResponse, + OllamaEmbeddingsResponse, + OllamaGenerateResponse, + OllamaMessage, + OllamaTool, + OllamaToolCall, + OllamaUsageResponse, +} from "../../vendor-sdk-types/ollama"; +import { BasePlugin } from "../core"; +import { + traceAsyncChannel, + traceStreamingChannel, + unsubscribeAll, +} from "../core/channel-tracing"; +import { ollamaChannels } from "./ollama-channels"; + +export class OllamaPlugin extends BasePlugin { + protected onEnable(): void { + this.unsubscribers.push( + traceStreamingChannel(ollamaChannels.chat, { + name: "ollama.chat", + type: SpanTypeAttribute.LLM, + extractInput: extractOllamaChatInput, + extractOutput: extractOllamaChatOutput, + extractMetadata: extractOllamaResponseMetadata, + extractMetrics: extractOllamaMetrics, + aggregateChunks: aggregateOllamaChatChunks, + }), + traceStreamingChannel(ollamaChannels.generate, { + name: "ollama.generate", + type: SpanTypeAttribute.LLM, + extractInput: extractOllamaGenerateInput, + extractOutput: extractOllamaGenerateOutput, + extractMetadata: extractOllamaResponseMetadata, + extractMetrics: extractOllamaMetrics, + aggregateChunks: aggregateOllamaGenerateChunks, + }), + traceAsyncChannel(ollamaChannels.embed, { + name: "ollama.embed", + type: SpanTypeAttribute.LLM, + extractInput: extractOllamaEmbedInput, + extractOutput: extractOllamaEmbedOutput, + extractMetadata: extractOllamaResponseMetadata, + extractMetrics: extractOllamaMetrics, + }), + traceAsyncChannel(ollamaChannels.embeddings, { + name: "ollama.embeddings", + type: SpanTypeAttribute.LLM, + extractInput: extractOllamaEmbeddingsInput, + extractOutput: extractOllamaEmbeddingsOutput, + extractMetrics: () => ({}), + }), + ); + } + + protected onDisable(): void { + this.unsubscribers = unsubscribeAll(this.unsubscribers); + } +} + +function getRequestArg(args: unknown): Record | undefined { + const values = + Array.isArray(args) || isArrayLike(args) ? Array.from(args) : [args]; + return values.find((value) => isObject(value)) as + | Record + | undefined; +} + +function isArrayLike(value: unknown): value is ArrayLike { + return ( + isObject(value) && + typeof value.length === "number" && + Number.isInteger(value.length) && + value.length >= 0 + ); +} + +function isNonNegativeNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value) && value >= 0; +} + +function normalizeFinishReason( + response: { done_reason?: string } | undefined, + hasToolCalls = false, +): string { + if (hasToolCalls) { + return "tool_calls"; + } + return response?.done_reason || "stop"; +} + +function stringifyArguments(value: unknown): string { + if (typeof value === "string") { + return value; + } + try { + return JSON.stringify(value ?? {}); + } catch { + return String(value); + } +} + +function syntheticToolCallId(name: string, index: number): string { + const normalizedName = name.replace(/[^a-zA-Z0-9_-]/g, "_") || "tool"; + return `ollama_call_${normalizedName}_${index}`; +} + +function normalizeToolCall( + toolCall: OllamaToolCall, + index: number, +): Record | undefined { + const name = toolCall.function?.name; + if (typeof name !== "string" || name.length === 0) { + return undefined; + } + + return { + id: + typeof toolCall.id === "string" && toolCall.id.length > 0 + ? toolCall.id + : syntheticToolCallId(name, index), + type: "function", + function: { + name, + arguments: stringifyArguments(toolCall.function?.arguments), + }, + }; +} + +function normalizeToolCalls( + toolCalls: OllamaToolCall[] | undefined, +): Record[] { + if (!Array.isArray(toolCalls)) { + return []; + } + + return toolCalls.flatMap((toolCall, index) => { + const normalized = normalizeToolCall(toolCall, index); + return normalized ? [normalized] : []; + }); +} + +function imageBytes(value: unknown): Uint8Array | undefined { + if (value instanceof Uint8Array) { + return value; + } + if (value instanceof ArrayBuffer) { + return new Uint8Array(value); + } + if (typeof value !== "string" || value.startsWith("data:")) { + return undefined; + } + try { + const decoded = atob(value.slice(0, 24)); + return Uint8Array.from(decoded, (character) => character.charCodeAt(0)); + } catch { + return undefined; + } +} + +function inferImageMediaType(value: unknown): string | undefined { + if (typeof value === "string") { + const dataUrlType = value.match(/^data:(image\/[^;]+);base64,/i)?.[1]; + if (dataUrlType) { + return dataUrlType; + } + } + + const bytes = imageBytes(value); + if (!bytes) { + return undefined; + } + if ( + bytes[0] === 0x89 && + bytes[1] === 0x50 && + bytes[2] === 0x4e && + bytes[3] === 0x47 + ) { + return "image/png"; + } + if (bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) { + return "image/jpeg"; + } + const signature = String.fromCharCode(...bytes.slice(0, 12)); + if (signature.startsWith("GIF87a") || signature.startsWith("GIF89a")) { + return "image/gif"; + } + if (signature.startsWith("RIFF") && signature.slice(8, 12) === "WEBP") { + return "image/webp"; + } + return undefined; +} + +function normalizeTextAndImages( + content: string | null, + images: unknown[] | undefined, +): { + content: unknown; + unrecognizedImages?: unknown[]; +} { + if (!images || images.length === 0) { + return { content }; + } + + const imageParts: Record[] = []; + const unrecognizedImages: unknown[] = []; + for (const image of images) { + const mediaType = inferImageMediaType(image); + if (!mediaType) { + unrecognizedImages.push(image); + continue; + } + const processed = processInputAttachments({ + type: "image", + image, + mediaType, + }); + imageParts.push({ + type: "image_url", + image_url: { url: processed.image }, + }); + } + + return { + content: + imageParts.length > 0 + ? [...(content ? [{ type: "text", text: content }] : []), ...imageParts] + : content, + ...(unrecognizedImages.length > 0 ? { unrecognizedImages } : {}), + }; +} + +function normalizeMessage( + message: OllamaMessage, +): Record | undefined { + if (typeof message.role !== "string") { + return undefined; + } + + const toolCalls = normalizeToolCalls(message.tool_calls); + if (message.role === "tool") { + const toolName = + typeof message.tool_name === "string" && message.tool_name.length > 0 + ? message.tool_name + : "tool"; + return { + role: "tool", + tool_call_id: syntheticToolCallId(toolName, 0), + content: typeof message.content === "string" ? message.content : "", + }; + } + + const normalizedContent = normalizeTextAndImages( + typeof message.content === "string" + ? message.content || (toolCalls.length > 0 ? null : "") + : toolCalls.length > 0 + ? null + : "", + message.images, + ); + return { + role: message.role, + content: normalizedContent.content, + ...(typeof message.thinking === "string" && message.thinking.length > 0 + ? { reasoning: message.thinking } + : {}), + ...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}), + ...(normalizedContent.unrecognizedImages + ? { images: normalizedContent.unrecognizedImages } + : {}), + }; +} + +function normalizeMessages(messages: unknown): Record[] { + if (!Array.isArray(messages)) { + return []; + } + return messages.flatMap((message) => { + if (!isObject(message)) { + return []; + } + const normalized = normalizeMessage(message as OllamaMessage); + return normalized ? [normalized] : []; + }); +} + +function normalizeToolDefinition( + tool: OllamaTool, +): Record | undefined { + const fn = tool.function; + if (!fn || typeof fn.name !== "string" || fn.name.length === 0) { + return undefined; + } + + return { + type: "function", + function: { + name: fn.name, + ...(typeof fn.description === "string" + ? { description: fn.description } + : {}), + ...(isObject(fn.parameters) ? { parameters: fn.parameters } : {}), + }, + }; +} + +function extractToolsMetadata(tools: unknown): Record { + if (!Array.isArray(tools)) { + return {}; + } + const normalized = tools.flatMap((tool) => { + if (!isObject(tool)) { + return []; + } + const definition = normalizeToolDefinition(tool as OllamaTool); + return definition ? [definition] : []; + }); + return normalized.length > 0 ? { tools: normalized } : {}; +} + +function extractOptionsMetadata(options: unknown): Record { + if (!isObject(options)) { + return {}; + } + + const metadata: Record = {}; + const mappings = [ + ["temperature", "temperature"], + ["top_p", "top_p"], + ["num_predict", "max_tokens"], + ["frequency_penalty", "frequency_penalty"], + ["presence_penalty", "presence_penalty"], + ["stop", "stop"], + ] as const; + for (const [source, target] of mappings) { + if (options[source] !== undefined) { + metadata[target] = options[source]; + } + } + return metadata; +} + +function extractRequestMetadata( + request: Record | undefined, +): Record { + return { + provider: "ollama", + ...(typeof request?.model === "string" ? { model: request.model } : {}), + ...extractOptionsMetadata(request?.options), + ...(request?.format !== undefined + ? { response_format: request.format } + : {}), + }; +} + +export function extractOllamaChatInput(args: unknown): { + input: unknown; + metadata: Record; +} { + const request = getRequestArg(args); + return { + input: normalizeMessages(request?.messages), + metadata: { + ...extractRequestMetadata(request), + ...extractToolsMetadata(request?.tools), + }, + }; +} + +export function extractOllamaGenerateInput(args: unknown): { + input: unknown; + metadata: Record; +} { + const request = getRequestArg(args); + const normalizedPrompt = normalizeTextAndImages( + typeof request?.prompt === "string" ? request.prompt : "", + Array.isArray(request?.images) ? request.images : undefined, + ); + const input = [ + ...(typeof request?.system === "string" + ? [{ role: "system", content: request.system }] + : []), + { + role: "user", + content: normalizedPrompt.content, + ...(normalizedPrompt.unrecognizedImages + ? { images: normalizedPrompt.unrecognizedImages } + : {}), + }, + ]; + + return { + input, + metadata: extractRequestMetadata(request), + }; +} + +function extractOllamaEmbedInput(args: unknown): { + input: unknown; + metadata: Record; +} { + const request = getRequestArg(args); + return { + input: request?.input, + metadata: extractRequestMetadata(request), + }; +} + +function extractOllamaEmbeddingsInput(args: unknown): { + input: unknown; + metadata: Record; +} { + const request = getRequestArg(args); + return { + input: request?.prompt, + metadata: extractRequestMetadata(request), + }; +} + +export function extractOllamaChatOutput(result: OllamaChatResponse): unknown { + if (!isObject(result) || !isObject(result.message)) { + return undefined; + } + + const message = normalizeMessage(result.message as OllamaMessage); + if (!message) { + return undefined; + } + const toolCalls = Array.isArray(message.tool_calls) ? message.tool_calls : []; + + return [ + { + index: 0, + finish_reason: normalizeFinishReason(result, toolCalls.length > 0), + message, + }, + ]; +} + +function extractOllamaGenerateOutput(result: OllamaGenerateResponse): unknown { + if (!isObject(result) || typeof result.response !== "string") { + return undefined; + } + + return [ + { + index: 0, + finish_reason: normalizeFinishReason(result), + message: { + role: "assistant", + content: result.response, + ...(typeof result.thinking === "string" && result.thinking.length > 0 + ? { reasoning: result.thinking } + : {}), + }, + }, + ]; +} + +export function extractOllamaEmbedOutput(result: OllamaEmbedResponse): unknown { + const embedding = Array.isArray(result?.embeddings) + ? result.embeddings[0] + : undefined; + return Array.isArray(embedding) + ? { embedding_length: embedding.length } + : undefined; +} + +export function extractOllamaEmbeddingsOutput( + result: OllamaEmbeddingsResponse, +): unknown { + return Array.isArray(result?.embedding) + ? { embedding_length: result.embedding.length } + : undefined; +} + +function extractOllamaResponseMetadata( + result: OllamaUsageResponse, +): Record | undefined { + return typeof result?.model === "string" + ? { model: result.model } + : undefined; +} + +export function extractOllamaMetrics( + result: OllamaUsageResponse, +): Record { + const metrics: Record = {}; + const promptTokens = result?.prompt_eval_count; + const completionTokens = result?.eval_count; + + if (isNonNegativeNumber(promptTokens)) { + metrics.prompt_tokens = promptTokens; + } + if (isNonNegativeNumber(completionTokens)) { + metrics.completion_tokens = completionTokens; + } + if ( + isNonNegativeNumber(promptTokens) || + isNonNegativeNumber(completionTokens) + ) { + metrics.tokens = (promptTokens ?? 0) + (completionTokens ?? 0); + } + return metrics; +} + +export function aggregateOllamaChatChunks( + chunks: OllamaChatResponse[], + _result?: unknown, + _event?: unknown, + _startTime?: number, +): { + output: unknown; + metrics: Record; + metadata?: Record; +} { + const last = chunks.at(-1); + const message: OllamaMessage = { + role: "assistant", + content: chunks.map((chunk) => chunk.message?.content ?? "").join(""), + thinking: chunks.map((chunk) => chunk.message?.thinking ?? "").join(""), + tool_calls: chunks.flatMap((chunk) => chunk.message?.tool_calls ?? []), + }; + const response: OllamaChatResponse = { + ...last, + message, + }; + return { + output: extractOllamaChatOutput(response), + metrics: extractOllamaMetrics(last ?? {}), + metadata: extractOllamaResponseMetadata(last ?? {}), + }; +} + +export function aggregateOllamaGenerateChunks( + chunks: OllamaGenerateResponse[], + _result?: unknown, + _event?: unknown, + _startTime?: number, +): { + output: unknown; + metrics: Record; + metadata?: Record; +} { + const last = chunks.at(-1); + const response: OllamaGenerateResponse = { + ...last, + response: chunks.map((chunk) => chunk.response ?? "").join(""), + thinking: chunks.map((chunk) => chunk.thinking ?? "").join(""), + }; + return { + output: extractOllamaGenerateOutput(response), + metrics: extractOllamaMetrics(last ?? {}), + metadata: extractOllamaResponseMetadata(last ?? {}), + }; +} diff --git a/js/src/span-origin.ts b/js/src/span-origin.ts index 0fd2f5fc8..cf41cb14b 100644 --- a/js/src/span-origin.ts +++ b/js/src/span-origin.ts @@ -29,6 +29,7 @@ export const INSTRUMENTATION_NAMES = { LANGSMITH: "langsmith", MASTRA: "mastra", MISTRAL: "mistral", + OLLAMA: "ollama", OPENAI: "openai", OPENAI_AGENTS: "openai-agents", OPENAI_CODEX: "openai-codex", diff --git a/js/src/vendor-sdk-types/ollama.ts b/js/src/vendor-sdk-types/ollama.ts new file mode 100644 index 000000000..1438dda39 --- /dev/null +++ b/js/src/vendor-sdk-types/ollama.ts @@ -0,0 +1,117 @@ +// ollama types + +export interface OllamaOptions { + num_predict?: number; + temperature?: number; + top_p?: number; + frequency_penalty?: number; + presence_penalty?: number; + stop?: string[]; +} + +export interface OllamaToolCall { + id?: string; + function?: { + name?: string; + arguments?: unknown; + }; +} + +export interface OllamaTool { + type?: string; + function?: { + name?: string; + description?: string; + parameters?: Record; + }; +} + +export interface OllamaMessage { + role?: string; + content?: string; + thinking?: string; + images?: unknown[]; + tool_calls?: OllamaToolCall[]; + tool_name?: string; +} + +export interface OllamaChatRequest { + model: string; + messages?: OllamaMessage[]; + stream?: boolean; + format?: string | Record; + tools?: OllamaTool[]; + options?: OllamaOptions; +} + +export interface OllamaGenerateRequest { + model: string; + prompt: string; + suffix?: string; + system?: string; + stream?: boolean; + format?: string | Record; + images?: unknown[]; + options?: OllamaOptions; +} + +export interface OllamaEmbedRequest { + model: string; + input: string | string[]; + dimensions?: number; + options?: OllamaOptions; +} + +export interface OllamaEmbeddingsRequest { + model: string; + prompt: string; + options?: OllamaOptions; +} + +export interface OllamaUsageResponse { + model?: string; + prompt_eval_count?: number; + eval_count?: number; +} + +export interface OllamaChatResponse extends OllamaUsageResponse { + message?: OllamaMessage; + done?: boolean; + done_reason?: string; +} + +export interface OllamaGenerateResponse extends OllamaUsageResponse { + response?: string; + thinking?: string; + done?: boolean; + done_reason?: string; +} + +export interface OllamaEmbedResponse extends OllamaUsageResponse { + embeddings?: number[][]; +} + +export interface OllamaEmbeddingsResponse { + embedding?: number[]; +} + +export interface OllamaAsyncIterator extends AsyncIterable { + abort?: () => void; +} + +export type OllamaChatResult = + | OllamaChatResponse + | OllamaAsyncIterator; + +export type OllamaGenerateResult = + | OllamaGenerateResponse + | OllamaAsyncIterator; + +export interface OllamaClient { + chat?: (request: OllamaChatRequest) => Promise; + generate?: (request: OllamaGenerateRequest) => Promise; + embed?: (request: OllamaEmbedRequest) => Promise; + embeddings?: ( + request: OllamaEmbeddingsRequest, + ) => Promise; +} diff --git a/js/src/wrappers/ollama.test.ts b/js/src/wrappers/ollama.test.ts new file mode 100644 index 000000000..c9355592d --- /dev/null +++ b/js/src/wrappers/ollama.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it, vi } from "vitest"; +import { ollamaChannels } from "../instrumentation/plugins/ollama-channels"; +import type { OllamaClient } from "../vendor-sdk-types/ollama"; +import { wrapOllama } from "./ollama"; + +describe("wrapOllama", () => { + it("emits channel events for every supported generation surface", async () => { + const client: OllamaClient = { + chat: vi.fn(async () => ({ + message: { role: "assistant", content: "OK" }, + done: true, + })), + generate: vi.fn(async () => ({ response: "OK", done: true })), + embed: vi.fn(async () => ({ embeddings: [[0.1, 0.2]] })), + embeddings: vi.fn(async () => ({ embedding: [0.1, 0.2] })), + }; + const chatSpy = vi + .spyOn(ollamaChannels.chat, "tracePromise") + .mockImplementation((fn) => fn()); + const generateSpy = vi + .spyOn(ollamaChannels.generate, "tracePromise") + .mockImplementation((fn) => fn()); + const embedSpy = vi + .spyOn(ollamaChannels.embed, "tracePromise") + .mockImplementation((fn) => fn()); + const embeddingsSpy = vi + .spyOn(ollamaChannels.embeddings, "tracePromise") + .mockImplementation((fn) => fn()); + + const wrapped = wrapOllama(client); + await wrapped.chat?.({ + model: "gpt-oss:20b", + messages: [{ role: "user", content: "Say OK." }], + }); + await wrapped.generate?.({ + model: "gpt-oss:20b", + prompt: "Say OK.", + }); + await wrapped.embed?.({ model: "embeddinggemma", input: "hello" }); + await wrapped.embeddings?.({ + model: "all-minilm", + prompt: "hello", + }); + + expect(chatSpy).toHaveBeenCalledOnce(); + expect(generateSpy).toHaveBeenCalledOnce(); + expect(embedSpy).toHaveBeenCalledOnce(); + expect(embeddingsSpy).toHaveBeenCalledOnce(); + expect(wrapOllama(client)).toBe(wrapped); + expect(wrapOllama(wrapped)).toBe(wrapped); + }); + + it("returns unsupported objects unchanged", () => { + const client = {}; + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + + expect(wrapOllama(client)).toBe(client); + expect(warn).toHaveBeenCalledWith( + "Unsupported Ollama library. Not wrapping.", + ); + }); +}); diff --git a/js/src/wrappers/ollama.ts b/js/src/wrappers/ollama.ts new file mode 100644 index 000000000..a573b0446 --- /dev/null +++ b/js/src/wrappers/ollama.ts @@ -0,0 +1,118 @@ +import { ollamaChannels } from "../instrumentation/plugins/ollama-channels"; +import type { + OllamaChatRequest, + OllamaChatResult, + OllamaClient, + OllamaEmbedRequest, + OllamaEmbedResponse, + OllamaEmbeddingsRequest, + OllamaEmbeddingsResponse, + OllamaGenerateRequest, + OllamaGenerateResult, +} from "../vendor-sdk-types/ollama"; + +/** + * Wrap an Ollama client so generation and embedding calls emit Braintrust + * diagnostics-channel events. + */ +export function wrapOllama(ollama: T): T { + if (isSupportedOllamaClient(ollama)) { + return ollamaProxy(ollama) as T; + } + + // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. + console.warn("Unsupported Ollama library. Not wrapping."); + return ollama; +} + +const ollamaProxyCache = new WeakMap(); + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function hasFunction(value: Record, name: string): boolean { + return typeof value[name] === "function"; +} + +function isSupportedOllamaClient(value: unknown): value is OllamaClient { + return ( + isRecord(value) && + ["chat", "generate", "embed", "embeddings"].some((name) => + hasFunction(value, name), + ) + ); +} + +function ollamaProxy(ollama: OllamaClient): OllamaClient { + const cached = ollamaProxyCache.get(ollama); + if (cached) { + return cached; + } + + const proxy = new Proxy(ollama, { + get(target, prop, receiver) { + switch (prop) { + case "chat": + return typeof target.chat === "function" + ? wrapChat(target.chat.bind(target)) + : target.chat; + case "generate": + return typeof target.generate === "function" + ? wrapGenerate(target.generate.bind(target)) + : target.generate; + case "embed": + return typeof target.embed === "function" + ? wrapEmbed(target.embed.bind(target)) + : target.embed; + case "embeddings": + return typeof target.embeddings === "function" + ? wrapEmbeddings(target.embeddings.bind(target)) + : target.embeddings; + default: + return Reflect.get(target, prop, receiver); + } + }, + }); + ollamaProxyCache.set(ollama, proxy); + ollamaProxyCache.set(proxy, proxy); + return proxy; +} + +function wrapChat( + chat: (request: OllamaChatRequest) => Promise, +): NonNullable { + return (request) => + ollamaChannels.chat.tracePromise(() => chat(request), { + arguments: [request], + }); +} + +function wrapGenerate( + generate: (request: OllamaGenerateRequest) => Promise, +): NonNullable { + return (request) => + ollamaChannels.generate.tracePromise(() => generate(request), { + arguments: [request], + }); +} + +function wrapEmbed( + embed: (request: OllamaEmbedRequest) => Promise, +): NonNullable { + return (request) => + ollamaChannels.embed.tracePromise(() => embed(request), { + arguments: [request], + }); +} + +function wrapEmbeddings( + embeddings: ( + request: OllamaEmbeddingsRequest, + ) => Promise, +): NonNullable { + return (request) => + ollamaChannels.embeddings.tracePromise(() => embeddings(request), { + arguments: [request], + }); +}