diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-with-pii.mjs index 4f7d6be20fc3..b1e6b92ab7c6 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-with-pii.mjs @@ -1,6 +1,10 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; +if (process.env.USE_ORCHESTRION) { + Sentry.experimentalUseDiagnosticsChannelInjection(); +} + Sentry.init({ dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument.mjs index dfe8424bb811..647af1ccdc13 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument.mjs @@ -1,6 +1,10 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; +if (process.env.USE_ORCHESTRION) { + Sentry.experimentalUseDiagnosticsChannelInjection(); +} + Sentry.init({ dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts index 860003189e73..7ecc51bdbffc 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts @@ -23,6 +23,7 @@ import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/r describe.each([ ['6', {}, '^6.0.0'], ['7', {}, '7.0.0-beta.179'], + ['7', { USE_ORCHESTRION: '1' }, '7.0.0-beta.179'], ])('Vercel AI integration (version %s, env: %o)', (_, env: { USE_ORCHESTRION?: string }, vercelAiVersion: string) => { afterAll(() => { cleanupChildProcesses(); diff --git a/packages/node/src/index.ts b/packages/node/src/index.ts index df90fd85e755..d149ed5c8b78 100644 --- a/packages/node/src/index.ts +++ b/packages/node/src/index.ts @@ -25,6 +25,7 @@ export { genericPoolIntegration } from './integrations/tracing/genericPool'; export { dataloaderIntegration } from './integrations/tracing/dataloader'; export { amqplibIntegration } from './integrations/tracing/amqplib'; export { vercelAIIntegration } from './integrations/tracing/vercelai'; +export { vercelAiChannelIntegration } from './integrations/tracing/vercelai/channel-integration'; export { openAIIntegration } from './integrations/tracing/openai'; export { anthropicAIIntegration } from './integrations/tracing/anthropic-ai'; export { googleGenAIIntegration } from './integrations/tracing/google-genai'; diff --git a/packages/node/src/integrations/tracing/vercelai/channel-integration.ts b/packages/node/src/integrations/tracing/vercelai/channel-integration.ts new file mode 100644 index 000000000000..08ef6f2a89fa --- /dev/null +++ b/packages/node/src/integrations/tracing/vercelai/channel-integration.ts @@ -0,0 +1,45 @@ +import type { Client, IntegrationFn } from '@sentry/core'; +import { defineIntegration } from '@sentry/core'; +import { tracingChannel as otelTracingChannel } from '@sentry/opentelemetry/tracing-channel'; +import { subscribeVercelAiOrchestrionChannels, subscribeVercelAiTracingChannel } from '@sentry/server-utils'; +import { INTEGRATION_NAME } from './constants'; + +// In channel-based (orchestrion) mode we emit our own `gen_ai.*` spans from the +// diagnostics channels. The `ai` SDK still emits its own native OpenTelemetry +// spans whenever the user enables `experimental_telemetry`, which would be +// duplicates. Every native `ai` span carries an `ai.operationId` attribute +// (e.g. `ai.generateText`, `ai.generateText.doGenerate`, `ai.toolCall`) at span +// start, whereas our channel spans use `vercel.ai.operationId` — so we drop the +// native ones up front via `ignoreSpans`, before any vercel-ai processing runs. +const NATIVE_VERCEL_AI_SPANS = { attributes: { 'ai.operationId': /^ai\./ } }; + +const _vercelAiChannelIntegration = (() => { + return { + name: INTEGRATION_NAME, + beforeSetup(client: Client) { + // Ensure we drop spans emitted by ai v6 or below + // To avoid double-instrumentation - in this scenario, we only want to rely on our own spans + const options = client.getOptions(); + options.ignoreSpans = [...(options.ignoreSpans || []), NATIVE_VERCEL_AI_SPANS]; + }, + setupOnce() { + // v7: subscribe to the `ai` SDK's native `ai:telemetry` tracing channel. + // No-op on versions that don't publish to it, so it is always safe to call. + // The factory needs the Sentry OTel context manager, which `initOpenTelemetry()` + // registers after `setupOnce`, so defer a tick. + void Promise.resolve().then(() => subscribeVercelAiTracingChannel(otelTracingChannel)); + + // v6: there is no native channel — orchestrion injects `orchestrion:ai:*` + // channels which this adapter consumes via the same span core. Inert when + // orchestrion isn't active or on `ai` >= 7. + subscribeVercelAiOrchestrionChannels(); + }, + }; +}) satisfies IntegrationFn; + +/** + * Adds Sentry tracing instrumentation for the [ai](https://www.npmjs.com/package/ai) library via + * diagnostics channels — the channel-based counterpart to the OpenTelemetry `vercelAIIntegration`, + * used when diagnostics-channel injection is opted into. + */ +export const vercelAiChannelIntegration = defineIntegration(_vercelAiChannelIntegration); diff --git a/packages/node/src/sdk/experimentalUseDiagnosticsChannelInjection.ts b/packages/node/src/sdk/experimentalUseDiagnosticsChannelInjection.ts index d51f2d86a610..10451964d1b1 100644 --- a/packages/node/src/sdk/experimentalUseDiagnosticsChannelInjection.ts +++ b/packages/node/src/sdk/experimentalUseDiagnosticsChannelInjection.ts @@ -2,6 +2,7 @@ import { mysqlChannelIntegration, detectOrchestrionSetup } from '@sentry/server- import { registerDiagnosticsChannelInjection } from '@sentry/server-utils/orchestrion/register'; import type { DiagnosticsChannelInjection } from './diagnosticsChannelInjection'; import { setDiagnosticsChannelInjectionLoader } from './diagnosticsChannelInjection'; +import { vercelAiChannelIntegration } from '../integrations/tracing/vercelai/channel-integration'; /** * EXPERIMENTAL: opt into diagnostics-channel-based auto-instrumentation. @@ -38,8 +39,8 @@ import { setDiagnosticsChannelInjectionLoader } from './diagnosticsChannelInject export function experimentalUseDiagnosticsChannelInjection(): void { setDiagnosticsChannelInjectionLoader( (): DiagnosticsChannelInjection => ({ - integrations: [mysqlChannelIntegration()], - replacedOtelIntegrationNames: ['Mysql'], + integrations: [mysqlChannelIntegration(), vercelAiChannelIntegration()], + replacedOtelIntegrationNames: ['Mysql', 'VercelAI'], register: registerDiagnosticsChannelInjection, detect: detectOrchestrionSetup, }), diff --git a/packages/server-utils/src/index.ts b/packages/server-utils/src/index.ts index 624790edda9a..03b9cbe1d104 100644 --- a/packages/server-utils/src/index.ts +++ b/packages/server-utils/src/index.ts @@ -24,3 +24,4 @@ export type { RedisTracingChannelSubscribers, } from './redis/redis-dc-subscriber'; export { subscribeVercelAiTracingChannel } from './vercel-ai/vercel-ai-dc-subscriber'; +export { subscribeVercelAiOrchestrionChannels } from './vercel-ai/vercel-ai-orchestrion-v6-subscriber'; diff --git a/packages/server-utils/src/orchestrion/channels.ts b/packages/server-utils/src/orchestrion/channels.ts index 28dcf0c33468..fd30b7433fd3 100644 --- a/packages/server-utils/src/orchestrion/channels.ts +++ b/packages/server-utils/src/orchestrion/channels.ts @@ -13,6 +13,18 @@ */ export const CHANNELS = { MYSQL_QUERY: 'orchestrion:mysql:query', + // Vercel AI (`ai`) v6: orchestrion injects these so the same channel-based + // integration that consumes `ai`'s native `ai:telemetry` channel (v7) can + // also instrument v6. Each maps to a top-level function in `ai`'s bundle. + VERCEL_AI_GENERATE_TEXT: 'orchestrion:ai:generateText', + VERCEL_AI_STREAM_TEXT: 'orchestrion:ai:streamText', + VERCEL_AI_EMBED: 'orchestrion:ai:embed', + VERCEL_AI_EXECUTE_TOOL_CALL: 'orchestrion:ai:executeToolCall', + // `resolveLanguageModel` is the single chokepoint every model call flows + // through; we wrap it to monkey-patch `doGenerate`/`doStream` on the returned + // model (the model-call site itself is an inline call with no injectable + // definition). + VERCEL_AI_RESOLVE_LANGUAGE_MODEL: 'orchestrion:ai:resolveLanguageModel', } as const; export type ChannelName = (typeof CHANNELS)[keyof typeof CHANNELS]; diff --git a/packages/server-utils/src/orchestrion/config.ts b/packages/server-utils/src/orchestrion/config.ts index 35b326fb8eb1..3cf2eee4c15e 100644 --- a/packages/server-utils/src/orchestrion/config.ts +++ b/packages/server-utils/src/orchestrion/config.ts @@ -11,6 +11,19 @@ import type { InstrumentationConfig } from '@apm-js-collab/code-transformer'; * `channelName` here is the unprefixed suffix; the actual diagnostics_channel * name is `orchestrion:${module.name}:${channelName}` (see `channels.ts`). */ +/** + * `ai` ships a single bundled entry per module system, so each instrumented + * function needs one config entry per file (the app loads whichever matches its + * module system). This expands a single target into both. + */ +function vercelAiV6Entries(channelName: string, functionName: string, kind: 'Async' | 'Sync'): InstrumentationConfig[] { + return ['dist/index.js', 'dist/index.mjs'].map(filePath => ({ + channelName, + module: { name: 'ai', versionRange: '>=6.0.0 <7.0.0', filePath }, + functionQuery: { functionName, kind }, + })); +} + export const SENTRY_INSTRUMENTATIONS: InstrumentationConfig[] = [ { channelName: 'query', @@ -32,6 +45,18 @@ export const SENTRY_INSTRUMENTATIONS: InstrumentationConfig[] = [ // attach `'end'`/`'error'` listeners that finish the span. functionQuery: { expressionName: 'query', kind: 'Auto' }, }, + // Vercel AI v6: mirror the v7 native `ai:telemetry` channel by injecting + // channels into the top-level entry points. `resolveLanguageModel` is wrapped + // not to span it, but so the subscriber can monkey-patch `doGenerate`/ + // `doStream` on the returned model (the only way to span the model call, + // which is an inline call with no injectable definition in `ai`). + // `streamText` returns its result synchronously (streaming is lazy), so it's + // `Sync`; the subscriber ends the span off the result's usage promise. + ...vercelAiV6Entries('generateText', 'generateText', 'Async'), + ...vercelAiV6Entries('streamText', 'streamText', 'Sync'), + ...vercelAiV6Entries('embed', 'embed', 'Async'), + ...vercelAiV6Entries('executeToolCall', 'executeToolCall', 'Async'), + ...vercelAiV6Entries('resolveLanguageModel', 'resolveLanguageModel', 'Sync'), ]; /** diff --git a/packages/server-utils/src/vercel-ai/vercel-ai-orchestrion-v6-subscriber.ts b/packages/server-utils/src/vercel-ai/vercel-ai-orchestrion-v6-subscriber.ts new file mode 100644 index 000000000000..b0783b4f2658 --- /dev/null +++ b/packages/server-utils/src/vercel-ai/vercel-ai-orchestrion-v6-subscriber.ts @@ -0,0 +1,325 @@ +import { tracingChannel } from 'node:diagnostics_channel'; +import type { Span } from '@sentry/core'; +import { debug, SPAN_STATUS_ERROR, withActiveSpan } from '@sentry/core'; +import { DEBUG_BUILD } from '../debug-build'; +import { CHANNELS } from '../orchestrion/channels'; +import { + clearOperationId, + createSpanFromMessage, + enrichSpanOnEnd, + type VercelAiChannelMessage, +} from './vercel-ai-dc-subscriber'; + +/** + * v6 channel adapter for the Vercel AI (`ai`) SDK. + * + * `ai` >= 7 publishes a normalized `ai:telemetry` tracing channel natively + * (consumed by `subscribeVercelAiTracingChannel`). v6 has no such channel, so + * orchestrion injects `orchestrion:ai:*` channels around the top-level + * functions (see `orchestrion/config.ts`). The injected channels carry only the + * wrapped call's `{ arguments, result, error }` — NOT v7's normalized `event` + * object — so this adapter reconstructs an equivalent {@link VercelAiChannelMessage} + * from v6's argument/result shapes and delegates to the SAME span-building core + * (`createSpanFromMessage` / `enrichSpanOnEnd`) the v7 subscriber uses, so the + * emitted spans are identical between v6 and v7. + * + * The model call (`languageModelCall` / `generate_content` span) has no + * injectable definition in `ai`, so we instead wrap `resolveLanguageModel` (the + * single chokepoint every model call flows through) and monkey-patch + * `doGenerate`/`doStream` on the returned model. + */ + +/** Shape orchestrion's transform attaches to the tracing-channel context. */ +interface OrchestrionContext { + arguments: unknown[]; + result?: unknown; + error?: unknown; +} + +/** Builds the normalized message for a channel from the wrapped call's first-arg options. */ +type MessageBuilder = (options: Record, telemetry: Record) => VercelAiChannelMessage; + +interface OperationConfig { + /** Push the opened span onto the operation stack so nested model calls can parent to it. */ + trackParent?: boolean; + /** The wrapped fn returns synchronously (e.g. `streamText`); finish when the result's usage settles. */ + lazy?: boolean; +} + +/** A resolved `ai` language model — has `doGenerate`/`doStream` and identity fields. */ +interface ResolvedModel { + modelId?: string; + provider?: string; + doGenerate?: (...args: unknown[]) => Promise; + doStream?: (...args: unknown[]) => Promise; +} + +const PATCHED = Symbol('SentryVercelAiModelPatched'); +const PARENT = Symbol('SentryVercelAiModelParent'); + +/** A resolved model with our patch bookkeeping (idempotency flag + captured parent span). */ +type PatchableModel = ResolvedModel & { [PATCHED]?: boolean; [PARENT]?: Span }; + +// Per-operation correlation id. No Date/random (unavailable / non-deterministic) — a counter is enough. +let callIdCounter = 0; +function nextCallId(): string { + return `v6-${++callIdCounter}`; +} + +// Stack of in-flight top-level operation spans. `resolveLanguageModel` runs +// synchronously inside an operation's body (before any await), so the top of +// the stack at resolve time is reliably the enclosing invoke_agent span — which +// we capture as the explicit parent for the model's (async) doGenerate spans, +// since orchestrion channels don't bind the active context the way v7's OTel +// factory does. +const operationSpanStack: Span[] = []; + +const spans = new WeakMap(); +const messages = new WeakMap(); + +let subscribed = false; + +/** + * Subscribe the v6 orchestrion channel adapter. Safe to always call: inert on + * `ai` >= 7 (those channels are never published) and when orchestrion injection + * isn't active. Idempotent. + */ +export function subscribeVercelAiOrchestrionChannels(): void { + if (subscribed) { + return; + } + subscribed = true; + + try { + subscribeOperation(CHANNELS.VERCEL_AI_GENERATE_TEXT, buildTextMessage('generateText'), { trackParent: true }); + subscribeOperation(CHANNELS.VERCEL_AI_STREAM_TEXT, buildTextMessage('streamText'), { + trackParent: true, + lazy: true, + }); + subscribeOperation(CHANNELS.VERCEL_AI_EMBED, (options, telemetry) => ({ + type: 'embed', + event: { + callId: nextCallId(), + ...modelFields(options.model), + maxRetries: options.maxRetries, + value: options.value, + ...recording(telemetry), + }, + })); + subscribeOperation(CHANNELS.VERCEL_AI_EXECUTE_TOOL_CALL, (options, telemetry) => ({ + type: 'executeTool', + // v6 carries the tool definitions on the executeToolCall args (a record keyed by name); + // the shared core reads the matching tool's `description` for the span. + event: { callId: nextCallId(), toolCall: options.toolCall, tools: options.tools, ...recording(telemetry) }, + })); + subscribeResolveLanguageModel(CHANNELS.VERCEL_AI_RESOLVE_LANGUAGE_MODEL); + } catch { + DEBUG_BUILD && debug.log('Vercel AI orchestrion channel subscription failed.'); + } +} + +/** + * Subscribe one operation channel: `start` opens a span from the built message; the span finishes on + * `asyncEnd` (success), or — for a `lazy` (synchronously-returning) operation — when the returned + * result's usage promise settles; `error` fails it. + */ +function subscribeOperation(channelName: string, build: MessageBuilder, config: OperationConfig = {}): void { + tracingChannel(channelName).subscribe({ + start(rawCtx) { + const ctx = rawCtx as OrchestrionContext; + const options = isRecord(ctx.arguments[0]) ? ctx.arguments[0] : {}; + const telemetry = isRecord(options.experimental_telemetry) ? options.experimental_telemetry : {}; + const span = openSpan(ctx, build(options, telemetry)); + if (config.trackParent) { + operationSpanStack.push(span); + } + }, + asyncEnd(rawCtx) { + const ctx = rawCtx as OrchestrionContext; + popParent(ctx, config); + if (!config.lazy) { + finishSpan(ctx, ctx.result); + } + }, + end(rawCtx) { + if (!config.lazy) { + return; + } + // Sync return: ctx.result is the lazy result (e.g. DefaultStreamTextResult). End when its usage + // promise settles (mirrors v7, whose streamText channel result is empty). + const ctx = rawCtx as OrchestrionContext; + popParent(ctx, config); + const completion = isRecord(ctx.result) ? (ctx.result.totalUsage ?? ctx.result.usage) : undefined; + void Promise.resolve(completion).then( + () => finishSpan(ctx, undefined), + () => finishSpan(ctx, undefined), + ); + }, + error(rawCtx) { + const ctx = rawCtx as OrchestrionContext; + popParent(ctx, config); + failSpan(ctx, ctx.error); + }, + asyncStart() { + /* no-op */ + }, + }); +} + +/** + * `resolveLanguageModel` returns the model every call flows through. We don't span it — on `end` we + * monkey-patch `doGenerate`/`doStream` on the returned model so each invocation produces a + * `languageModelCall` span parented to the enclosing invoke_agent span (captured from the operation + * stack at resolve time, since this runs synchronously inside the operation body). + */ +function subscribeResolveLanguageModel(channelName: string): void { + tracingChannel(channelName).subscribe({ + end(rawCtx) { + const ctx = rawCtx as OrchestrionContext; + if (!isRecord(ctx.result)) { + return; + } + const model = ctx.result as PatchableModel; + // Capture/refresh the parent for this resolve, so a model reused across operations spans its + // calls under the right invoke_agent span. + const parent = operationSpanStack[operationSpanStack.length - 1]; + if (parent) { + model[PARENT] = parent; + } + if (!model[PATCHED]) { + model[PATCHED] = true; + patchModelMethod(model, 'doGenerate'); + patchModelMethod(model, 'doStream'); + } + }, + start() { + /* no-op */ + }, + asyncStart() { + /* no-op */ + }, + asyncEnd() { + /* no-op */ + }, + error() { + /* no-op */ + }, + }); +} + +function patchModelMethod(model: PatchableModel, method: 'doGenerate' | 'doStream'): void { + const original = model[method]; + if (typeof original !== 'function') { + return; + } + model[method] = function (this: unknown, ...args: unknown[]): Promise { + const parent = model[PARENT]; + const callArgs = isRecord(args[0]) ? args[0] : {}; + const message: VercelAiChannelMessage = { + type: 'languageModelCall', + event: { provider: model.provider, modelId: model.modelId, tools: callArgs.tools, messages: callArgs.prompt }, + }; + const span = parent ? withActiveSpan(parent, () => createSpanFromMessage(message)) : createSpanFromMessage(message); + + let result: Promise; + try { + result = Promise.resolve(original.apply(this, args)); + } catch (error) { + span.setStatus({ code: SPAN_STATUS_ERROR, message: error instanceof Error ? error.message : 'unknown_error' }); + span.end(); + throw error; + } + // `doStream` resolves to `{ stream, ... }` before the stream is consumed; we end here (start/end + // bracket the call) to match the channel timing. + return result.then( + value => { + message.result = value; + enrichSpanOnEnd(span, message); + span.end(); + return value; + }, + error => { + span.setStatus({ code: SPAN_STATUS_ERROR, message: error instanceof Error ? error.message : 'unknown_error' }); + span.end(); + throw error; + }, + ); + }; +} + +function openSpan(ctx: OrchestrionContext, message: VercelAiChannelMessage): Span { + const span = createSpanFromMessage(message); + spans.set(ctx, span); + messages.set(ctx, message); + return span; +} + +function finishSpan(ctx: OrchestrionContext, result: unknown): void { + const span = spans.get(ctx); + const message = messages.get(ctx); + if (!span || !message) { + return; + } + message.result = result; + enrichSpanOnEnd(span, message); + span.end(); + clearOperationId(message); +} + +function failSpan(ctx: OrchestrionContext, error: unknown): void { + const message = messages.get(ctx); + if (message) { + clearOperationId(message); + } + const span = spans.get(ctx); + if (span) { + span.setStatus({ code: SPAN_STATUS_ERROR, message: error instanceof Error ? error.message : 'unknown_error' }); + } +} + +function popParent(ctx: OrchestrionContext, config: OperationConfig): void { + if (!config.trackParent) { + return; + } + const span = spans.get(ctx); + const index = span ? operationSpanStack.lastIndexOf(span) : -1; + if (index !== -1) { + operationSpanStack.splice(index, 1); + } +} + +function buildTextMessage(type: 'generateText' | 'streamText'): MessageBuilder { + return (options, telemetry) => ({ + type, + event: { + callId: nextCallId(), + operationId: type === 'streamText' ? 'ai.streamText' : 'ai.generateText', + functionId: asString(telemetry.functionId), + ...modelFields(options.model), + maxRetries: options.maxRetries, + messages: options.messages, + prompt: options.prompt, + ...recording(telemetry), + }, + }); +} + +function recording(telemetry: Record): { recordInputs: unknown; recordOutputs: unknown } { + return { recordInputs: telemetry.recordInputs, recordOutputs: telemetry.recordOutputs }; +} + +function modelFields(model: unknown): { provider?: string; modelId?: string } { + return { provider: modelField(model, 'provider'), modelId: modelField(model, 'modelId') }; +} + +function modelField(model: unknown, field: 'modelId' | 'provider'): string | undefined { + return isRecord(model) ? asString(model[field]) : undefined; +} + +function asString(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +}