From 07b09481343533a7c114d2552a42d60f3272e6b8 Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Thu, 9 Jul 2026 12:27:13 +0000 Subject: [PATCH 1/4] fix: honor ignore on enter for toolsets --- .changeset/honor-ignore-on-enter-toolsets.md | 5 ++ agents/src/beta/tools/end_call.ts | 5 ++ agents/src/llm/tool_context.test.ts | 44 ++++++++++++ agents/src/llm/tool_context.ts | 38 ++++++++++ agents/src/voice/agent_activity.test.ts | 73 +++++++++++++++++++- agents/src/voice/agent_activity.ts | 42 +++++++---- 6 files changed, 192 insertions(+), 15 deletions(-) create mode 100644 .changeset/honor-ignore-on-enter-toolsets.md diff --git a/.changeset/honor-ignore-on-enter-toolsets.md b/.changeset/honor-ignore-on-enter-toolsets.md new file mode 100644 index 000000000..6370558fd --- /dev/null +++ b/.changeset/honor-ignore-on-enter-toolsets.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents': patch +--- + +Honor `ToolFlag.IGNORE_ON_ENTER` for tools nested inside `Toolset`s. diff --git a/agents/src/beta/tools/end_call.ts b/agents/src/beta/tools/end_call.ts index 962deda36..d444dd17e 100644 --- a/agents/src/beta/tools/end_call.ts +++ b/agents/src/beta/tools/end_call.ts @@ -8,6 +8,7 @@ import { RealtimeModel, type ToolCalledEvent, type ToolCompletedEvent, + ToolFlag, Toolset, tool, } from '../../llm/index.js'; @@ -68,6 +69,8 @@ export type EndCallToolOptions = { deleteRoom?: boolean; /** Tool output to the LLM for generating the tool response. */ endInstructions?: string | null; + /** Hide the tool during onEnter so the model can't end the call while greeting. */ + ignoreOnEnter?: boolean; /** Callback to call when the tool is called. */ onToolCalled?: (event: EndCallToolCalledEvent) => Promise | void; /** Callback to call when the tool is completed. */ @@ -81,6 +84,7 @@ export function createEndCallTool({ extraDescription = '', deleteRoom = true, endInstructions = 'say goodbye to the user', + ignoreOnEnter = false, onToolCalled, onToolCompleted, }: EndCallToolOptions = {}): Toolset { @@ -116,6 +120,7 @@ export function createEndCallTool({ tool({ name: 'end_call', description: `${END_CALL_DESCRIPTION}\n${extraDescription}`, + flags: ignoreOnEnter ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE, execute: async (_args, { ctx, abortSignal }) => { log().debug('end_call tool called'); const session = ctx.session; diff --git a/agents/src/llm/tool_context.test.ts b/agents/src/llm/tool_context.test.ts index 10bfdc90f..15d492fec 100644 --- a/agents/src/llm/tool_context.test.ts +++ b/agents/src/llm/tool_context.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it } from 'vitest'; import { z } from 'zod'; import * as z3 from 'zod/v3'; import * as z4 from 'zod/v4'; +import { createEndCallTool } from '../beta/tools/end_call.js'; import { voice } from '../index.js'; import { CONFIRM_DUPLICATE_PARAM, @@ -15,6 +16,7 @@ import { type ToolOptions, Toolset, type ToolsetContext, + isFunctionTool, tool, } from './tool_context.js'; import { createToolOptions, oaiParams } from './utils.js'; @@ -701,6 +703,32 @@ describe('ToolContext', () => { expect(ctx.getFunctionTool('b')).toBe(b); }); + it('_exclude hides a toolset member but keeps the toolset', () => { + const a = makeFn('a'); + const b = makeFn('b'); + const direct = makeFn('direct'); + const ts = new Toolset({ id: 'set', tools: [a, b] }); + const ctx = new ToolContext([ts, direct]); + + ctx._exclude([a]); + + expect(ctx.getFunctionTool('a')).toBeUndefined(); + expect(ctx.flatten()).not.toContain(a); + expect(ctx.getFunctionTool('b')).toBe(b); + expect(ctx.getFunctionTool('direct')).toBe(direct); + expect(ctx.toolsets).toEqual([ts]); + }); + + it('_exclude with no tools is a no-op', () => { + const a = makeFn('a'); + const b = makeFn('b'); + const ctx = new ToolContext([a, b]); + + ctx._exclude([]); + + expect(Object.keys(ctx.functionTools).sort()).toEqual(['a', 'b']); + }); + it('copy() yields an independent context with the same tools', () => { const a = makeFn('a'); const ctx = new ToolContext([a]); @@ -768,6 +796,22 @@ describe('ToolContext', () => { }); }); +describe('createEndCallTool', () => { + it('sets IGNORE_ON_ENTER only when requested', () => { + const [defaultTool] = createEndCallTool().tools; + expect(defaultTool).toBeDefined(); + expect(isFunctionTool(defaultTool)).toBe(true); + expect(isFunctionTool(defaultTool) && defaultTool.flags & ToolFlag.IGNORE_ON_ENTER).toBe(0); + + const [ignoredTool] = createEndCallTool({ ignoreOnEnter: true }).tools; + expect(ignoredTool).toBeDefined(); + expect(isFunctionTool(ignoredTool)).toBe(true); + expect( + isFunctionTool(ignoredTool) && ignoredTool.flags & ToolFlag.IGNORE_ON_ENTER, + ).toBeTruthy(); + }); +}); + describe('Toolset', () => { const makeFn = (name: string) => tool({ diff --git a/agents/src/llm/tool_context.ts b/agents/src/llm/tool_context.ts index 6b947a021..d7df12934 100644 --- a/agents/src/llm/tool_context.ts +++ b/agents/src/llm/tool_context.ts @@ -515,7 +515,12 @@ export class ToolContext { } updateTools(tools: ToolContextInit): void { + this._updateTools(tools); + } + + private _updateTools(tools: ToolContextInit, exclude: readonly Tool[] = []): void { const normalizedTools = normalizeToolContextInit(tools); + const excludedTools = new Set(exclude); this._tools = normalizedTools; this._functionToolsMap = new Map(); this._providerTools = []; @@ -523,6 +528,10 @@ export class ToolContext { // eslint-disable-next-line @typescript-eslint/no-explicit-any -- accepts any tool shape const addTool = (tool: any): void => { + if (excludedTools.has(tool)) { + return; + } + if (isToolset(tool)) { for (const inner of tool.tools) { addTool(inner); @@ -556,6 +565,35 @@ export class ToolContext { } } + private _syncFlattened(tools: readonly Tool[]): void { + const current = this.flatten(); + const currentTools = new Set(current); + const nextTools = new Set(tools); + if (currentTools.size === nextTools.size && current.every((tool) => nextTools.has(tool))) { + return; + } + + const added: Extract, Tool>[] = tools + .filter((tool) => !currentTools.has(tool)) + .filter( + (tool): tool is Extract, Tool> => + isFunctionTool(tool) || isProviderTool(tool), + ); + const removed = current.filter((tool) => !nextTools.has(tool)); + const structured = this._tools.filter((tool) => !removed.includes(tool as Tool)); + this._updateTools([...structured, ...added], removed); + } + + /** Hide tools from the callable set while keeping their toolsets intact. @internal */ + _exclude(tools: readonly Tool[]): void { + if (tools.length === 0) { + return; + } + + const excludedTools = new Set(tools); + this._syncFlattened(this.flatten().filter((tool) => !excludedTools.has(tool))); + } + copy(): ToolContext { return new ToolContext([...this._tools]); } diff --git a/agents/src/voice/agent_activity.test.ts b/agents/src/voice/agent_activity.test.ts index e8c36bd00..e7de6cef5 100644 --- a/agents/src/voice/agent_activity.test.ts +++ b/agents/src/voice/agent_activity.test.ts @@ -21,7 +21,7 @@ import { LLM, type LLMStream } from '../llm/llm.js'; import { type Tool, ToolContext, ToolFlag, Toolset, tool } from '../llm/tool_context.js'; import { Future, Task } from '../utils.js'; import { _getActivityTaskInfo } from './agent.js'; -import { AgentActivity } from './agent_activity.js'; +import { AgentActivity, onEnterStorage } from './agent_activity.js'; import type { PreemptiveGenerationInfo } from './audio_recognition.js'; import type { AgentSessionEventTypes, UserInputTranscribedEvent } from './events.js'; import { SpeechHandle } from './speech_handle.js'; @@ -582,6 +582,77 @@ describe('AgentActivity - onPreemptiveGeneration guards', () => { }); }); +describe('AgentActivity - onEnter ignored tools', () => { + const makeFn = (name: string, flags = ToolFlag.NONE) => + tool({ name, description: `${name} tool`, flags, execute: async () => name }); + + function buildIgnoredToolsActivity() { + const activity = Object.create(AgentActivity.prototype) as AgentActivity; + Object.assign(activity, { + agent: {}, + agentSession: {}, + }); + return activity; + } + + it('returns bare and toolset-nested IGNORE_ON_ENTER tools only inside this onEnter', () => { + const endCall = makeFn('end_call', ToolFlag.IGNORE_ON_ENTER); + const keep = makeFn('keep'); + const bareIgnored = makeFn('bare_ignored', ToolFlag.IGNORE_ON_ENTER); + const bareKeep = makeFn('bare_keep'); + const toolset = new Toolset({ id: 'ts', tools: [endCall, keep] }); + const toolCtx = new ToolContext([toolset, bareIgnored, bareKeep]); + const activity = buildIgnoredToolsActivity(); + + expect(activity._onEnterIgnoredTools(toolCtx)).toEqual([]); + + onEnterStorage.run({ session: activity.agentSession, agent: activity.agent }, () => { + expect( + activity + ._onEnterIgnoredTools(toolCtx) + .map((t) => t.id) + .sort(), + ).toEqual(['bare_ignored', 'end_call']); + }); + + onEnterStorage.run({ session: activity.agentSession, agent: {} as never }, () => { + expect(activity._onEnterIgnoredTools(toolCtx)).toEqual([]); + }); + }); + + it('hides ignored bare and toolset tools while preserving normal tools for nested replies', async () => { + const endCall = makeFn('end_call', ToolFlag.IGNORE_ON_ENTER); + const keep = makeFn('keep'); + const bareIgnored = makeFn('bare_ignored', ToolFlag.IGNORE_ON_ENTER); + const toolset = new Toolset({ id: 'ts', tools: [endCall, keep] }); + const activity = buildIgnoredToolsActivity(); + + await onEnterStorage.run( + { session: activity.agentSession, agent: activity.agent }, + async () => { + const greetingTools = new ToolContext([toolset, bareIgnored]); + greetingTools._exclude(activity._onEnterIgnoredTools(greetingTools)); + expect(greetingTools.flatten().map((t) => t.id)).toEqual(['keep']); + expect(greetingTools.toolsets).toEqual([toolset]); + + await Promise.resolve(); + + const toolReplyTools = new ToolContext([toolset, bareIgnored]); + toolReplyTools._exclude(activity._onEnterIgnoredTools(toolReplyTools)); + expect(toolReplyTools.flatten().map((t) => t.id)).toEqual(['keep']); + }, + ); + + const restoredTools = new ToolContext([toolset, bareIgnored]); + expect( + restoredTools + .flatten() + .map((t) => t.id) + .sort(), + ).toEqual(['bare_ignored', 'end_call', 'keep']); + }); +}); + /** * Regression test for the dynamic-toolset push path. * diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts index 9fd906783..abaa0a2c3 100644 --- a/agents/src/voice/agent_activity.ts +++ b/agents/src/voice/agent_activity.ts @@ -41,13 +41,13 @@ import { RealtimeModel, type RealtimeModelError, type RealtimeSession, + type Tool, type ToolChoice, ToolContext, type ToolContextEntry, type ToolContextLike, ToolFlag, isFunctionTool, - isToolset, toToolContext, } from '../llm/index.js'; import type { LLMError } from '../llm/llm.js'; @@ -2136,19 +2136,8 @@ export class AgentActivity implements RecognitionHooks { instructions = concatInstructions(this.agent.instructions, '\n', instructions); } - // Filter out tools with IGNORE_ON_ENTER flag when generateReply is called inside onEnter - const onEnterData = onEnterStorage.getStore(); - const shouldFilterTools = - onEnterData?.agent === this.agent && onEnterData?.session === this.agentSession; - - const tools: ToolContext = shouldFilterTools - ? new ToolContext( - this.tools.tools.filter((t): boolean => { - if (isToolset(t) || !isFunctionTool(t)) return true; - return !(t.flags & ToolFlag.IGNORE_ON_ENTER); - }), - ) - : this.tools; + const tools = this.tools; + tools._exclude(this._onEnterIgnoredTools(tools)); const task = this.createSpeechTask({ taskFn: (abortController: AbortController) => @@ -2234,6 +2223,19 @@ export class AgentActivity implements RecognitionHooks { } } + _onEnterIgnoredTools(toolCtx: ToolContext): Tool[] { + const onEnterData = onEnterStorage.getStore(); + if (onEnterData?.agent !== this.agent || onEnterData.session !== this.agentSession) { + return []; + } + + return toolCtx + .flatten() + .filter( + (tool): tool is Tool => isFunctionTool(tool) && !!(tool.flags & ToolFlag.IGNORE_ON_ENTER), + ); + } + /** * Commit a user turn whose reply is being skipped: append the transcript to the * agent chat context (when non-empty) without triggering reply generation. @@ -3910,6 +3912,15 @@ export class AgentActivity implements RecognitionHooks { } const originalToolChoice = this.toolChoice; + let originalTools: ToolContext | undefined; + const turnTools = this.realtimeSession.tools; + const onEnterIgnoredTools = this._onEnterIgnoredTools(turnTools); + if (onEnterIgnoredTools.length > 0) { + originalTools = this.realtimeSession.tools; + turnTools._exclude(onEnterIgnoredTools); + await this.realtimeSession.updateTools(turnTools); + } + if (toolChoice !== undefined) { this.realtimeSession.updateOptions({ toolChoice }); } @@ -3942,6 +3953,9 @@ export class AgentActivity implements RecognitionHooks { if (toolChoice !== undefined && toolChoice !== originalToolChoice) { this.realtimeSession.updateOptions({ toolChoice: originalToolChoice }); } + if (originalTools !== undefined) { + await this.realtimeSession.updateTools(originalTools); + } } } From a54cd93c5a289ee40f25efca38810f0865033158 Mon Sep 17 00:00:00 2001 From: Toubat Date: Wed, 15 Jul 2026 11:17:19 -0700 Subject: [PATCH 2/4] fix: preserve realtime ignored tool restoration Exercise full pipeline and realtime sessions so nested on-enter filtering remains scoped through tool responses without leaking into later user turns. Co-authored-by: Cursor --- agents/src/llm/tool_context.ts | 7 + agents/src/voice/agent_activity.ts | 7 +- .../src/voice/agent_ignore_on_enter.test.ts | 280 ++++++++++++++++++ 3 files changed, 291 insertions(+), 3 deletions(-) create mode 100644 agents/src/voice/agent_ignore_on_enter.test.ts diff --git a/agents/src/llm/tool_context.ts b/agents/src/llm/tool_context.ts index d7df12934..a8dfb00ac 100644 --- a/agents/src/llm/tool_context.ts +++ b/agents/src/llm/tool_context.ts @@ -594,6 +594,13 @@ export class ToolContext { this._syncFlattened(this.flatten().filter((tool) => !excludedTools.has(tool))); } + /** Return a copy containing only flattened callable/provider entries. @internal */ + _flattenedCopy(): ToolContext { + const copy = ToolContext.empty(); + copy._syncFlattened(this.flatten()); + return copy; + } + copy(): ToolContext { return new ToolContext([...this._tools]); } diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts index abaa0a2c3..579f3751b 100644 --- a/agents/src/voice/agent_activity.ts +++ b/agents/src/voice/agent_activity.ts @@ -3913,10 +3913,11 @@ export class AgentActivity implements RecognitionHooks { const originalToolChoice = this.toolChoice; let originalTools: ToolContext | undefined; - const turnTools = this.realtimeSession.tools; - const onEnterIgnoredTools = this._onEnterIgnoredTools(turnTools); + const sessionTools = this.realtimeSession.tools; + const onEnterIgnoredTools = this._onEnterIgnoredTools(sessionTools); if (onEnterIgnoredTools.length > 0) { - originalTools = this.realtimeSession.tools; + originalTools = sessionTools; + const turnTools = sessionTools._flattenedCopy(); turnTools._exclude(onEnterIgnoredTools); await this.realtimeSession.updateTools(turnTools); } diff --git a/agents/src/voice/agent_ignore_on_enter.test.ts b/agents/src/voice/agent_ignore_on_enter.test.ts new file mode 100644 index 000000000..2841e1bbc --- /dev/null +++ b/agents/src/voice/agent_ignore_on_enter.test.ts @@ -0,0 +1,280 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import type { AudioFrame } from '@livekit/rtc-node'; +import { ReadableStream } from 'node:stream/web'; +import { describe, expect, it, vi } from 'vitest'; +import { ChatContext, FunctionCall } from '../llm/chat_context.js'; +import type { LLMStream } from '../llm/llm.js'; +import { + type GenerationCreatedEvent, + type RealtimeCapabilities, + RealtimeModel, + RealtimeSession, +} from '../llm/realtime.js'; +import { + type ToolChoice, + ToolContext, + type ToolContextLike, + ToolFlag, + Toolset, + tool, +} from '../llm/tool_context.js'; +import { initializeLogger } from '../log.js'; +import type { APIConnectOptions } from '../types.js'; +import { Agent } from './agent.js'; +import { AgentSession } from './agent_session.js'; +import { FakeLLM } from './testing/fake_llm.js'; + +initializeLogger({ pretty: false, level: 'silent' }); + +const EXPECTED_GREETING_TOOLS = ['allowed_nested', 'allowed_plain']; +const EXPECTED_USER_TURN_TOOLS = [ + 'allowed_nested', + 'allowed_plain', + 'ignored_nested', + 'ignored_plain', +]; + +function emptyStream(): ReadableStream { + return new ReadableStream({ + start(controller) { + controller.close(); + }, + }); +} + +function oneItemStream(item: T): ReadableStream { + return new ReadableStream({ + start(controller) { + controller.enqueue(item); + controller.close(); + }, + }); +} + +function toolNames(toolCtx: ToolContextLike | undefined): string[] { + if (!(toolCtx instanceof ToolContext)) { + return []; + } + return toolCtx + .flatten() + .map((entry) => entry.id) + .sort(); +} + +function createTools(): ToolContext { + const allowedNested = tool({ + name: 'allowed_nested', + description: 'Allowed nested probe.', + execute: async () => 'nested allowed result', + }); + const ignoredNested = tool({ + name: 'ignored_nested', + description: 'Ignored nested probe.', + flags: ToolFlag.IGNORE_ON_ENTER, + execute: async () => 'nested ignored result', + }); + const inner = new Toolset({ + id: 'inner', + tools: [allowedNested, ignoredNested], + }); + const outer = new Toolset({ + id: 'outer', + tools: [inner], + }); + const allowedPlain = tool({ + name: 'allowed_plain', + description: 'Allowed plain probe.', + execute: async () => 'plain allowed result', + }); + const ignoredPlain = tool({ + name: 'ignored_plain', + description: 'Ignored plain probe.', + flags: ToolFlag.IGNORE_ON_ENTER, + execute: async () => 'plain ignored result', + }); + return new ToolContext([outer, allowedPlain, ignoredPlain]); +} + +class GreetingAgent extends Agent { + constructor() { + super({ + instructions: 'Test tool visibility.', + tools: createTools(), + }); + } + + async onEnter(): Promise { + this.session.generateReply({ userInput: 'greeting' }); + } +} + +class RecordingLLM extends FakeLLM { + readonly toolSnapshots: string[][] = []; + + override chat({ + chatCtx, + toolCtx, + connOptions, + parallelToolCalls, + toolChoice, + extraKwargs, + }: { + chatCtx: ChatContext; + toolCtx?: ToolContextLike; + connOptions?: APIConnectOptions; + parallelToolCalls?: boolean; + toolChoice?: ToolChoice; + extraKwargs?: Record; + }): LLMStream { + this.toolSnapshots.push(toolNames(toolCtx)); + return super.chat({ + chatCtx, + toolCtx, + connOptions, + parallelToolCalls, + toolChoice, + extraKwargs, + }); + } +} + +class RecordingRealtimeSession extends RealtimeSession { + private _chatCtx = ChatContext.empty(); + private _tools = ToolContext.empty(); + readonly toolSnapshots: string[][] = []; + + get chatCtx(): ChatContext { + return this._chatCtx; + } + + get tools(): ToolContext { + return this._tools; + } + + async updateInstructions(_instructions: string): Promise {} + + async updateChatCtx(chatCtx: ChatContext): Promise { + this._chatCtx = chatCtx.copy(); + } + + async updateTools(tools: ToolContext): Promise { + this._tools = tools.copy(); + } + + updateOptions(_options: { toolChoice?: ToolChoice | null }): void {} + + pushAudio(_frame: AudioFrame): void {} + + async generateReply(): Promise { + this.toolSnapshots.push(toolNames(this._tools)); + const functionStream = + this.toolSnapshots.length === 1 + ? oneItemStream( + FunctionCall.create({ + callId: 'allowed-call', + name: 'allowed_nested', + args: '{}', + }), + ) + : emptyStream(); + return { + messageStream: emptyStream(), + functionStream, + userInitiated: true, + }; + } + + async commitAudio(): Promise {} + + async clearAudio(): Promise {} + + async interrupt(): Promise {} + + async truncate(): Promise {} +} + +class RecordingRealtimeModel extends RealtimeModel { + readonly recordingSession: RecordingRealtimeSession; + + constructor() { + const capabilities: RealtimeCapabilities = { + messageTruncation: false, + turnDetection: false, + userTranscription: false, + autoToolReplyGeneration: false, + audioOutput: false, + manualFunctionCalls: false, + midSessionChatCtxUpdate: true, + midSessionInstructionsUpdate: true, + midSessionToolsUpdate: true, + perResponseToolChoice: false, + }; + super(capabilities); + this.recordingSession = new RecordingRealtimeSession(this); + } + + get model(): string { + return 'recording-realtime'; + } + + session(): RealtimeSession { + return this.recordingSession; + } + + async close(): Promise {} +} + +describe('AgentSession IGNORE_ON_ENTER tool filtering', () => { + it('filters nested tools from pipeline greeting and tool response, then restores them', async () => { + const llm = new RecordingLLM([ + { + input: 'greeting', + toolCalls: [{ name: 'allowed_nested', args: {} }], + }, + { + input: 'nested allowed result', + content: 'hello', + }, + { + input: 'later user turn', + content: 'welcome back', + }, + ]); + const session = new AgentSession({ llm }); + + try { + await session.start({ agent: new GreetingAgent() }); + await vi.waitFor(() => expect(llm.toolSnapshots).toHaveLength(2)); + await session.run({ userInput: 'later user turn' }).wait(); + + expect(llm.toolSnapshots).toEqual([ + EXPECTED_GREETING_TOOLS, + EXPECTED_GREETING_TOOLS, + EXPECTED_USER_TURN_TOOLS, + ]); + } finally { + await session.close(); + } + }); + + it('filters nested tools from realtime greeting and tool response, then restores them', async () => { + const llm = new RecordingRealtimeModel(); + const session = new AgentSession({ llm }); + + try { + await session.start({ agent: new GreetingAgent() }); + await vi.waitFor(() => expect(llm.recordingSession.toolSnapshots).toHaveLength(2)); + await session.run({ userInput: 'later user turn' }).wait(); + + expect(llm.recordingSession.toolSnapshots).toEqual([ + EXPECTED_GREETING_TOOLS, + EXPECTED_GREETING_TOOLS, + EXPECTED_USER_TURN_TOOLS, + ]); + } finally { + await session.close(); + } + }); +}); From 4ce7dc660518d5ab90d67956c5e87472a12f4f53 Mon Sep 17 00:00:00 2001 From: Toubat Date: Wed, 15 Jul 2026 11:25:50 -0700 Subject: [PATCH 3/4] fix: restore realtime tools after rejected update Protect temporary realtime setup so providers that mutate before rejecting cannot leave ignored tools removed. Co-authored-by: Cursor --- agents/src/voice/agent_activity.ts | 24 ++++----- .../src/voice/agent_ignore_on_enter.test.ts | 51 ++++++++++++++++++- 2 files changed, 61 insertions(+), 14 deletions(-) diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts index 579f3751b..63b0492d0 100644 --- a/agents/src/voice/agent_activity.ts +++ b/agents/src/voice/agent_activity.ts @@ -3913,20 +3913,20 @@ export class AgentActivity implements RecognitionHooks { const originalToolChoice = this.toolChoice; let originalTools: ToolContext | undefined; - const sessionTools = this.realtimeSession.tools; - const onEnterIgnoredTools = this._onEnterIgnoredTools(sessionTools); - if (onEnterIgnoredTools.length > 0) { - originalTools = sessionTools; - const turnTools = sessionTools._flattenedCopy(); - turnTools._exclude(onEnterIgnoredTools); - await this.realtimeSession.updateTools(turnTools); - } + try { + const sessionTools = this.realtimeSession.tools; + const onEnterIgnoredTools = this._onEnterIgnoredTools(sessionTools); + if (onEnterIgnoredTools.length > 0) { + originalTools = sessionTools; + const turnTools = sessionTools._flattenedCopy(); + turnTools._exclude(onEnterIgnoredTools); + await this.realtimeSession.updateTools(turnTools); + } - if (toolChoice !== undefined) { - this.realtimeSession.updateOptions({ toolChoice }); - } + if (toolChoice !== undefined) { + this.realtimeSession.updateOptions({ toolChoice }); + } - try { const generateReplyAbortController = new AbortController(); const generationPromise = this.realtimeSession.generateReply( instructions !== undefined diff --git a/agents/src/voice/agent_ignore_on_enter.test.ts b/agents/src/voice/agent_ignore_on_enter.test.ts index 2841e1bbc..a7a0ec522 100644 --- a/agents/src/voice/agent_ignore_on_enter.test.ts +++ b/agents/src/voice/agent_ignore_on_enter.test.ts @@ -143,7 +143,14 @@ class RecordingLLM extends FakeLLM { class RecordingRealtimeSession extends RealtimeSession { private _chatCtx = ChatContext.empty(); private _tools = ToolContext.empty(); + private readonly rejectFilteredUpdate: boolean; readonly toolSnapshots: string[][] = []; + rejectedTemporaryUpdates = 0; + + constructor(model: RealtimeModel, rejectFilteredUpdate = false) { + super(model); + this.rejectFilteredUpdate = rejectFilteredUpdate; + } get chatCtx(): ChatContext { return this._chatCtx; @@ -161,6 +168,14 @@ class RecordingRealtimeSession extends RealtimeSession { async updateTools(tools: ToolContext): Promise { this._tools = tools.copy(); + if ( + this.rejectFilteredUpdate && + this.rejectedTemporaryUpdates === 0 && + !toolNames(this._tools).includes('ignored_nested') + ) { + this.rejectedTemporaryUpdates++; + throw new Error('temporary realtime tools rejected after mutation'); + } } updateOptions(_options: { toolChoice?: ToolChoice | null }): void {} @@ -198,7 +213,7 @@ class RecordingRealtimeSession extends RealtimeSession { class RecordingRealtimeModel extends RealtimeModel { readonly recordingSession: RecordingRealtimeSession; - constructor() { + constructor(rejectFilteredUpdate = false) { const capabilities: RealtimeCapabilities = { messageTruncation: false, turnDetection: false, @@ -212,7 +227,7 @@ class RecordingRealtimeModel extends RealtimeModel { perResponseToolChoice: false, }; super(capabilities); - this.recordingSession = new RecordingRealtimeSession(this); + this.recordingSession = new RecordingRealtimeSession(this, rejectFilteredUpdate); } get model(): string { @@ -277,4 +292,36 @@ describe('AgentSession IGNORE_ON_ENTER tool filtering', () => { await session.close(); } }); + + it('restores realtime tools when the temporary provider update mutates then rejects', async () => { + const llm = new RecordingRealtimeModel(true); + const session = new AgentSession({ llm }); + + try { + await session.start({ agent: new GreetingAgent() }); + await vi.waitFor(() => expect(llm.recordingSession.rejectedTemporaryUpdates).toBe(1)); + + expect(toolNames(llm.recordingSession.tools)).toEqual(EXPECTED_USER_TURN_TOOLS); + const restoredOuter = llm.recordingSession.tools.tools[0]; + expect(restoredOuter).toBeInstanceOf(Toolset); + if (!(restoredOuter instanceof Toolset)) { + throw new Error('expected restored outer toolset'); + } + expect(restoredOuter.id).toBe('outer'); + const restoredInner = restoredOuter.tools[0]; + expect(restoredInner).toBeInstanceOf(Toolset); + if (!(restoredInner instanceof Toolset)) { + throw new Error('expected restored inner toolset'); + } + expect(restoredInner.id).toBe('inner'); + + await session.run({ userInput: 'later user turn' }).wait(); + expect(llm.recordingSession.toolSnapshots).toEqual([ + EXPECTED_USER_TURN_TOOLS, + EXPECTED_USER_TURN_TOOLS, + ]); + } finally { + await session.close(); + } + }); }); From 280e7bdbb78d196db1f9c82c8a2efaec2ca4ac12 Mon Sep 17 00:00:00 2001 From: Toubat Date: Wed, 15 Jul 2026 11:37:57 -0700 Subject: [PATCH 4/4] fix: isolate realtime cleanup failures Restore tool choice and tools independently so cleanup errors cannot mask the primary generation failure or skip later cleanup. Co-authored-by: Cursor --- agents/src/voice/agent_activity.ts | 12 +- .../src/voice/agent_ignore_on_enter.test.ts | 145 ++++++++++++++++-- 2 files changed, 145 insertions(+), 12 deletions(-) diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts index 63b0492d0..75cb22ac0 100644 --- a/agents/src/voice/agent_activity.ts +++ b/agents/src/voice/agent_activity.ts @@ -3952,10 +3952,18 @@ export class AgentActivity implements RecognitionHooks { } finally { // reset toolChoice value if (toolChoice !== undefined && toolChoice !== originalToolChoice) { - this.realtimeSession.updateOptions({ toolChoice: originalToolChoice }); + try { + this.realtimeSession.updateOptions({ toolChoice: originalToolChoice }); + } catch (error) { + this.logger.error({ error }, 'failed to reset tool_choice'); + } } if (originalTools !== undefined) { - await this.realtimeSession.updateTools(originalTools); + try { + await this.realtimeSession.updateTools(originalTools); + } catch (error) { + this.logger.error({ error }, 'failed to reset tools'); + } } } } diff --git a/agents/src/voice/agent_ignore_on_enter.test.ts b/agents/src/voice/agent_ignore_on_enter.test.ts index a7a0ec522..a8eb8d369 100644 --- a/agents/src/voice/agent_ignore_on_enter.test.ts +++ b/agents/src/voice/agent_ignore_on_enter.test.ts @@ -24,6 +24,7 @@ import { initializeLogger } from '../log.js'; import type { APIConnectOptions } from '../types.js'; import { Agent } from './agent.js'; import { AgentSession } from './agent_session.js'; +import type { SpeechHandle } from './speech_handle.js'; import { FakeLLM } from './testing/fake_llm.js'; initializeLogger({ pretty: false, level: 'silent' }); @@ -110,6 +111,24 @@ class GreetingAgent extends Agent { } } +class CleanupFailureAgent extends Agent { + speech?: SpeechHandle; + + constructor() { + super({ + instructions: 'Test cleanup failure isolation.', + tools: createTools(), + }); + } + + async onEnter(): Promise { + this.speech = this.session.generateReply({ + userInput: 'greeting', + toolChoice: 'none', + }); + } +} + class RecordingLLM extends FakeLLM { readonly toolSnapshots: string[][] = []; @@ -140,16 +159,27 @@ class RecordingLLM extends FakeLLM { } } +type RealtimeFailureOptions = { + rejectFilteredUpdate?: boolean; + generationError?: Error; + emptyGeneration?: boolean; + throwToolChoiceRestore?: boolean; + throwToolRestore?: 'before' | 'after'; +}; + class RecordingRealtimeSession extends RealtimeSession { private _chatCtx = ChatContext.empty(); private _tools = ToolContext.empty(); - private readonly rejectFilteredUpdate: boolean; + private readonly failureOptions: RealtimeFailureOptions; + private temporaryToolsApplied = false; readonly toolSnapshots: string[][] = []; rejectedTemporaryUpdates = 0; + toolChoiceRestorationAttempts = 0; + toolRestorationAttempts = 0; - constructor(model: RealtimeModel, rejectFilteredUpdate = false) { + constructor(model: RealtimeModel, failureOptions: RealtimeFailureOptions = {}) { super(model); - this.rejectFilteredUpdate = rejectFilteredUpdate; + this.failureOptions = failureOptions; } get chatCtx(): ChatContext { @@ -167,25 +197,47 @@ class RecordingRealtimeSession extends RealtimeSession { } async updateTools(tools: ToolContext): Promise { + const isFilteredUpdate = !toolNames(tools).includes('ignored_nested'); + const isRestoration = this.temporaryToolsApplied && !isFilteredUpdate; + if (isRestoration) { + this.toolRestorationAttempts++; + if (this.failureOptions.throwToolRestore === 'before') { + throw new Error('tool restoration failure'); + } + } + this._tools = tools.copy(); + this.temporaryToolsApplied = isFilteredUpdate; + + if (isRestoration && this.failureOptions.throwToolRestore === 'after') { + throw new Error('tool restoration failure after apply'); + } if ( - this.rejectFilteredUpdate && + this.failureOptions.rejectFilteredUpdate && this.rejectedTemporaryUpdates === 0 && - !toolNames(this._tools).includes('ignored_nested') + isFilteredUpdate ) { this.rejectedTemporaryUpdates++; throw new Error('temporary realtime tools rejected after mutation'); } } - updateOptions(_options: { toolChoice?: ToolChoice | null }): void {} + updateOptions(options: { toolChoice?: ToolChoice | null }): void { + if (options.toolChoice !== 'none' && this.failureOptions.throwToolChoiceRestore) { + this.toolChoiceRestorationAttempts++; + throw new Error('tool-choice restoration failure'); + } + } pushAudio(_frame: AudioFrame): void {} async generateReply(): Promise { this.toolSnapshots.push(toolNames(this._tools)); + if (this.failureOptions.generationError) { + throw this.failureOptions.generationError; + } const functionStream = - this.toolSnapshots.length === 1 + !this.failureOptions.emptyGeneration && this.toolSnapshots.length === 1 ? oneItemStream( FunctionCall.create({ callId: 'allowed-call', @@ -213,7 +265,7 @@ class RecordingRealtimeSession extends RealtimeSession { class RecordingRealtimeModel extends RealtimeModel { readonly recordingSession: RecordingRealtimeSession; - constructor(rejectFilteredUpdate = false) { + constructor(failureOptions: RealtimeFailureOptions = {}) { const capabilities: RealtimeCapabilities = { messageTruncation: false, turnDetection: false, @@ -227,7 +279,7 @@ class RecordingRealtimeModel extends RealtimeModel { perResponseToolChoice: false, }; super(capabilities); - this.recordingSession = new RecordingRealtimeSession(this, rejectFilteredUpdate); + this.recordingSession = new RecordingRealtimeSession(this, failureOptions); } get model(): string { @@ -294,7 +346,7 @@ describe('AgentSession IGNORE_ON_ENTER tool filtering', () => { }); it('restores realtime tools when the temporary provider update mutates then rejects', async () => { - const llm = new RecordingRealtimeModel(true); + const llm = new RecordingRealtimeModel({ rejectFilteredUpdate: true }); const session = new AgentSession({ llm }); try { @@ -324,4 +376,77 @@ describe('AgentSession IGNORE_ON_ENTER tool filtering', () => { await session.close(); } }); + + it('restores tools and preserves the primary failure when tool-choice cleanup throws', async () => { + const primaryError = new Error('primary generation failure'); + const llm = new RecordingRealtimeModel({ + generationError: primaryError, + throwToolChoiceRestore: true, + }); + const agent = new CleanupFailureAgent(); + const session = new AgentSession({ llm }); + + try { + await session.start({ agent }); + await vi.waitFor(() => expect(agent.speech?._tasks).toHaveLength(1)); + const speechTask = agent.speech?._tasks[0]; + if (!speechTask) { + throw new Error('expected greeting speech task'); + } + await expect(speechTask.result).rejects.toBe(primaryError); + expect(llm.recordingSession.toolChoiceRestorationAttempts).toBe(1); + expect(llm.recordingSession.toolRestorationAttempts).toBe(1); + expect(toolNames(llm.recordingSession.tools)).toEqual(EXPECTED_USER_TURN_TOOLS); + } finally { + await session.close(); + } + }); + + it('preserves the primary failure when tool restoration throws', async () => { + const primaryError = new Error('primary generation failure'); + const llm = new RecordingRealtimeModel({ + generationError: primaryError, + throwToolRestore: 'before', + }); + const agent = new CleanupFailureAgent(); + const session = new AgentSession({ llm }); + + try { + await session.start({ agent }); + await vi.waitFor(() => expect(agent.speech?._tasks).toHaveLength(1)); + const speechTask = agent.speech?._tasks[0]; + if (!speechTask) { + throw new Error('expected greeting speech task'); + } + await expect(speechTask.result).rejects.toBe(primaryError); + expect(llm.recordingSession.toolRestorationAttempts).toBe(1); + } finally { + await session.close(); + } + }); + + it('contains successful-path cleanup failures and restores tools when possible', async () => { + const llm = new RecordingRealtimeModel({ + emptyGeneration: true, + throwToolChoiceRestore: true, + throwToolRestore: 'after', + }); + const agent = new CleanupFailureAgent(); + const session = new AgentSession({ llm }); + + try { + await session.start({ agent }); + await vi.waitFor(() => expect(agent.speech?._tasks).toHaveLength(1)); + const speechTask = agent.speech?._tasks[0]; + if (!speechTask) { + throw new Error('expected greeting speech task'); + } + await expect(speechTask.result).resolves.toBeUndefined(); + expect(llm.recordingSession.toolChoiceRestorationAttempts).toBe(1); + expect(llm.recordingSession.toolRestorationAttempts).toBe(1); + expect(toolNames(llm.recordingSession.tools)).toEqual(EXPECTED_USER_TURN_TOOLS); + } finally { + await session.close(); + } + }); });