From 178cdd57f200a97569e6d7613698290de9a5170f Mon Sep 17 00:00:00 2001 From: Mike Olson Date: Thu, 23 Jul 2026 12:53:42 -0400 Subject: [PATCH 1/2] feat(orchestrator): Surface waiting background work --- .../Adapters/AcpAdapterV2.test.ts | 3268 ++++++++++++++++- .../orchestration-v2/Adapters/AcpAdapterV2.ts | 751 +++- .../Adapters/ClaudeAdapterV2.test.ts | 1848 +++++++++- .../Adapters/ClaudeAdapterV2.ts | 829 ++++- .../Adapters/GrokAdapterV2.test.ts | 19 +- apps/server/src/orchestration-v2/EventSink.ts | 88 + .../FoundationPersistence.test.ts | 223 ++ .../orchestration-v2/ProjectionStore.test.ts | 40 +- .../src/orchestration-v2/ProjectionStore.ts | 87 +- .../src/orchestration-v2/ProviderAdapter.ts | 9 + .../ProviderContinuationService.test.ts | 19 + .../ProviderEventIngestor.test.ts | 268 ++ .../orchestration-v2/ProviderEventIngestor.ts | 21 + .../ProviderRuntimeRecoveryService.test.ts | 365 ++ .../ProviderRuntimeRecoveryService.ts | 141 +- .../ProviderTurnStartService.ts | 18 +- .../RunExecutionService.test.ts | 1212 +++++- .../orchestration-v2/RunExecutionService.ts | 195 +- apps/web/src/components/ChatView.tsx | 22 + apps/web/src/components/Sidebar.logic.test.ts | 80 + apps/web/src/components/Sidebar.logic.ts | 12 + .../chat/MessagesTimeline.logic.test.ts | 63 + .../components/chat/MessagesTimeline.logic.ts | 39 +- .../src/components/chat/MessagesTimeline.tsx | 27 + apps/web/src/hooks/useHandleNewThread.ts | 2 +- .../client-runtime/src/state/entities.test.ts | 154 + packages/client-runtime/src/state/models.ts | 13 +- .../src/state/threadExecution.ts | 24 +- .../contracts/src/orchestrationV2.test.ts | 96 + packages/contracts/src/orchestrationV2.ts | 21 + packages/shared/package.json | 4 + ...chestrationV2PendingBackgroundWork.test.ts | 349 ++ .../orchestrationV2PendingBackgroundWork.ts | 226 ++ 33 files changed, 10036 insertions(+), 497 deletions(-) create mode 100644 packages/shared/src/orchestrationV2PendingBackgroundWork.test.ts create mode 100644 packages/shared/src/orchestrationV2PendingBackgroundWork.ts diff --git a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts index 18b538898d3..a4798f5d606 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts @@ -49,6 +49,8 @@ import { ServerConfig } from "../../config.ts"; import * as McpProviderSession from "../../mcp/McpProviderSession.ts"; import * as AcpSessionRuntime from "../../provider/acp/AcpSessionRuntime.ts"; import { + extractXAiAcpSubagentEndNotice, + extractXAiAcpSubagentUpdate, normalizeXAiAcpToolCallState, registerXAiBackgroundTaskTracking, } from "../../provider/acp/XAiAcpExtension.ts"; @@ -61,6 +63,7 @@ import { import type { ProviderContinuationRequest } from "../ProviderContinuationRequests.ts"; import { AcpProviderCapabilitiesV2, + acpCarryoverTerminalShouldClearContinuation, acpCanonicalJson, acpClaimNativeTransportRequest, acpNativeUserInputRequestMatches, @@ -69,10 +72,12 @@ import { acpPostSettleWakeEvidence, acpPostSettleWakeShouldBuffer, acpProjectedCommandExitCode, + acpTurnStartShouldPreserveContinuation, makeAcpAdapterV2, type AcpAdapterV2ExtensionContext, type AcpAdapterV2Flavor, type AcpAdapterV2RuntimeInput, + type AcpAdapterV2SubagentUpdate, } from "./AcpAdapterV2.ts"; const serverConfigLayer = ServerConfig.layerTest(process.cwd(), { @@ -101,6 +106,53 @@ describe("acpProjectedCommandExitCode", () => { }); }); +describe("ACP continuation ownership", () => { + it("preserves a continuation offered during non-buffered carryover handling", () => { + assert.isFalse( + acpCarryoverTerminalShouldClearContinuation({ + continuationOffered: true, + wakeBufferLength: 0, + }), + ); + assert.isFalse( + acpCarryoverTerminalShouldClearContinuation({ + continuationOffered: false, + wakeBufferLength: 1, + }), + ); + assert.isTrue( + acpCarryoverTerminalShouldClearContinuation({ + continuationOffered: false, + wakeBufferLength: 0, + }), + ); + }); + + it("preserves only user-raced offers that still own buffered wake traffic", () => { + assert.isTrue( + acpTurnStartShouldPreserveContinuation({ + continuationRequested: true, + isContinuationTurn: false, + wakeBufferLength: 1, + }), + ); + assert.isFalse( + acpTurnStartShouldPreserveContinuation({ + continuationRequested: true, + isContinuationTurn: false, + wakeBufferLength: 0, + }), + ); + assert.isFalse( + acpTurnStartShouldPreserveContinuation({ + continuationRequested: true, + isContinuationTurn: true, + wakeBufferLength: 1, + }), + ); + }); +}); + const taskkillPlatformError = (method: string) => PlatformError.systemError({ _tag: "Unknown", module: "taskkill-test", method }); @@ -408,6 +460,10 @@ function makeTurnInput(input: { readonly now: DateTime.Utc; readonly ordinal?: number; readonly modelSelection?: ModelSelection; + /** agent+provider marks a post-settle continuation attach (drains wakeBuffer). */ + readonly messageCreatedBy?: "user" | "agent"; + readonly messageCreationSource?: "web" | "mobile" | "mcp" | "provider" | "server"; + readonly messageText?: string; }): ProviderAdapterV2TurnInput { const ordinal = input.ordinal ?? 1; const suffix = `${input.threadId}:${ordinal}`; @@ -448,10 +504,10 @@ function makeTurnInput(input: { rootNodeId: NodeId.make(`node:${suffix}`), providerThread: input.providerThread, message: { - createdBy: "user", - creationSource: "web", + createdBy: input.messageCreatedBy ?? "user", + creationSource: input.messageCreationSource ?? "web", messageId: MessageId.make(`message:${suffix}`), - text: "test prompt", + text: input.messageText ?? "test prompt", attachments: [], }, modelSelection, @@ -2943,7 +2999,7 @@ describe("AcpAdapterV2", () => { ); it.effect( - "preserveRuntimeOnSettledInterrupt keeps the process alive and carries subagents through a settled steering interrupt", + "pins hasPendingBackgroundWork while carryover holds a live subagent after root settle", () => Effect.gen(function* () { const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; @@ -2957,8 +3013,6 @@ describe("AcpAdapterV2", () => { const protocolEvents = yield* Queue.bounded(256); const instanceId = ProviderInstanceId.make("acp-test"); let subagentPhase: "spawn" | "complete" = "spawn"; - let cancelCalled = false; - let runtimeOrdinalSeen = 0; const adapter = makeAcpAdapterV2({ crypto: yield* Crypto.Crypto, instanceId, @@ -2966,14 +3020,7 @@ describe("AcpAdapterV2", () => { driver: ACP_TEST_DRIVER, capabilities: AcpProviderCapabilitiesV2, deferFinalizeForBackgroundWork: true, - // Hard interrupt flags (stricter than production Grok, which no - // longer sets restartRuntimeOnEveryInterrupt): every interrupt - // would hard-kill the process group without the settled-soft gate - // under test. - restartRuntimeAfterInterrupt: true, - restartRuntimeOnEveryInterrupt: true, - terminateRuntimeProcessGroupOnInterrupt: true, - preserveRuntimeOnSettledInterrupt: true, + enablePostSettleContinuation: true, extractSubagentUpdate: (toolCall) => toolCall.toolCallId !== "tool-call-generic-1" ? undefined @@ -2996,28 +3043,19 @@ describe("AcpAdapterV2", () => { childSessionId: null, result: "SUB_DONE", }, - // No ownDetachedProcessGroup: if the interrupt wrongly takes the - // hard path, terminateProcessGroup is missing and the interrupt - // fails loudly with a poisoned session. makeRuntime: makeMockRuntime({ childProcessSpawner, mockAgentPath, - environment: (runtimeOrdinal) => { - runtimeOrdinalSeen = Math.max(runtimeOrdinalSeen, runtimeOrdinal); - return { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }; - }, + environment: { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }, protocolEvents, - wrapCancel: (cancel) => - Effect.sync(() => { - cancelCalled = true; - }).pipe(Effect.andThen(cancel)), }), }, fileSystem, idAllocator, serverConfig, + continuationRequests: { offer: () => Effect.void }, }); - const threadId = ThreadId.make("thread-acp-settled-soft-steer"); + const threadId = ThreadId.make("thread-acp-carryover-pending-pin"); const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ runtimeMode: "full-access", interactionMode: "default", @@ -3026,10 +3064,16 @@ describe("AcpAdapterV2", () => { const modelSelection = { instanceId, model: "default" } as const; const runtime = yield* adapter.openSession({ threadId, - providerSessionId: ProviderSessionId.make("provider-session-acp-settled-soft-steer"), + providerSessionId: ProviderSessionId.make("provider-session-acp-carryover-pending-pin"), modelSelection, runtimePolicy, }); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "ACP runtime must expose hasPendingBackgroundWork when post-settle continuation is enabled.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; const events = yield* Queue.unbounded(); yield* runtime.events.pipe( Stream.runForEach((event) => Queue.offer(events, event)), @@ -3044,8 +3088,6 @@ describe("AcpAdapterV2", () => { yield* runtime.startTurn( makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), ); - // The still-running subagent defers finalize after session/prompt returns, - // so the interrupt below hits a settled turn held open for background work. yield* Stream.fromQueue(protocolEvents).pipe( Stream.filter( (event) => @@ -3068,24 +3110,20 @@ describe("AcpAdapterV2", () => { .pipe(Effect.forkScoped); yield* TestClock.adjust("10 seconds"); yield* Fiber.join(interruptFiber); - assert.isFalse( - cancelCalled, - "settled soft steer must not send session/cancel (the real Grok CLI kills background subagents on cancel)", - ); - let subagentTurnItemId: string | null = null; let firstTerminalStatus: string | null = null; while (firstTerminalStatus === null) { const event = yield* Queue.take(events); - if (event.type === "turn_item.updated" && event.turnItem.type === "subagent") { - subagentTurnItemId = event.turnItem.id; - } if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { firstTerminalStatus = event.status; } } assert.equal(firstTerminalStatus, "interrupted"); - assert.notEqual(subagentTurnItemId, null); + // Root settled with a live projected subagent in carryover: pin idle release. + assert.isTrue( + yield* hasPendingBackgroundWork, + "carryover live subagent must pin hasPendingBackgroundWork after root settle", + ); subagentPhase = "complete"; const secondNow = yield* DateTime.now; @@ -3099,38 +3137,27 @@ describe("AcpAdapterV2", () => { ordinal: 2, }), ); - // Same runtime process (mock-session-1): a respawn would start - // mock-session-2 and drop the carryover on the session mismatch. const secondProviderTurnId = idAllocator.derive.providerTurn({ driver: ACP_TEST_DRIVER, nativeTurnId: "mock-session-1:turn:2", }); - let carriedItemStatus: string | null = null; let secondTerminalStatus: string | null = null; while (secondTerminalStatus === null) { const event = yield* Queue.take(events); - if ( - event.type === "turn_item.updated" && - event.turnItem.type === "subagent" && - event.turnItem.id === subagentTurnItemId - ) { - carriedItemStatus = event.turnItem.status; - } if (event.type === "turn.terminal" && event.providerTurnId === secondProviderTurnId) { secondTerminalStatus = event.status; } } - assert.equal(carriedItemStatus, "completed"); assert.equal(secondTerminalStatus, "completed"); - assert.equal( - runtimeOrdinalSeen, - 1, - "settled soft steer must not respawn the ACP runtime process", + // Carryover is consumed into the next turn and terminalized; pin clears. + assert.isFalse( + yield* hasPendingBackgroundWork, + "hasPendingBackgroundWork must clear after carryover subagent terminals", ); }).pipe(Effect.provide(testLayer), Effect.scoped), ); - it.live("preserveRuntimeOnSettledInterrupt does not soften a mid-prompt steering interrupt", () => + it.effect("handles a child terminal after carryover rehydrate", () => Effect.gen(function* () { const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; const fileSystem = yield* FileSystem.FileSystem; @@ -3141,35 +3168,74 @@ describe("AcpAdapterV2", () => { new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), ); const protocolEvents = yield* Queue.bounded(256); + const secondPromptWireReturned = yield* Deferred.make(); + const releaseSecondPromptCompletion = yield* Deferred.make(); const instanceId = ProviderInstanceId.make("acp-test"); + const childSessionId = "mock-child-session-active-carryover"; + let promptCount = 0; + let subagentPhase: "spawn" | "complete" = "spawn"; + type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; + let sessionUpdateHandler: Parameters[0] | undefined; const adapter = makeAcpAdapterV2({ crypto: yield* Crypto.Crypto, instanceId, flavor: { driver: ACP_TEST_DRIVER, capabilities: AcpProviderCapabilitiesV2, - restartRuntimeAfterInterrupt: true, - // Local hard-flavor gate: production Grok no longer sets - // restartRuntimeOnEveryInterrupt, but when a flavor does, the - // settled-soft gate must not leak onto an unsettled prompt. - restartRuntimeOnEveryInterrupt: true, - terminateRuntimeProcessGroupOnInterrupt: true, - preserveRuntimeOnSettledInterrupt: true, - // No ownDetachedProcessGroup: the expected hard path fails loudly - // on the missing terminateProcessGroup, proving the settled-soft - // gate did not apply to an unsettled prompt. + deferFinalizeForBackgroundWork: true, + enablePostSettleContinuation: true, + extractSubagentUpdate: (toolCall) => + toolCall.toolCallId !== "tool-call-generic-1" + ? undefined + : subagentPhase === "spawn" + ? { + nativeTaskId: "task-generic-1", + prompt: "background subagent", + title: "background subagent", + model: null, + status: "running", + childSessionId, + result: null, + } + : { + nativeTaskId: "task-generic-1", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId, + result: "SUB_DONE", + }, makeRuntime: makeMockRuntime({ childProcessSpawner, mockAgentPath, - environment: { T3_ACP_HANG_PROMPT_FOREVER: "1" }, + environment: { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }, protocolEvents, + wrapRuntime: (runtime) => ({ + ...runtime, + handleSessionUpdate: (handler) => + Effect.sync(() => { + sessionUpdateHandler = handler; + }).pipe(Effect.andThen(runtime.handleSessionUpdate(handler))), + prompt: (payload) => + Effect.gen(function* () { + const currentPrompt = ++promptCount; + const result = yield* runtime.prompt(payload); + if (currentPrompt === 2) { + yield* Deferred.succeed(secondPromptWireReturned, undefined); + yield* Deferred.await(releaseSecondPromptCompletion); + } + return result; + }), + }), }), }, fileSystem, idAllocator, serverConfig, + continuationRequests: { offer: () => Effect.void }, }); - const threadId = ThreadId.make("thread-acp-unsettled-steer-stays-hard"); + const threadId = ThreadId.make("thread-acp-active-carryover-pending-pin"); const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ runtimeMode: "full-access", interactionMode: "default", @@ -3179,45 +3245,165 @@ describe("AcpAdapterV2", () => { const runtime = yield* adapter.openSession({ threadId, providerSessionId: ProviderSessionId.make( - "provider-session-acp-unsettled-steer-stays-hard", + "provider-session-acp-active-carryover-pending-pin", ), modelSelection, runtimePolicy, }); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "ACP runtime must expose hasPendingBackgroundWork when post-settle continuation is enabled.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); const providerThread = yield* runtime.ensureThread({ threadId, modelSelection, runtimePolicy, }); const now = yield* DateTime.now; - yield* runtime - .startTurn( - makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 1 }), - ) - .pipe(Effect.forkScoped); + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); yield* Stream.fromQueue(protocolEvents).pipe( Stream.filter( (event) => - event.direction === "outgoing" && rawProtocolMethod(event) === "session/prompt", + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes('"stopReason"'), ), Stream.runHead, ); - const providerTurnId = idAllocator.derive.providerTurn({ + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const firstProviderTurnId = idAllocator.derive.providerTurn({ driver: ACP_TEST_DRIVER, nativeTurnId: "mock-session-1:turn:1", }); - const interruptExit = yield* runtime - .interruptTurn({ providerThread, providerTurnId }) - .pipe(Effect.exit); - if (Exit.isSuccess(interruptExit)) { - assert.fail("mid-prompt steering interrupt must still take the hard teardown path"); + const interruptFiber = yield* runtime + .interruptTurn({ providerThread, providerTurnId: firstProviderTurnId }) + .pipe(Effect.forkScoped); + yield* TestClock.adjust("10 seconds"); + yield* Fiber.join(interruptFiber); + + let firstTerminalStatus: string | null = null; + while (firstTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + firstTerminalStatus = event.status; + } } - assert.include(Cause.pretty(interruptExit.cause), "session is poisoned"); + assert.equal(firstTerminalStatus, "interrupted"); + assert.isTrue(yield* hasPendingBackgroundWork); + assert.isDefined(sessionUpdateHandler, "session update handler must be wired"); + + const secondNow = yield* DateTime.now; + const secondTurnFiber = yield* runtime + .startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: secondNow, + ordinal: 2, + }), + ) + .pipe(Effect.forkScoped); + yield* Deferred.await(secondPromptWireReturned); + assert.isTrue( + yield* hasPendingBackgroundWork, + "rehydrated live subagent must pin from activeTurn", + ); + while (Option.isSome(yield* Queue.poll(events))) { + // Discard setup events so the assertions below cover only child-session + // tool traffic after rehydration. + } + + subagentPhase = "complete"; + yield* sessionUpdateHandler!({ + sessionId: childSessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-1", + title: "background subagent", + kind: "other", + status: "completed", + rawOutput: { content: "SUB_DONE" }, + }, + }); + // Duplicate terminal replay is harmless, and an older running frame cannot + // resurrect the completed lineage. + yield* sessionUpdateHandler!({ + sessionId: childSessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-1", + title: "background subagent", + kind: "other", + status: "completed", + rawOutput: { content: "SUB_DONE" }, + }, + }); + subagentPhase = "spawn"; + yield* sessionUpdateHandler!({ + sessionId: childSessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-1", + title: "background subagent", + kind: "other", + status: "in_progress", + rawOutput: {}, + }, + }); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + assert.isFalse( + yield* hasPendingBackgroundWork, + "active-turn pin must clear after the child-session terminal is projected", + ); + let completedSubagentUpdates = 0; + let resurrectedSubagentUpdates = 0; + let normalChildToolUpdates = 0; + let polled = yield* Queue.poll(events); + while (Option.isSome(polled)) { + const event = polled.value; + if (event.type === "turn_item.updated") { + if (event.turnItem.type === "subagent") { + if (event.turnItem.status === "completed") { + completedSubagentUpdates += 1; + } + if ( + event.turnItem.status === "running" && + event.turnItem.nativeItemRef?.nativeId === "task-generic-1" + ) { + resurrectedSubagentUpdates += 1; + } + } else if (event.turnItem.nativeItemRef?.nativeId === "tool-call-generic-1") { + normalChildToolUpdates += 1; + } + } + polled = yield* Queue.poll(events); + } + assert.equal(completedSubagentUpdates, 1); + assert.equal(resurrectedSubagentUpdates, 0); + assert.equal(normalChildToolUpdates, 0); + + yield* Deferred.succeed(releaseSecondPromptCompletion, undefined); + yield* Fiber.join(secondTurnFiber); }).pipe(Effect.provide(testLayer), Effect.scoped), ); - it.live( - "soft mid-prompt interrupt cancels in place, reuses the runtime, and tracks cancel-backgrounded work", + it.effect( + "projects a child terminal in the completed-root finalization window exactly once", () => Effect.gen(function* () { const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; @@ -3229,44 +3415,2334 @@ describe("AcpAdapterV2", () => { new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), ); const protocolEvents = yield* Queue.bounded(256); - const continuationRequests: Array = []; - const instanceId = ProviderInstanceId.make("acp-test"); - let cancelCalled = false; - let runtimeOrdinalSeen = 0; - const adapter = makeAcpAdapterV2({ - crypto: yield* Crypto.Crypto, - instanceId, - flavor: { - driver: ACP_TEST_DRIVER, - capabilities: AcpProviderCapabilitiesV2, - enablePostSettleContinuation: true, - // Production Grok interrupt flags: hard teardown only with - // requestRuntimeRestart (user Stop). Without - // restartRuntimeOnEveryInterrupt a mid-prompt steering interrupt - // stays soft: session/cancel, same process, session reuse. - restartRuntimeAfterInterrupt: true, - terminateRuntimeProcessGroupOnInterrupt: true, - preserveRuntimeOnSettledInterrupt: true, - registerExtensions: ({ runtime: extensionRuntime, applyBackgroundTaskMutation }) => - registerXAiBackgroundTaskTracking(extensionRuntime, applyBackgroundTaskMutation), - // No ownDetachedProcessGroup: if the interrupt wrongly takes the - // hard path, terminateProcessGroup is missing and the interrupt - // fails loudly with a poisoned session. - makeRuntime: makeMockRuntime({ - childProcessSpawner, - mockAgentPath, - environment: (runtimeOrdinal) => { - runtimeOrdinalSeen = Math.max(runtimeOrdinalSeen, runtimeOrdinal); - return { - T3_ACP_EMIT_RUNNING_COMMAND_THEN_HANG_FIRST_PROMPT: "1", - T3_ACP_EMIT_TASK_BACKGROUNDED_AFTER_CANCEL: "1", - }; + const baseClock = yield* Clock.Clock; + const finalizationClockRead = yield* Deferred.make(); + const releaseFinalizationClockRead = yield* Deferred.make(); + let blockNextClockRead = false; + const blockingClock: Clock.Clock = { + ...baseClock, + currentTimeMillis: Effect.suspend(() => { + if (!blockNextClockRead) return baseClock.currentTimeMillis; + blockNextClockRead = false; + return Deferred.succeed(finalizationClockRead, undefined).pipe( + Effect.andThen(Deferred.await(releaseFinalizationClockRead)), + Effect.andThen(baseClock.currentTimeMillis), + ); + }), + }; + yield* Effect.gen(function* () { + const instanceId = ProviderInstanceId.make("acp-test"); + const firstChildSessionId = "mock-child-session-finalize-window-first"; + const secondChildSessionId = "mock-child-session-finalize-window-second"; + let firstSubagentStatus: "running" | "completed" = "running"; + // Keep the pending-work pin without blocking the deferred-finalize + // timer, so the test can stop inside finalizeTurn deterministically. + let secondSubagentStatus: "waiting" | "completed" = "waiting"; + type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; + let sessionUpdateHandler: + | Parameters[0] + | undefined; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + enablePostSettleContinuation: true, + settleRootTurnWhenIdle: true, + extractSubagentUpdate: (toolCall) => { + if (toolCall.toolCallId === "tool-call-generic-1") { + return { + nativeTaskId: "task-finalize-window-first", + prompt: "first background subagent", + title: "first background subagent", + model: null, + status: firstSubagentStatus, + childSessionId: firstChildSessionId, + result: firstSubagentStatus === "completed" ? "FIRST_DONE" : null, + suppressNormalTool: true, + }; + } + if (toolCall.toolCallId === "tool-call-generic-2") { + return { + nativeTaskId: "task-finalize-window-second", + prompt: "second background subagent", + title: "second background subagent", + model: null, + status: secondSubagentStatus, + childSessionId: secondChildSessionId, + result: secondSubagentStatus === "completed" ? "SECOND_DONE" : null, + suppressNormalTool: true, + } as AcpAdapterV2SubagentUpdate; + } + return undefined; }, - protocolEvents, - wrapCancel: (cancel) => - Effect.sync(() => { - cancelCalled = true; - }).pipe(Effect.andThen(cancel)), + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }, + protocolEvents, + wrapRuntime: (runtime) => ({ + ...runtime, + handleSessionUpdate: (handler) => + Effect.sync(() => { + sessionUpdateHandler = handler; + }).pipe(Effect.andThen(runtime.handleSessionUpdate(handler))), + }), + }), + }, + fileSystem, + idAllocator, + serverConfig, + continuationRequests: { offer: () => Effect.void }, + }); + const threadId = ThreadId.make("thread-acp-subagent-finalize-window"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-subagent-finalize-window", + ), + modelSelection, + runtimePolicy, + }); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "ACP runtime must expose hasPendingBackgroundWork when post-settle continuation is enabled.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes('"stopReason"'), + ), + Stream.runHead, + ); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + yield* TestClock.adjust("1 second"); + assert.isDefined(sessionUpdateHandler, "session update handler must be wired"); + + firstSubagentStatus = "completed"; + yield* sessionUpdateHandler!({ + sessionId: "mock-session-1", + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-1", + title: "first background subagent", + kind: "other", + status: "completed", + rawOutput: { content: "FIRST_DONE" }, + }, + }).pipe(Effect.provideService(Clock.Clock, blockingClock)); + for (let attempt = 0; attempt < 10; attempt += 1) { + yield* Effect.yieldNow; + } + + yield* sessionUpdateHandler!({ + sessionId: "mock-session-1", + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-2", + title: "second background subagent", + kind: "other", + status: "in_progress", + rawOutput: {}, + }, + }); + assert.isTrue(yield* hasPendingBackgroundWork); + + while (Option.isSome(yield* Queue.poll(events))) { + // Discard setup and first-subagent events. + } + + blockNextClockRead = true; + const adjustFiber = yield* TestClock.adjust("3 seconds").pipe(Effect.forkScoped); + yield* Deferred.await(finalizationClockRead); + secondSubagentStatus = "completed"; + yield* sessionUpdateHandler!({ + sessionId: secondChildSessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-2", + title: "second background subagent", + kind: "other", + status: "completed", + rawOutput: { content: "SECOND_DONE" }, + }, + }); + yield* Deferred.succeed(releaseFinalizationClockRead, undefined); + yield* Fiber.join(adjustFiber); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const providerTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + let completedSubagentUpdates = 0; + let rootTerminalStatus: string | null = null; + let polled = yield* Queue.poll(events); + while (Option.isSome(polled)) { + const event = polled.value; + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.nativeItemRef?.nativeId === "task-finalize-window-second" && + event.turnItem.status === "completed" + ) { + completedSubagentUpdates += 1; + } + if (event.type === "turn.terminal" && event.providerTurnId === providerTurnId) { + rootTerminalStatus = event.status; + } + polled = yield* Queue.poll(events); + } + assert.equal(rootTerminalStatus, "completed"); + assert.equal(completedSubagentUpdates, 1); + assert.isFalse( + yield* hasPendingBackgroundWork, + "finalize-window terminal must clear the carryover pin", + ); + }).pipe(Effect.provideService(Clock.Clock, blockingClock)); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect( + "defers an interrupted pending-spawn carryover terminal until the next observable attach", + () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const continuationRequests: Array = []; + const instanceId = ProviderInstanceId.make("acp-test"); + const childSessionId = "mock-child-session-post-settle"; + let subagentPhase: "spawn" | "complete" = "spawn"; + type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; + let sessionUpdateHandler: Parameters[0] | undefined; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + enablePostSettleContinuation: true, + extractSubagentUpdate: (toolCall) => + toolCall.toolCallId !== "tool-call-generic-1" + ? undefined + : subagentPhase === "spawn" + ? { + nativeTaskId: "task-generic-1", + prompt: "background subagent", + title: "background subagent", + model: null, + status: "pending", + childSessionId, + result: null, + } + : { + nativeTaskId: "task-generic-1", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId, + result: "SUB_DONE", + }, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }, + protocolEvents, + wrapRuntime: (runtime) => ({ + ...runtime, + handleSessionUpdate: (handler) => + Effect.sync(() => { + sessionUpdateHandler = handler; + }).pipe(Effect.andThen(runtime.handleSessionUpdate(handler))), + }), + }), + }, + fileSystem, + idAllocator, + serverConfig, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + }); + const threadId = ThreadId.make("thread-acp-carryover-child-session-post-settle"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-carryover-child-session-post-settle", + ), + modelSelection, + runtimePolicy, + }); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "ACP runtime must expose hasPendingBackgroundWork when post-settle continuation is enabled.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes('"stopReason"'), + ), + Stream.runHead, + ); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const firstProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + const interruptFiber = yield* runtime + .interruptTurn({ providerThread, providerTurnId: firstProviderTurnId }) + .pipe(Effect.forkScoped); + yield* TestClock.adjust("10 seconds"); + yield* Fiber.join(interruptFiber); + + let firstTerminalStatus: string | null = null; + while (firstTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + firstTerminalStatus = event.status; + } + } + assert.equal(firstTerminalStatus, "interrupted"); + assert.isTrue( + yield* hasPendingBackgroundWork, + "carryover live subagent must pin hasPendingBackgroundWork after root settle", + ); + assert.isDefined(sessionUpdateHandler, "session update handler must be wired"); + + // The interrupted root's subscription is already closed in execution + // service, so the adapter must retain this terminal without claiming it + // reached the projection. + subagentPhase = "complete"; + yield* sessionUpdateHandler!({ + sessionId: childSessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-1", + title: "background subagent", + kind: "other", + status: "completed", + rawOutput: { content: "SUB_DONE" }, + }, + }); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + let completedBeforeAttach = 0; + let polled = yield* Queue.poll(events); + while (Option.isSome(polled)) { + const event = polled.value; + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedBeforeAttach += 1; + } + polled = yield* Queue.poll(events); + } + assert.equal( + completedBeforeAttach, + 0, + "closed interrupted subscription must not be treated as projected", + ); + assert.isTrue( + yield* hasPendingBackgroundWork, + "terminal carryover must stay pinned until an attach can project it", + ); + assert.lengthOf( + continuationRequests, + 0, + "child-session completion must not open a root continuation", + ); + + const attachNow = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: attachNow, + ordinal: 2, + messageCreatedBy: "agent", + messageCreationSource: "provider", + messageText: "Attach deferred completion.", + }), + ); + const attachProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:2", + }); + let attachTerminal: string | null = null; + let completedAfterAttach = 0; + while (attachTerminal === null) { + const event = yield* Queue.take(events); + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedAfterAttach += 1; + } + if (event.type === "turn.terminal" && event.providerTurnId === attachProviderTurnId) { + attachTerminal = event.status; + } + } + assert.equal(attachTerminal, "completed"); + assert.equal(completedAfterAttach, 1); + assert.isFalse(yield* hasPendingBackgroundWork); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect("projects completed-root carryover eagerly and drain cannot resurrect it", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const continuationRequests: Array = []; + const instanceId = ProviderInstanceId.make("acp-test"); + const childSessionId = "019f44a6-4820-7402-925d-bc862ee711dd"; + let subagentPhase: "spawn" | "complete" = "spawn"; + type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; + let sessionUpdateHandler: Parameters[0] | undefined; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + enablePostSettleContinuation: true, + normalizeToolCall: normalizeXAiAcpToolCallState, + extractSubagentUpdate: (toolCall) => + extractXAiAcpSubagentUpdate(toolCall) ?? + (toolCall.toolCallId !== "tool-call-generic-1" + ? undefined + : subagentPhase === "spawn" + ? { + nativeTaskId: "task-generic-1", + prompt: "background subagent", + title: "background subagent", + model: null, + status: "running", + childSessionId, + result: null, + } + : { + nativeTaskId: "task-generic-1", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId, + result: "SUB_DONE", + }), + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }, + protocolEvents, + wrapRuntime: (runtime) => ({ + ...runtime, + handleSessionUpdate: (handler) => + Effect.sync(() => { + sessionUpdateHandler = handler; + }).pipe(Effect.andThen(runtime.handleSessionUpdate(handler))), + }), + }), + }, + fileSystem, + idAllocator, + serverConfig, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + }); + const threadId = ThreadId.make("thread-acp-completed-root-eager-carryover"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-completed-root-eager-carryover", + ), + modelSelection, + runtimePolicy, + }); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "ACP runtime must expose hasPendingBackgroundWork when post-settle continuation is enabled.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes('"stopReason"'), + ), + Stream.runHead, + ); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + yield* TestClock.adjust("1 second"); + + const firstProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + let firstTerminalStatus: string | null = null; + while (firstTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + firstTerminalStatus = event.status; + } + } + assert.equal(firstTerminalStatus, "completed"); + assert.isTrue(yield* hasPendingBackgroundWork); + assert.isDefined(sessionUpdateHandler, "session update handler must be wired"); + + // A production Grok spawn ACK has raw tool status completed but extracts + // as a running subagent with its original non-empty prompt. It buffers and + // offers a continuation after root settlement. + yield* sessionUpdateHandler!({ + sessionId: "mock-session-1", + update: { + sessionUpdate: "tool_call", + toolCallId: "tool-call-generic-1", + title: "spawn_subagent", + kind: "other", + status: "completed", + rawInput: { + description: "background subagent", + prompt: "background subagent", + subagent_type: "general-purpose", + }, + rawOutput: { + type: "Text", + text: [ + "Subagent started in background.", + `subagent_id: ${childSessionId}`, + "type: general-purpose", + "description: background subagent", + "", + `Use get_command_or_subagent_output with task_ids=["${childSessionId}"] and timeout_ms to wait for results.`, + ].join("\n"), + }, + }, + }); + + // The child-session terminal bypasses the root wake buffer and projects + // eagerly through the still-observable completed-root subscriber. + subagentPhase = "complete"; + yield* sessionUpdateHandler!({ + sessionId: childSessionId, + update: { + sessionUpdate: "tool_call_update" as const, + toolCallId: "tool-call-generic-1", + title: "background subagent", + kind: "other" as const, + status: "completed" as const, + rawOutput: { content: "SUB_DONE" }, + }, + }); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + let eagerCompletedUpdates = 0; + let eagerRunningUpdates = 0; + let polled = yield* Queue.poll(events); + while (Option.isSome(polled)) { + const event = polled.value; + if (event.type === "turn_item.updated" && event.turnItem.type === "subagent") { + if (event.turnItem.status === "completed") eagerCompletedUpdates += 1; + if (event.turnItem.status === "running") eagerRunningUpdates += 1; + } + polled = yield* Queue.poll(events); + } + assert.equal(eagerCompletedUpdates, 1, "completed root must project before any attach"); + assert.equal(eagerRunningUpdates, 0); + assert.lengthOf(continuationRequests, 1); + assert.isTrue(yield* hasPendingBackgroundWork, "buffered spawn ACK still requires a drain"); + + const continuationNow = yield* DateTime.now; + const continuationInput = makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: continuationNow, + ordinal: 2, + messageCreatedBy: "agent", + messageCreationSource: "provider", + messageText: "Background task completed.", + }); + yield* runtime.startTurn(continuationInput); + yield* Effect.yieldNow; + let replayedLineages = 0; + let replayedSubagentUpdates = 0; + let replayedTurnItems = 0; + polled = yield* Queue.poll(events); + while (Option.isSome(polled)) { + const event = polled.value; + if (event.type === "app_thread.created") { + replayedLineages += 1; + } + if (event.type === "subagent.updated" && event.subagent.runId === continuationInput.runId) { + replayedSubagentUpdates += 1; + } + if ( + event.type === "turn_item.updated" && + event.turnItem.runId === continuationInput.runId + ) { + replayedTurnItems += 1; + } + polled = yield* Queue.poll(events); + } + assert.equal(replayedLineages, 0, "buffered spawn ACK must not create a second lineage"); + assert.equal( + replayedSubagentUpdates, + 0, + "buffered spawn ACK must not re-open the terminal subagent", + ); + assert.equal( + replayedTurnItems, + 0, + "buffered spawn ACK must not create a continuation-owned turn item", + ); + assert.isFalse(yield* hasPendingBackgroundWork); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect( + "user attach flushes a deferred terminal but leaves wake traffic for its continuation", + () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const continuationRequests: Array = []; + const bufferedAssistantText = "BUFFERED_WAKE_AFTER_USER_ATTACH"; + const instanceId = ProviderInstanceId.make("acp-test"); + let subagentPhase: "spawn" | "complete" = "spawn"; + type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; + let sessionUpdateHandler: Parameters[0] | undefined; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + enablePostSettleContinuation: true, + extractSubagentUpdate: (toolCall) => + toolCall.toolCallId !== "tool-call-generic-1" + ? undefined + : subagentPhase === "spawn" + ? { + nativeTaskId: "task-generic-1", + prompt: "background subagent", + title: "background subagent", + model: null, + status: "running", + childSessionId: null, + result: null, + } + : { + nativeTaskId: "task-generic-1", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId: null, + result: "SUB_DONE", + }, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }, + protocolEvents, + wrapRuntime: (runtime) => ({ + ...runtime, + handleSessionUpdate: (handler) => + Effect.sync(() => { + sessionUpdateHandler = handler; + }).pipe(Effect.andThen(runtime.handleSessionUpdate(handler))), + }), + }), + }, + fileSystem, + idAllocator, + serverConfig, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + }); + const threadId = ThreadId.make("thread-acp-carryover-root-session-continuation"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-carryover-root-session-continuation", + ), + modelSelection, + runtimePolicy, + }); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "ACP runtime must expose hasPendingBackgroundWork when post-settle continuation is enabled.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes('"stopReason"'), + ), + Stream.runHead, + ); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const firstProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + const interruptFiber = yield* runtime + .interruptTurn({ providerThread, providerTurnId: firstProviderTurnId }) + .pipe(Effect.forkScoped); + yield* TestClock.adjust("10 seconds"); + yield* Fiber.join(interruptFiber); + + let firstTerminalStatus: string | null = null; + while (firstTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + firstTerminalStatus = event.status; + } + } + assert.equal(firstTerminalStatus, "interrupted"); + assert.isTrue( + yield* hasPendingBackgroundWork, + "carryover live subagent must pin hasPendingBackgroundWork after root settle", + ); + assert.isDefined(sessionUpdateHandler, "session update handler must be wired"); + + // Root-session terminal tool is wake evidence and will buffer: in-memory + // only so the continuation drain projects exactly once. + subagentPhase = "complete"; + yield* sessionUpdateHandler!({ + sessionId: "mock-session-1", + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-1", + title: "background subagent", + kind: "other", + status: "completed", + rawOutput: { content: "SUB_DONE" }, + }, + }); + yield* sessionUpdateHandler!({ + sessionId: "mock-session-1", + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: bufferedAssistantText }, + }, + }); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + let completedBeforeDrain = 0; + let polled = yield* Queue.poll(events); + while (Option.isSome(polled)) { + const event = polled.value; + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedBeforeDrain += 1; + } + polled = yield* Queue.poll(events); + } + assert.equal( + completedBeforeDrain, + 0, + "interrupted root-session terminal must wait for an observable attach", + ); + assert.lengthOf( + continuationRequests, + 1, + "root-session post-settle subagent terminal must offer a continuation", + ); + assert.isTrue( + yield* hasPendingBackgroundWork, + "interrupted unprojected terminal must pin hasPendingBackgroundWork until attach", + ); + + const userTurnNow = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: userTurnNow, + ordinal: 2, + messageText: "What finished while the prior turn was settling?", + }), + ); + yield* TestClock.adjust("3 seconds"); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const userProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:2", + }); + let userTurnTerminal: string | null = null; + let completedSubagentTurnItems = 0; + let bufferedTextSeenInUserTurn = false; + while (userTurnTerminal === null) { + const event = yield* Queue.take(events); + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedSubagentTurnItems += 1; + } + if ( + event.type === "message.updated" && + event.message.role === "assistant" && + event.message.text.includes(bufferedAssistantText) + ) { + bufferedTextSeenInUserTurn = true; + } + if (event.type === "turn.terminal" && event.providerTurnId === userProviderTurnId) { + userTurnTerminal = event.status; + } + } + assert.equal(userTurnTerminal, "completed"); + assert.equal( + completedSubagentTurnItems, + 1, + "user attach must project the deferred terminal subagent turn item once", + ); + assert.isFalse(bufferedTextSeenInUserTurn, "user attach must not drain wake traffic"); + assert.isTrue( + yield* hasPendingBackgroundWork, + "wake traffic must remain pinned for the already-dispatched continuation", + ); + + const continuationNow = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: continuationNow, + ordinal: 3, + messageCreatedBy: "agent", + messageCreationSource: "provider", + messageText: "Background task completed.", + }), + ); + yield* TestClock.adjust("3 seconds"); + const continuationProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:3", + }); + let continuationTerminal: string | null = null; + let bufferedTextSeenInContinuation = false; + while (continuationTerminal === null) { + const event = yield* Queue.take(events); + if ( + event.type === "message.updated" && + event.message.role === "assistant" && + event.message.text.includes(bufferedAssistantText) + ) { + bufferedTextSeenInContinuation = true; + } + if ( + event.type === "turn.terminal" && + event.providerTurnId === continuationProviderTurnId + ) { + continuationTerminal = event.status; + } + } + assert.equal(continuationTerminal, "completed"); + assert.isTrue( + bufferedTextSeenInContinuation, + "queued continuation must drain the wake content after the user run", + ); + assert.isFalse(yield* hasPendingBackgroundWork); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect( + "projects a carryover terminal when an in-turn-handled tool is re-reported post-settle", + () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const continuationRequests: Array = []; + const protocolEvents = yield* Queue.bounded(256); + const promptWireReturned = yield* Deferred.make(); + const releasePromptCompletion = yield* Deferred.make(); + const instanceId = ProviderInstanceId.make("acp-test"); + let subagentPhase: "spawn" | "complete" = "spawn"; + type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; + let sessionUpdateHandler: Parameters[0] | undefined; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + enablePostSettleContinuation: true, + extractBackgroundTaskId: (toolCall) => + toolCall.toolCallId === "tool-call-generic-1" ? "task-generic-1" : undefined, + extractSubagentUpdate: (toolCall) => + toolCall.toolCallId !== "tool-call-generic-1" + ? undefined + : subagentPhase === "spawn" + ? { + nativeTaskId: "task-generic-1", + prompt: "background subagent", + title: "background subagent", + model: null, + status: "running", + childSessionId: null, + result: null, + } + : { + nativeTaskId: "task-generic-1", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId: null, + result: "SUB_DONE", + }, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }, + protocolEvents, + wrapRuntime: (runtime) => ({ + ...runtime, + handleSessionUpdate: (handler) => + Effect.sync(() => { + sessionUpdateHandler = handler; + }).pipe(Effect.andThen(runtime.handleSessionUpdate(handler))), + prompt: (payload) => + Effect.gen(function* () { + const result = yield* runtime.prompt(payload); + yield* Deferred.succeed(promptWireReturned, undefined); + yield* Deferred.await(releasePromptCompletion); + return result; + }), + }), + }), + }, + fileSystem, + idAllocator, + serverConfig, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + }); + const threadId = ThreadId.make("thread-acp-carryover-already-handled-re-report"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-carryover-already-handled-re-report", + ), + modelSelection, + runtimePolicy, + }); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "ACP runtime must expose hasPendingBackgroundWork when post-settle continuation is enabled.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + yield* Deferred.await(promptWireReturned); + assert.isDefined(sessionUpdateHandler, "session update handler must be wired"); + + // The root turn consumes a terminal re-report while the native prompt is + // still open. The subagent flavor keeps its carried lineage running. + yield* sessionUpdateHandler!({ + sessionId: "mock-session-1", + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-1", + title: "background subagent", + kind: "other", + status: "completed", + rawOutput: { content: "IN_TURN_RESULT" }, + }, + }); + yield* Deferred.succeed(releasePromptCompletion, undefined); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const firstProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + const interruptFiber = yield* runtime + .interruptTurn({ providerThread, providerTurnId: firstProviderTurnId }) + .pipe(Effect.forkScoped); + yield* TestClock.adjust("10 seconds"); + yield* Fiber.join(interruptFiber); + + let firstTerminalStatus: string | null = null; + while (firstTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + firstTerminalStatus = event.status; + } + } + assert.equal(firstTerminalStatus, "interrupted"); + assert.isTrue( + yield* hasPendingBackgroundWork, + "live carryover must remain pending after the superseded root settles", + ); + + // The same tool is now a terminal carryover update. Since it was handled + // in-turn, bufferPostSettleWake drops it instead of offering a drain. + subagentPhase = "complete"; + yield* sessionUpdateHandler!({ + sessionId: "mock-session-1", + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-1", + title: "background subagent", + kind: "other", + status: "completed", + rawOutput: { content: "SUB_DONE" }, + }, + }); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + let completedSubagentTurnItems = 0; + let polled = yield* Queue.poll(events); + while (Option.isSome(polled)) { + const event = polled.value; + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedSubagentTurnItems += 1; + } + polled = yield* Queue.poll(events); + } + assert.equal( + completedSubagentTurnItems, + 0, + "an interrupted root's non-buffered re-report must wait for an observable attach", + ); + assert.lengthOf( + continuationRequests, + 0, + "an in-turn-handled re-report must not offer a continuation", + ); + assert.isTrue( + yield* hasPendingBackgroundWork, + "the unprojected terminal carryover must remain pinned", + ); + + const attachNow = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: attachNow, + ordinal: 2, + messageCreatedBy: "agent", + messageCreationSource: "provider", + messageText: "Attach deferred completion.", + }), + ); + const attachProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:2", + }); + let attachTerminal: string | null = null; + while (attachTerminal === null) { + const event = yield* Queue.take(events); + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedSubagentTurnItems += 1; + } + if (event.type === "turn.terminal" && event.providerTurnId === attachProviderTurnId) { + attachTerminal = event.status; + } + } + assert.equal(completedSubagentTurnItems, 1); + assert.isFalse(yield* hasPendingBackgroundWork); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect("projects an interrupted root-session end notice at the next attach", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const continuationRequests: Array = []; + const instanceId = ProviderInstanceId.make("acp-test"); + const childSessionId = "019f5470-bf92-7a90-afb3-5a6cea5b34a3"; + let subagentPhase: "spawn" | "complete" = "spawn"; + type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; + let sessionUpdateHandler: Parameters[0] | undefined; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + enablePostSettleContinuation: true, + extractSubagentEndNotice: (text) => { + // Prefer the production parser; fall back only if the harness text + // is too short for its UUID + outcome rules. + const parsed = extractXAiAcpSubagentEndNotice(text); + if (parsed !== undefined) return parsed; + if (!text.includes(childSessionId)) return undefined; + if (/completed successfully/i.test(text)) { + return { childSessionId, status: "completed" as const }; + } + return undefined; + }, + extractSubagentUpdate: (toolCall) => + toolCall.toolCallId !== "tool-call-generic-1" + ? undefined + : subagentPhase === "spawn" + ? { + nativeTaskId: "task-generic-1", + prompt: "background subagent", + title: "background subagent", + model: null, + status: "running", + childSessionId, + result: null, + } + : { + nativeTaskId: "task-generic-1", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId, + result: "SUB_DONE", + }, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }, + protocolEvents, + wrapRuntime: (runtime) => ({ + ...runtime, + handleSessionUpdate: (handler) => + Effect.sync(() => { + sessionUpdateHandler = handler; + }).pipe(Effect.andThen(runtime.handleSessionUpdate(handler))), + }), + }), + }, + fileSystem, + idAllocator, + serverConfig, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + }); + const threadId = ThreadId.make("thread-acp-carryover-root-end-notice"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-carryover-root-end-notice"), + modelSelection, + runtimePolicy, + }); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "ACP runtime must expose hasPendingBackgroundWork when post-settle continuation is enabled.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes('"stopReason"'), + ), + Stream.runHead, + ); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const firstProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + const interruptFiber = yield* runtime + .interruptTurn({ providerThread, providerTurnId: firstProviderTurnId }) + .pipe(Effect.forkScoped); + yield* TestClock.adjust("10 seconds"); + yield* Fiber.join(interruptFiber); + + let firstTerminalStatus: string | null = null; + while (firstTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + firstTerminalStatus = event.status; + } + } + assert.equal(firstTerminalStatus, "interrupted"); + assert.isTrue( + yield* hasPendingBackgroundWork, + "carryover live subagent must pin hasPendingBackgroundWork after root settle", + ); + assert.isDefined(sessionUpdateHandler, "session update handler must be wired"); + + // End notices are root user_message_chunk text and never buffer. The + // interrupted root still cannot project them until the next subscription + // attaches. + yield* sessionUpdateHandler!({ + sessionId: "mock-session-1", + update: { + sessionUpdate: "user_message_chunk", + content: { + type: "text", + text: `Background subagent "${childSessionId}" (general-purpose: "background subagent") completed successfully.`, + }, + }, + }); + // Do not use Effect.timeout under TestClock; poll after yielding so a + // missed projection fails the assertion instead of hanging the suite. + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + let completedSubagentTurnItems = 0; + let polled = yield* Queue.poll(events); + while (Option.isSome(polled)) { + const event = polled.value; + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedSubagentTurnItems += 1; + } + polled = yield* Queue.poll(events); + } + assert.equal( + completedSubagentTurnItems, + 0, + "root-session end notice must remain memory-only after interrupted settle", + ); + assert.lengthOf( + continuationRequests, + 0, + "root-session end notice must not open a continuation", + ); + assert.isTrue( + yield* hasPendingBackgroundWork, + "hasPendingBackgroundWork must retain the unprojected end notice", + ); + + const attachNow = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: attachNow, + ordinal: 2, + messageCreatedBy: "agent", + messageCreationSource: "provider", + messageText: "Attach deferred completion.", + }), + ); + const attachProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:2", + }); + let attachTerminal: string | null = null; + while (attachTerminal === null) { + const event = yield* Queue.take(events); + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedSubagentTurnItems += 1; + } + if (event.type === "turn.terminal" && event.providerTurnId === attachProviderTurnId) { + attachTerminal = event.status; + } + } + assert.equal(completedSubagentTurnItems, 1); + assert.isFalse(yield* hasPendingBackgroundWork); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect( + "preserves wakeBuffer when a child-session completes while a continuation is pending", + () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const continuationRequests: Array = []; + const instanceId = ProviderInstanceId.make("acp-test"); + const childSessionId = "mock-child-session-pending-continuation"; + const bufferedAssistantText = "POST_SETTLE_BUFFERED_ASSISTANT_TEXT"; + let subagentPhase: "spawn" | "complete" = "spawn"; + type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; + let sessionUpdateHandler: Parameters[0] | undefined; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + enablePostSettleContinuation: true, + extractSubagentUpdate: (toolCall) => + toolCall.toolCallId !== "tool-call-generic-1" + ? undefined + : subagentPhase === "spawn" + ? { + nativeTaskId: "task-generic-1", + prompt: "background subagent", + title: "background subagent", + model: null, + status: "running", + childSessionId, + result: null, + } + : { + nativeTaskId: "task-generic-1", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId, + result: "SUB_DONE", + }, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }, + protocolEvents, + wrapRuntime: (runtime) => ({ + ...runtime, + handleSessionUpdate: (handler) => + Effect.sync(() => { + sessionUpdateHandler = handler; + }).pipe(Effect.andThen(runtime.handleSessionUpdate(handler))), + }), + }), + }, + fileSystem, + idAllocator, + serverConfig, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + }); + const threadId = ThreadId.make("thread-acp-carryover-child-pending-continuation"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-carryover-child-pending-continuation", + ), + modelSelection, + runtimePolicy, + }); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "ACP runtime must expose hasPendingBackgroundWork when post-settle continuation is enabled.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes('"stopReason"'), + ), + Stream.runHead, + ); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const firstProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + const interruptFiber = yield* runtime + .interruptTurn({ providerThread, providerTurnId: firstProviderTurnId }) + .pipe(Effect.forkScoped); + yield* TestClock.adjust("10 seconds"); + yield* Fiber.join(interruptFiber); + + let firstTerminalStatus: string | null = null; + while (firstTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + firstTerminalStatus = event.status; + } + } + assert.equal(firstTerminalStatus, "interrupted"); + assert.isTrue( + yield* hasPendingBackgroundWork, + "carryover live subagent must pin hasPendingBackgroundWork after root settle", + ); + assert.isDefined(sessionUpdateHandler, "session update handler must be wired"); + + // Root-session terminal + distinctive assistant text: both enter wakeBuffer + // and the terminal offers a continuation that will drain them. + subagentPhase = "complete"; + yield* sessionUpdateHandler!({ + sessionId: "mock-session-1", + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-1", + title: "background subagent", + kind: "other", + status: "completed", + rawOutput: { content: "SUB_DONE" }, + }, + }); + yield* sessionUpdateHandler!({ + sessionId: "mock-session-1", + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: bufferedAssistantText }, + }, + }); + assert.lengthOf( + continuationRequests, + 1, + "root-session terminal must offer a continuation before child completion", + ); + assert.isTrue( + yield* hasPendingBackgroundWork, + "pending continuation must keep hasPendingBackgroundWork pinned", + ); + + // Child path must not wipe wakeBuffer or clear the sticky continuation pin. + yield* sessionUpdateHandler!({ + sessionId: childSessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-1", + title: "background subagent", + kind: "other", + status: "completed", + rawOutput: { content: "SUB_DONE" }, + }, + }); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + let completedBeforeDrain = 0; + let polled = yield* Queue.poll(events); + while (Option.isSome(polled)) { + const event = polled.value; + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedBeforeDrain += 1; + } + polled = yield* Queue.poll(events); + } + assert.equal( + completedBeforeDrain, + 0, + "interrupted child-session terminal must wait for the continuation attach", + ); + assert.isTrue( + yield* hasPendingBackgroundWork, + "child-session must not clear the pin while wakeBuffer still has delivery", + ); + + // Continuation attach drains the preserved buffer and projects it. + const continuationNow = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: continuationNow, + ordinal: 2, + messageCreatedBy: "agent", + messageCreationSource: "provider", + messageText: "Background task completed.", + }), + ); + yield* TestClock.adjust("3 seconds"); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const continuationProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:2", + }); + let continuationTerminal: string | null = null; + let completedSubagentTurnItems = 0; + let bufferedTextSeen = false; + while (continuationTerminal === null) { + const event = yield* Queue.take(events); + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedSubagentTurnItems += 1; + } + if ( + event.type === "message.updated" && + event.message.role === "assistant" && + event.message.text.includes(bufferedAssistantText) + ) { + bufferedTextSeen = true; + } + if ( + event.type === "turn.terminal" && + event.providerTurnId === continuationProviderTurnId + ) { + continuationTerminal = event.status; + } + } + assert.equal(continuationTerminal, "completed"); + assert.isTrue( + bufferedTextSeen, + "continuation drain must project buffered post-settle assistant text (buffer not wiped)", + ); + assert.equal( + completedSubagentTurnItems, + 1, + "continuation drain must project the terminal subagent turn item once", + ); + assert.isFalse( + yield* hasPendingBackgroundWork, + "pin must clear after the continuation drains, not when the child session completed", + ); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect( + "projects once when root-session then child-session complete the same carryover subagent", + () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const continuationRequests: Array = []; + const instanceId = ProviderInstanceId.make("acp-test"); + const childSessionId = "mock-child-session-root-then-child"; + let subagentPhase: "spawn" | "complete" = "spawn"; + type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; + let sessionUpdateHandler: Parameters[0] | undefined; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + enablePostSettleContinuation: true, + extractSubagentUpdate: (toolCall) => + toolCall.toolCallId !== "tool-call-generic-1" + ? undefined + : subagentPhase === "spawn" + ? { + nativeTaskId: "task-generic-1", + prompt: "background subagent", + title: "background subagent", + model: null, + status: "running", + childSessionId, + result: null, + } + : { + nativeTaskId: "task-generic-1", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId, + result: "SUB_DONE", + }, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }, + protocolEvents, + wrapRuntime: (runtime) => ({ + ...runtime, + handleSessionUpdate: (handler) => + Effect.sync(() => { + sessionUpdateHandler = handler; + }).pipe(Effect.andThen(runtime.handleSessionUpdate(handler))), + }), + }), + }, + fileSystem, + idAllocator, + serverConfig, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + }); + const threadId = ThreadId.make("thread-acp-carryover-root-then-child"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-carryover-root-then-child", + ), + modelSelection, + runtimePolicy, + }); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "ACP runtime must expose hasPendingBackgroundWork when post-settle continuation is enabled.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes('"stopReason"'), + ), + Stream.runHead, + ); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const firstProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + const interruptFiber = yield* runtime + .interruptTurn({ providerThread, providerTurnId: firstProviderTurnId }) + .pipe(Effect.forkScoped); + yield* TestClock.adjust("10 seconds"); + yield* Fiber.join(interruptFiber); + + let firstTerminalStatus: string | null = null; + while (firstTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + firstTerminalStatus = event.status; + } + } + assert.equal(firstTerminalStatus, "interrupted"); + assert.isTrue( + yield* hasPendingBackgroundWork, + "carryover live subagent must pin hasPendingBackgroundWork after root settle", + ); + assert.isDefined(sessionUpdateHandler, "session update handler must be wired"); + + // The interrupted root-session completion advances in-memory carryover + // without projecting and offers a continuation. Child-session replay + // must not project a second copy before that observable attach. + subagentPhase = "complete"; + yield* sessionUpdateHandler!({ + sessionId: "mock-session-1", + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-1", + title: "background subagent", + kind: "other", + status: "completed", + rawOutput: { content: "SUB_DONE" }, + }, + }); + assert.lengthOf( + continuationRequests, + 1, + "root-session terminal must still offer a continuation", + ); + + yield* sessionUpdateHandler!({ + sessionId: childSessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-1", + title: "background subagent", + kind: "other", + status: "completed", + rawOutput: { content: "SUB_DONE" }, + }, + }); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + let completedBeforeDrain = 0; + let polled = yield* Queue.poll(events); + while (Option.isSome(polled)) { + const event = polled.value; + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedBeforeDrain += 1; + } + polled = yield* Queue.poll(events); + } + assert.equal( + completedBeforeDrain, + 0, + "interrupted root-then-child completion must defer to the continuation attach", + ); + + const continuationNow = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: continuationNow, + ordinal: 2, + messageCreatedBy: "agent", + messageCreationSource: "provider", + messageText: "Background task completed.", + }), + ); + yield* TestClock.adjust("3 seconds"); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const continuationProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:2", + }); + let continuationTerminal: string | null = null; + let completedSubagentTurnItems = 0; + while (continuationTerminal === null) { + const event = yield* Queue.take(events); + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedSubagentTurnItems += 1; + } + if ( + event.type === "turn.terminal" && + event.providerTurnId === continuationProviderTurnId + ) { + continuationTerminal = event.status; + } + } + assert.equal(continuationTerminal, "completed"); + assert.equal( + completedSubagentTurnItems, + 1, + "root-then-child completion must project the terminal subagent turn item exactly once", + ); + assert.isFalse( + yield* hasPendingBackgroundWork, + "hasPendingBackgroundWork must end false after the continuation drains", + ); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.effect( + "preserveRuntimeOnSettledInterrupt keeps the process alive and carries subagents through a settled steering interrupt", + () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const instanceId = ProviderInstanceId.make("acp-test"); + let subagentPhase: "spawn" | "complete" = "spawn"; + let cancelCalled = false; + let runtimeOrdinalSeen = 0; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + // Hard interrupt flags (stricter than production Grok, which no + // longer sets restartRuntimeOnEveryInterrupt): every interrupt + // would hard-kill the process group without the settled-soft gate + // under test. + restartRuntimeAfterInterrupt: true, + restartRuntimeOnEveryInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + preserveRuntimeOnSettledInterrupt: true, + extractSubagentUpdate: (toolCall) => + toolCall.toolCallId !== "tool-call-generic-1" + ? undefined + : subagentPhase === "spawn" + ? { + nativeTaskId: "task-generic-1", + prompt: "background subagent", + title: "background subagent", + model: null, + status: "running", + childSessionId: null, + result: null, + } + : { + nativeTaskId: "task-generic-1", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId: null, + result: "SUB_DONE", + }, + // No ownDetachedProcessGroup: if the interrupt wrongly takes the + // hard path, terminateProcessGroup is missing and the interrupt + // fails loudly with a poisoned session. + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: (runtimeOrdinal) => { + runtimeOrdinalSeen = Math.max(runtimeOrdinalSeen, runtimeOrdinal); + return { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }; + }, + protocolEvents, + wrapCancel: (cancel) => + Effect.sync(() => { + cancelCalled = true; + }).pipe(Effect.andThen(cancel)), + }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-settled-soft-steer"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-acp-settled-soft-steer"), + modelSelection, + runtimePolicy, + }); + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + // The still-running subagent defers finalize after session/prompt returns, + // so the interrupt below hits a settled turn held open for background work. + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes('"stopReason"'), + ), + Stream.runHead, + ); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const firstProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + const interruptFiber = yield* runtime + .interruptTurn({ providerThread, providerTurnId: firstProviderTurnId }) + .pipe(Effect.forkScoped); + yield* TestClock.adjust("10 seconds"); + yield* Fiber.join(interruptFiber); + assert.isFalse( + cancelCalled, + "settled soft steer must not send session/cancel (the real Grok CLI kills background subagents on cancel)", + ); + + let subagentTurnItemId: string | null = null; + let firstTerminalStatus: string | null = null; + while (firstTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn_item.updated" && event.turnItem.type === "subagent") { + subagentTurnItemId = event.turnItem.id; + } + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + firstTerminalStatus = event.status; + } + } + assert.equal(firstTerminalStatus, "interrupted"); + assert.notEqual(subagentTurnItemId, null); + + subagentPhase = "complete"; + const secondNow = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: secondNow, + ordinal: 2, + }), + ); + // Same runtime process (mock-session-1): a respawn would start + // mock-session-2 and drop the carryover on the session mismatch. + const secondProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:2", + }); + let carriedItemStatus: string | null = null; + let secondTerminalStatus: string | null = null; + while (secondTerminalStatus === null) { + const event = yield* Queue.take(events); + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.id === subagentTurnItemId + ) { + carriedItemStatus = event.turnItem.status; + } + if (event.type === "turn.terminal" && event.providerTurnId === secondProviderTurnId) { + secondTerminalStatus = event.status; + } + } + assert.equal(carriedItemStatus, "completed"); + assert.equal(secondTerminalStatus, "completed"); + assert.equal( + runtimeOrdinalSeen, + 1, + "settled soft steer must not respawn the ACP runtime process", + ); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live("preserveRuntimeOnSettledInterrupt does not soften a mid-prompt steering interrupt", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const instanceId = ProviderInstanceId.make("acp-test"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + restartRuntimeAfterInterrupt: true, + // Local hard-flavor gate: production Grok no longer sets + // restartRuntimeOnEveryInterrupt, but when a flavor does, the + // settled-soft gate must not leak onto an unsettled prompt. + restartRuntimeOnEveryInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + preserveRuntimeOnSettledInterrupt: true, + // No ownDetachedProcessGroup: the expected hard path fails loudly + // on the missing terminateProcessGroup, proving the settled-soft + // gate did not apply to an unsettled prompt. + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_HANG_PROMPT_FOREVER: "1" }, + protocolEvents, + }), + }, + fileSystem, + idAllocator, + serverConfig, + }); + const threadId = ThreadId.make("thread-acp-unsettled-steer-stays-hard"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-unsettled-steer-stays-hard", + ), + modelSelection, + runtimePolicy, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime + .startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 1 }), + ) + .pipe(Effect.forkScoped); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "outgoing" && rawProtocolMethod(event) === "session/prompt", + ), + Stream.runHead, + ); + const providerTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + const interruptExit = yield* runtime + .interruptTurn({ providerThread, providerTurnId }) + .pipe(Effect.exit); + if (Exit.isSuccess(interruptExit)) { + assert.fail("mid-prompt steering interrupt must still take the hard teardown path"); + } + assert.include(Cause.pretty(interruptExit.cause), "session is poisoned"); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live( + "soft mid-prompt interrupt cancels in place, reuses the runtime, and tracks cancel-backgrounded work", + () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const continuationRequests: Array = []; + const instanceId = ProviderInstanceId.make("acp-test"); + let cancelCalled = false; + let runtimeOrdinalSeen = 0; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + enablePostSettleContinuation: true, + // Production Grok interrupt flags: hard teardown only with + // requestRuntimeRestart (user Stop). Without + // restartRuntimeOnEveryInterrupt a mid-prompt steering interrupt + // stays soft: session/cancel, same process, session reuse. + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + preserveRuntimeOnSettledInterrupt: true, + registerExtensions: ({ runtime: extensionRuntime, applyBackgroundTaskMutation }) => + registerXAiBackgroundTaskTracking(extensionRuntime, applyBackgroundTaskMutation), + // No ownDetachedProcessGroup: if the interrupt wrongly takes the + // hard path, terminateProcessGroup is missing and the interrupt + // fails loudly with a poisoned session. + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: (runtimeOrdinal) => { + runtimeOrdinalSeen = Math.max(runtimeOrdinalSeen, runtimeOrdinal); + return { + T3_ACP_EMIT_RUNNING_COMMAND_THEN_HANG_FIRST_PROMPT: "1", + T3_ACP_EMIT_TASK_BACKGROUNDED_AFTER_CANCEL: "1", + }; + }, + protocolEvents, + wrapCancel: (cancel) => + Effect.sync(() => { + cancelCalled = true; + }).pipe(Effect.andThen(cancel)), }), }, fileSystem, @@ -3907,87 +6383,280 @@ describe("AcpAdapterV2", () => { subagentUpdatedResult = event.subagent.result; } } - assert.equal( - subagentStopStatus, - "interrupted", - "orphan Stop must emit interrupted terminal for turn-1 carryover subagent", - ); - assert.equal( - subagentStopResult, - streamedSubagentText, - "orphan Stop must merge streamed assistantText into the interrupted result", - ); - assert.equal( - subagentStopProviderThreadId, - providerThread.id, - "orphan Stop parent-level events must use the spawn-time parent provider thread id", - ); - assert.equal( - subagentUpdatedResult, - streamedSubagentText, - "orphan Stop subagent.updated must also carry the streamed result", - ); + assert.equal( + subagentStopStatus, + "interrupted", + "orphan Stop must emit interrupted terminal for turn-1 carryover subagent", + ); + assert.equal( + subagentStopResult, + streamedSubagentText, + "orphan Stop must merge streamed assistantText into the interrupted result", + ); + assert.equal( + subagentStopProviderThreadId, + providerThread.id, + "orphan Stop parent-level events must use the spawn-time parent provider thread id", + ); + assert.equal( + subagentUpdatedResult, + streamedSubagentText, + "orphan Stop subagent.updated must also carry the streamed result", + ); + + yield* Queue.clear(protocolEvents); + const secondNow = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: secondNow, + ordinal: 2, + }), + ); + const loadAfterRestart = yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "outgoing" && + (rawProtocolMethod(event) === "session/load" || + rawProtocolMethod(event) === "session/new"), + ), + Stream.runHead, + ); + assert.isTrue( + Option.isSome(loadAfterRestart), + "orphan containment must force a runtime respawn before the next turn", + ); + assert.equal( + runtimeOrdinalSeen, + 2, + "Stop after soft steer must replace the orphan ACP runtime process", + ); + + // nativeTurnId is `${sessionId}:turn:${ordinal}`. The mock always uses + // mock-session-1; ordinal 2 yields turn:2 on the replacement process. + // Carryover was quarantined by Stop, so turn-1's subagent must not re-attach. + subagentPhase = "complete"; + const secondProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:2", + }); + let carriedItemStatus: string | null = null; + let secondTerminalStatus: string | null = null; + while (secondTerminalStatus === null) { + const event = yield* Queue.take(events); + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.id === subagentTurnItemId + ) { + carriedItemStatus = event.turnItem.status; + } + if (event.type === "turn.terminal" && event.providerTurnId === secondProviderTurnId) { + secondTerminalStatus = event.status; + } + } + assert.isNull( + carriedItemStatus, + "Stop quarantine must drop turn-1 subagent carryover on the respawned runtime", + ); + assert.equal(secondTerminalStatus, "completed"); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + + it.live("Direct Stop projects an interrupt-deferred terminal exactly once", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const protocolEvents = yield* Queue.bounded(256); + const instanceId = ProviderInstanceId.make("acp-test"); + const childSessionId = "mock-child-session-direct-stop-deferred-terminal"; + let subagentPhase: "spawn" | "complete" = "spawn"; + type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; + let sessionUpdateHandler: Parameters[0] | undefined; + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + enablePostSettleContinuation: true, + restartRuntimeAfterInterrupt: true, + terminateRuntimeProcessGroupOnInterrupt: true, + preserveRuntimeOnSettledInterrupt: true, + extractSubagentUpdate: (toolCall) => + toolCall.toolCallId !== "tool-call-generic-1" + ? undefined + : subagentPhase === "spawn" + ? { + nativeTaskId: "task-generic-1", + prompt: "background subagent", + title: "background subagent", + model: null, + status: "running", + childSessionId, + result: null, + } + : { + nativeTaskId: "task-generic-1", + prompt: "", + title: null, + model: null, + status: "completed", + childSessionId, + result: "SUB_DONE", + }, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + environment: { T3_ACP_EMIT_GENERIC_TOOL_PLACEHOLDERS: "1" }, + ownDetachedProcessGroup: true, + protocolEvents, + wrapRuntime: (runtime) => ({ + ...runtime, + handleSessionUpdate: (handler) => + Effect.sync(() => { + sessionUpdateHandler = handler; + }).pipe(Effect.andThen(runtime.handleSessionUpdate(handler))), + }), + }), + }, + fileSystem, + idAllocator, + serverConfig, + continuationRequests: { offer: () => Effect.void }, + }); + const threadId = ThreadId.make("thread-acp-direct-stop-deferred-terminal"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-direct-stop-deferred-terminal", + ), + modelSelection, + runtimePolicy, + }); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "ACP runtime must expose hasPendingBackgroundWork when post-settle continuation is enabled.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const now = yield* DateTime.now; + yield* runtime.startTurn( + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }), + ); + yield* Stream.fromQueue(protocolEvents).pipe( + Stream.filter( + (event) => + event.direction === "incoming" && + event.stage === "raw" && + typeof event.payload === "string" && + event.payload.includes('"stopReason"'), + ), + Stream.runHead, + ); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + const providerTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + yield* runtime.interruptTurn({ providerThread, providerTurnId }); + let rootTerminal: string | null = null; + while (rootTerminal === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === providerTurnId) { + rootTerminal = event.status; + } + } + assert.equal(rootTerminal, "interrupted"); + assert.isDefined(sessionUpdateHandler, "session update handler must be wired"); - yield* Queue.clear(protocolEvents); - const secondNow = yield* DateTime.now; - yield* runtime.startTurn( - makeTurnInput({ - threadId, - providerThread, - instanceId, - runtimePolicy, - now: secondNow, - ordinal: 2, - }), - ); - const loadAfterRestart = yield* Stream.fromQueue(protocolEvents).pipe( - Stream.filter( - (event) => - event.direction === "outgoing" && - (rawProtocolMethod(event) === "session/load" || - rawProtocolMethod(event) === "session/new"), - ), - Stream.runHead, - ); - assert.isTrue( - Option.isSome(loadAfterRestart), - "orphan containment must force a runtime respawn before the next turn", - ); - assert.equal( - runtimeOrdinalSeen, - 2, - "Stop after soft steer must replace the orphan ACP runtime process", - ); + subagentPhase = "complete"; + yield* sessionUpdateHandler!({ + sessionId: childSessionId, + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-call-generic-1", + title: "background subagent", + kind: "other", + status: "completed", + rawOutput: { content: "SUB_DONE" }, + }, + }); + yield* Effect.yieldNow; + let completedBeforeStop = 0; + let polled = yield* Queue.poll(events); + while (Option.isSome(polled)) { + const event = polled.value; + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedBeforeStop += 1; + } + polled = yield* Queue.poll(events); + } + assert.equal(completedBeforeStop, 0); + assert.isTrue( + yield* hasPendingBackgroundWork, + "interrupt-deferred terminal must pin until hard-stop projection", + ); - // nativeTurnId is `${sessionId}:turn:${ordinal}`. The mock always uses - // mock-session-1; ordinal 2 yields turn:2 on the replacement process. - // Carryover was quarantined by Stop, so turn-1's subagent must not re-attach. - subagentPhase = "complete"; - const secondProviderTurnId = idAllocator.derive.providerTurn({ - driver: ACP_TEST_DRIVER, - nativeTurnId: "mock-session-1:turn:2", - }); - let carriedItemStatus: string | null = null; - let secondTerminalStatus: string | null = null; - while (secondTerminalStatus === null) { - const event = yield* Queue.take(events); - if ( - event.type === "turn_item.updated" && - event.turnItem.type === "subagent" && - event.turnItem.id === subagentTurnItemId - ) { - carriedItemStatus = event.turnItem.status; - } - if (event.type === "turn.terminal" && event.providerTurnId === secondProviderTurnId) { - secondTerminalStatus = event.status; - } + const stopExit = yield* runtime + .interruptTurn({ + providerThread, + providerTurnId, + requestRuntimeRestart: true, + }) + .pipe(Effect.exit); + if (Exit.isFailure(stopExit)) { + assert.fail(`Direct Stop must contain the orphan runtime: ${Cause.pretty(stopExit.cause)}`); + } + + let completedAfterStop = 0; + for (let attempt = 0; attempt < 64; attempt += 1) { + const maybeEvent = yield* Queue.take(events).pipe(Effect.timeoutOption("50 millis")); + if (Option.isNone(maybeEvent)) break; + const event = maybeEvent.value; + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "subagent" && + event.turnItem.status === "completed" + ) { + completedAfterStop += 1; } - assert.isNull( - carriedItemStatus, - "Stop quarantine must drop turn-1 subagent carryover on the respawned runtime", - ); - assert.equal(secondTerminalStatus, "completed"); - }).pipe(Effect.provide(testLayer), Effect.scoped), + } + assert.equal(completedAfterStop, 1); + assert.isFalse(yield* hasPendingBackgroundWork); + }).pipe(Effect.provide(testLayer), Effect.scoped), ); it.effect( @@ -4407,6 +7076,211 @@ describe("AcpAdapterV2", () => { }).pipe(Effect.provide(testLayer), Effect.scoped), ); + it.effect("keeps a buffered continuation current when a user turn starts before dispatch", () => + Effect.gen(function* () { + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const path = yield* Path.Path; + const serverConfig = yield* ServerConfig; + const mockAgentPath = yield* path.fromFileUrl( + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url), + ); + const continuationRequests: Array = []; + const userPromptStarted = yield* Deferred.make(); + const releaseUserPrompt = yield* Deferred.make(); + const bufferedAssistantText = "BUFFERED_WAKE_QUEUED_AFTER_USER"; + let promptOrdinal = 0; + type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"]; + let sessionUpdateHandler: Parameters[0] | undefined; + const instanceId = ProviderInstanceId.make("acp-test"); + const adapter = makeAcpAdapterV2({ + crypto: yield* Crypto.Crypto, + instanceId, + flavor: { + driver: ACP_TEST_DRIVER, + capabilities: AcpProviderCapabilitiesV2, + deferFinalizeForBackgroundWork: true, + enablePostSettleContinuation: true, + makeRuntime: makeMockRuntime({ + childProcessSpawner, + mockAgentPath, + wrapRuntime: (runtime) => ({ + ...runtime, + handleSessionUpdate: (handler) => + Effect.sync(() => { + sessionUpdateHandler = handler; + }).pipe(Effect.andThen(runtime.handleSessionUpdate(handler))), + prompt: (payload) => + Effect.gen(function* () { + promptOrdinal += 1; + const currentPromptOrdinal = promptOrdinal; + const result = yield* runtime.prompt(payload); + if (currentPromptOrdinal === 2) { + yield* Deferred.succeed(userPromptStarted, undefined); + yield* Deferred.await(releaseUserPrompt); + } + return result; + }), + }), + }), + }, + fileSystem, + idAllocator, + serverConfig, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + }); + const threadId = ThreadId.make("thread-acp-buffered-continuation-user-race"); + const runtimePolicy = ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + }); + const modelSelection = { instanceId, model: "default" } as const; + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-acp-buffered-continuation-user-race", + ), + modelSelection, + runtimePolicy, + }); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "ACP runtime must expose hasPendingBackgroundWork when post-settle continuation is enabled.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const events = yield* Queue.unbounded(); + yield* runtime.events.pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped, + ); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + }), + ); + const firstProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:1", + }); + let firstTerminalStatus: string | null = null; + while (firstTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === firstProviderTurnId) { + firstTerminalStatus = event.status; + } + } + assert.equal(firstTerminalStatus, "completed"); + assert.isDefined(sessionUpdateHandler, "session update handler must be wired"); + + yield* sessionUpdateHandler!({ + sessionId: "mock-session-1", + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: bufferedAssistantText }, + }, + }); + assert.lengthOf(continuationRequests, 1); + const continuationRequest = continuationRequests[0]!; + assert.isDefined(continuationRequest.dispatchIfCurrent); + + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + ordinal: 2, + messageText: "User turn won the continuation dispatch race.", + }), + ); + yield* Deferred.await(userPromptStarted); + let continuationDispatched = false; + const dispatchOutcome = yield* continuationRequest.dispatchIfCurrent!( + Effect.sync(() => { + continuationDispatched = true; + }), + ); + assert.isTrue( + Option.isSome(dispatchOutcome), + "queue_after_active dispatch must remain current while the user run is active", + ); + assert.isTrue(continuationDispatched); + yield* Deferred.succeed(releaseUserPrompt, undefined); + + const userProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:2", + }); + let userTerminalStatus: string | null = null; + while (userTerminalStatus === null) { + const event = yield* Queue.take(events); + if (event.type === "turn.terminal" && event.providerTurnId === userProviderTurnId) { + userTerminalStatus = event.status; + } + } + assert.equal(userTerminalStatus, "completed"); + assert.isTrue( + yield* hasPendingBackgroundWork, + "the queued continuation must retain ownership of its buffered wake traffic", + ); + + yield* runtime.startTurn( + makeTurnInput({ + threadId, + providerThread, + instanceId, + runtimePolicy, + now: yield* DateTime.now, + ordinal: 3, + messageCreatedBy: "agent", + messageCreationSource: "provider", + messageText: "Background task completed.", + }), + ); + yield* TestClock.adjust("3 seconds"); + const continuationProviderTurnId = idAllocator.derive.providerTurn({ + driver: ACP_TEST_DRIVER, + nativeTurnId: "mock-session-1:turn:3", + }); + let continuationTerminalStatus: string | null = null; + let bufferedTextSeen = false; + while (continuationTerminalStatus === null) { + const event = yield* Queue.take(events); + if ( + event.type === "message.updated" && + event.message.role === "assistant" && + event.message.text.includes(bufferedAssistantText) + ) { + bufferedTextSeen = true; + } + if (event.type === "turn.terminal" && event.providerTurnId === continuationProviderTurnId) { + continuationTerminalStatus = event.status; + } + } + assert.equal(continuationTerminalStatus, "completed"); + assert.isTrue(bufferedTextSeen, "continuation must drain the wake buffer after the user run"); + assert.isFalse(yield* hasPendingBackgroundWork); + }).pipe(Effect.provide(testLayer), Effect.scoped), + ); + it.effect( "holds a settled turn until the injected monitor report streams instead of finalizing into it", () => diff --git a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.ts index da059b3862f..75dfb8f7090 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.ts @@ -147,7 +147,7 @@ export interface AcpRootTurnIdleSnapshot { readonly hasRunningTool: boolean; readonly hasPendingRuntimeRequest: boolean; readonly hasToolHistory: boolean; - readonly hasRunningSubagent: boolean; + readonly hasActiveSubagent: boolean; readonly hasOutput: boolean; } @@ -172,7 +172,7 @@ export function acpRootTurnIsIdle(snapshot: AcpRootTurnIdleSnapshot): boolean { if (snapshot.finalized || snapshot.interrupted) return false; if (snapshot.assistantStreamOpen || snapshot.reasoningStreamOpen) return false; if (snapshot.hasRunningTool || snapshot.hasPendingRuntimeRequest) return false; - if (snapshot.hasRunningSubagent) return false; + if (snapshot.hasActiveSubagent) return false; if (!snapshot.hasOutput) return false; // Structural gates above stay for unit tests / future re-enable. Speculative // idle completion is intentionally disabled. @@ -323,7 +323,7 @@ export interface AcpAdapterV2SubagentUpdate { readonly prompt: string; readonly title: string | null; readonly model: string | null; - readonly status: "running" | "completed" | "failed" | "interrupted" | "cancelled"; + readonly status: "pending" | "running" | "completed" | "failed" | "interrupted" | "cancelled"; readonly childSessionId: string | null; readonly result: string | null; /** @@ -943,6 +943,7 @@ interface ActiveAcpTurn { } | null; interrupted: boolean; finalized: boolean; + finalizedStatus: "completed" | "interrupted" | "failed" | "cancelled" | null; settleScheduleGeneration: number; /** session/prompt already returned; finalize deferred for background work. */ promptSettled: boolean; @@ -1067,6 +1068,21 @@ export function acpPostSettleWakeShouldBuffer( ); } +export function acpCarryoverTerminalShouldClearContinuation(input: { + readonly continuationOffered: boolean; + readonly wakeBufferLength: number; +}): boolean { + return !input.continuationOffered && input.wakeBufferLength === 0; +} + +export function acpTurnStartShouldPreserveContinuation(input: { + readonly continuationRequested: boolean; + readonly isContinuationTurn: boolean; + readonly wakeBufferLength: number; +}): boolean { + return !input.isContinuationTurn && input.continuationRequested && input.wakeBufferLength > 0; +} + export function acpPostSettleMonitorPromptShouldSuppress( mutation: | { @@ -1100,8 +1116,44 @@ interface ActiveAcpSubagent { childSessionId: string | null; assistantText: string; nextChildOrdinal: number; + /** + * Whether a terminal carryover status has been projected to events. + * Completed roots project post-settle terminals immediately while their + * subscriber remains observable. Non-completed roots retain terminals in + * memory until the next attach. A later project:true path for the same entry + * must still project once; this flag prevents double emission and pins + * hasPendingBackgroundWork until projection lands. + */ + terminalStatusProjected: boolean; +} + +function acpSubagentStatusIsTerminal(status: OrchestrationV2Subagent["status"]): boolean { + return ( + status === "completed" || + status === "failed" || + status === "interrupted" || + status === "cancelled" + ); +} + +export function acpSubagentStatusBlocksTurnSettlement( + status: OrchestrationV2Subagent["status"], +): boolean { + return status === "running" || status === "pending"; +} + +function acpSubagentHasPendingBackgroundWork(subagent: ActiveAcpSubagent): boolean { + return ( + acpSubagentStatusBlocksTurnSettlement(subagent.task.status) || !subagent.terminalStatusProjected + ); } +type AcpCarryoverSubagents = { + readonly sessionId: string; + readonly rootTerminalStatus: "completed" | "interrupted" | "failed" | "cancelled"; + readonly subagents: ReadonlyArray; +}; + function acpTurnHasPendingRuntimeRequest( providerTurnId: OrchestrationV2ProviderTurn["id"], pending: ReadonlyMap, @@ -1449,10 +1501,7 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV // lineages into the next turn on the same session so their terminal // signals can still flip the original turn items instead of leaving // them running forever. - const carryoverSubagents = yield* Ref.make<{ - readonly sessionId: string; - readonly subagents: ReadonlyArray; - } | null>(null); + const carryoverSubagents = yield* Ref.make(null); const handledBackgroundTaskIdsInActiveTurn = yield* Ref.make>( new Set(), ); @@ -1882,6 +1931,22 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV (update.childSessionId !== null ? context.subagentsBySessionId.get(update.childSessionId) : undefined); + const updateIsTerminal = acpSubagentStatusIsTerminal(update.status); + if ( + existing !== undefined && + acpSubagentStatusIsTerminal(existing.task.status) && + !updateIsTerminal + ) { + return; + } + if ( + existing !== undefined && + existing.task.status === update.status && + updateIsTerminal && + existing.terminalStatusProjected + ) { + return; + } // get_command_or_subagent_output may target monitors/bash tasks. Only // hydrate when we already have a matching subagent lineage. Spawn ACKs // (non-empty prompt) may create a new lineage; empty-prompt hydration @@ -1947,7 +2012,7 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV }), status: taskStatus, result: existing?.assistantText || update.result, - completedAt: taskStatus === "running" ? null : now, + completedAt: acpSubagentStatusIsTerminal(taskStatus) ? now : null, updatedAt: now, }; const subagent: ActiveAcpSubagent = existing ?? { @@ -1961,6 +2026,7 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV childSessionId: null, assistantText: "", nextChildOrdinal: 101, + terminalStatusProjected: false, }; subagent.task = task; context.subagents.set(nativeTaskId, subagent); @@ -2065,7 +2131,7 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV ...subagent.task, status: taskStatus, result, - completedAt: taskStatus === "running" ? null : now, + completedAt: acpSubagentStatusIsTerminal(taskStatus) ? now : null, updatedAt: now, }; const providerThreadId = subagent.task.providerThreadId; @@ -2140,6 +2206,9 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV result, }, }); + if (acpSubagentStatusIsTerminal(taskStatus)) { + subagent.terminalStatusProjected = true; + } }); const toolOutputText = (toolCall: AcpToolCallState): string => { @@ -2188,7 +2257,7 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV if (status === "pending" || status === "running") return true; } for (const subagent of context.subagents.values()) { - if (subagent.task.status === "running" || subagent.task.status === "pending") { + if (acpSubagentStatusBlocksTurnSettlement(subagent.task.status)) { return true; } } @@ -2680,7 +2749,7 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV const offerContinuationRun = Effect.fnUntraced(function* (_sessionId: string) { if (continuationRequests === undefined) { - return; + return false; } const pending = yield* continuationPermit.withPermit( Effect.gen(function* () { @@ -2697,7 +2766,7 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV return Option.some({ route, generation }); }), ); - if (Option.isNone(pending)) return; + if (Option.isNone(pending)) return false; const { route, generation } = pending.value; yield* Effect.logInfo("orchestration-v2.acp-wake-turn-detected", { driver, @@ -2749,6 +2818,7 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV }), ), }); + return true; }); const applyLateBackgroundMutation = Effect.fnUntraced(function* ( @@ -2786,19 +2856,35 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV notification: EffectAcpSchema.SessionNotification, ) { if (!postSettleContinuationEnabled || continuationRequests === undefined) { - return false; + return { + buffered: false, + offerContinuation: false, + stopProcessing: false, + }; } // Direct Stop quarantine: drop residual wake evidence instead of // buffering it for a later continuation or follow-up run. if (yield* Ref.get(stoppedRunQuarantine)) { - return true; + return { + buffered: false, + offerContinuation: false, + stopProcessing: true, + }; } const rootSessionId = yield* Ref.get(activeSessionId); if (rootSessionId === null || notification.sessionId !== rootSessionId) { - return false; + return { + buffered: false, + offerContinuation: false, + stopProcessing: false, + }; } if (!acpPostSettleWakeEvidence(notification, flavor)) { - return false; + return { + buffered: false, + offerContinuation: false, + stopProcessing: false, + }; } const update = notification.update; let alreadyHandledToolUpdate = false; @@ -2848,15 +2934,21 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV // (the already-handled gate below intentionally skips // `offerContinuationRun` to avoid synthetic "Background task // completed." spam). + let buffered = false; if ( !alreadyHandledToolUpdate && acpPostSettleWakeShouldBuffer(notification, backgroundWorkRunning) ) { yield* Ref.update(wakeBuffer, (current) => [...current, notification]); + buffered = true; } // Buffer progress without offering; only completion-like frames open a run. if (!acpPostSettleContinuationOfferEvidence(notification, flavor)) { - return true; + return { + buffered, + offerContinuation: false, + stopProcessing: true, + }; } // While a monitor is still streaming, tool re-reports buffer without // offering and per-event agent commentary is consumed without being @@ -2868,7 +2960,11 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV // actually ends (end-notice mutation below, or the first frame after // it). if (backgroundWorkRunning) { - return true; + return { + buffered, + offerContinuation: false, + stopProcessing: true, + }; } if (alreadyHandledToolUpdate) { // Drop leftover wake noise for in-turn-handled work so idle release @@ -2878,7 +2974,11 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV if (!(yield* Ref.get(continuationRequested))) { yield* Ref.set(wakeBuffer, []); } - return true; + return { + buffered, + offerContinuation: false, + stopProcessing: true, + }; } // Residual Grok agent/thought chatter after in-turn-handled background // work must not open synthetic continuation runs. Tool-path @@ -2893,17 +2993,35 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV (update.sessionUpdate === "agent_message_chunk" || update.sessionUpdate === "agent_thought_chunk") ) { - return true; + return { + buffered, + offerContinuation: false, + stopProcessing: true, + }; } - yield* offerContinuationRun(notification.sessionId); - return true; + return { + buffered, + offerContinuation: true, + stopProcessing: true, + }; }); + let applyFinalizedActiveTurnSubagentTerminal: ( + context: ActiveAcpTurn, + notification: EffectAcpSchema.SessionNotification, + ) => Effect.Effect = () => Effect.succeed(false); + const handleSessionUpdate = Effect.fnUntraced(function* ( notification: EffectAcpSchema.SessionNotification, ) { const context = yield* Ref.get(activeTurn); const update = notification.update; + if ( + context?.finalized === true && + (yield* applyFinalizedActiveTurnSubagentTerminal(context, notification)) + ) { + return; + } // Only while a finalized turn is still the active context. When // activeTurn is null, post-settle agent frames must reach // bufferPostSettleWake so continuation can attach (context?.finalized @@ -2943,9 +3061,102 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV if (yield* Ref.get(stoppedRunQuarantine)) { return; } + const bufferOutcome = yield* bufferPostSettleWake(notification); + // Post-settle carryover sync: keep in-memory carryover accurate so + // hasPendingBackgroundWork reasons correctly after root settle. + // A completed root keeps its subscriber open while background items + // remain, so project its terminals immediately even when the wake + // frame is buffered for a continuation. Non-completed roots stay + // memory-only: durable ingest requires a subscriber that owns the + // original runId, which the interrupted path does not guarantee. + const carryover = yield* Ref.get(carryoverSubagents); + const rootTerminalCanStillProject = + carryover !== null && + carryover.sessionId === (yield* Ref.get(activeSessionId)) && + carryover.rootTerminalStatus === "completed"; + const projectCarryover = rootTerminalCanStillProject; + let carryoverTerminalized = false; + if ( + flavor.extractSubagentUpdate !== undefined && + (update.sessionUpdate === "tool_call" || update.sessionUpdate === "tool_call_update") + ) { + for (const event of parseSessionUpdateEvent(notification).events) { + if (event._tag !== "ToolCallUpdated") continue; + const toolCall = flavor.normalizeToolCall?.(event.toolCall) ?? event.toolCall; + const subagentUpdate = flavor.extractSubagentUpdate(toolCall); + if (subagentUpdate === undefined) continue; + if (!acpSubagentStatusIsTerminal(subagentUpdate.status)) { + continue; + } + if ( + yield* updateCarryoverSubagentStatus( + subagentUpdate.nativeTaskId, + subagentUpdate.status, + subagentUpdate.result, + { project: projectCarryover }, + ) + ) { + carryoverTerminalized = true; + } + if ( + subagentUpdate.childSessionId !== null && + (yield* updateCarryoverSubagentStatus( + subagentUpdate.childSessionId, + subagentUpdate.status, + subagentUpdate.result, + { project: projectCarryover }, + )) + ) { + carryoverTerminalized = true; + } + } + } + if ( + update.sessionUpdate === "user_message_chunk" && + update.content.type === "text" && + flavor.extractSubagentEndNotice !== undefined + ) { + const notice = flavor.extractSubagentEndNotice(update.content.text); + if ( + notice !== undefined && + (yield* updateCarryoverSubagentStatus( + notice.childSessionId, + notice.status, + undefined, + { + project: projectCarryover, + }, + )) + ) { + carryoverTerminalized = true; + } + } + // Synchronize carryover before an eager continuation can attach and + // drain the frame. + const continuationOffered = bufferOutcome.offerContinuation + ? yield* offerContinuationRun(notification.sessionId) + : false; + const wakeOutcome = { ...bufferOutcome, continuationOffered }; + // Frames that will not buffer never reach the continuation drain for + // this traffic. Once carryover is terminalized, skip history append / + // residual handling (and avoid double-projecting on child re-entry). + if (carryoverTerminalized && !wakeOutcome.buffered) { + // Projected above. Never clear a non-empty wakeBuffer. A sticky + // continuationRequested with an empty buffer is safe to drop so + // idle release is not wed on a pin with nothing to deliver. + if ( + acpCarryoverTerminalShouldClearContinuation({ + continuationOffered: wakeOutcome.continuationOffered, + wakeBufferLength: (yield* Ref.get(wakeBuffer)).length, + }) + ) { + yield* Ref.set(continuationRequested, false); + } + return; + } // Prefer continuation buffering over history append so the same // frames are not double-counted once a continuation run attaches. - if (yield* bufferPostSettleWake(notification)) { + if (wakeOutcome.stopProcessing) { return; } if ( @@ -2984,10 +3195,32 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV // Finalize may have completed during the activeSessionId yield. if (context.finalized) return; if (flavor.extractSubagentUpdate === undefined) return; + const subagent = context.subagentsBySessionId.get(notification.sessionId); + if ( + update.sessionUpdate === "tool_call" || + update.sessionUpdate === "tool_call_update" + ) { + if (subagent === undefined) return; + const nativeTaskId = + subagent.task.nativeTaskRef?.nativeId ?? String(subagent.task.id); + for (const event of parseSessionUpdateEvent(notification).events) { + if (event._tag !== "ToolCallUpdated") continue; + const toolCall = flavor.normalizeToolCall?.(event.toolCall) ?? event.toolCall; + const subagentUpdate = flavor.extractSubagentUpdate(toolCall); + if ( + subagentUpdate === undefined || + (subagentUpdate.nativeTaskId !== nativeTaskId && + subagentUpdate.childSessionId !== notification.sessionId) + ) { + continue; + } + yield* emitSubagent(context, subagentUpdate); + } + return; + } if (update.sessionUpdate !== "agent_message_chunk" || update.content.type !== "text") { return; } - const subagent = context.subagentsBySessionId.get(notification.sessionId); if (subagent !== undefined) { yield* projectSubagentNotification(subagent, notification); return; @@ -3471,113 +3704,305 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV ); }); + /** + * Mutate a carryover subagent's in-memory task status without projecting. + * Used after a non-completed root whose original run subscriber is no + * longer guaranteed to ingest the terminal event. + */ + const mutateCarryoverSubagentStatus = Effect.fnUntraced(function* ( + subagent: ActiveAcpSubagent, + status: OrchestrationV2Subagent["status"], + resultOverride?: string | null, + ) { + const now = yield* DateTime.now; + const result = + resultOverride !== undefined && resultOverride !== null && resultOverride.length > 0 + ? resultOverride + : subagent.assistantText || subagent.task.result; + const completedAt = acpSubagentStatusIsTerminal(status) ? now : null; + subagent.task = { + ...subagent.task, + status, + result, + completedAt, + updatedAt: now, + }; + }); + + /** + * Project a carryover subagent status change without an ActiveAcpTurn. + * Shared by completed-root post-settle completion, deferred attach, and + * Direct Stop terminalization. + */ + const projectCarryoverSubagentStatus = Effect.fnUntraced(function* ( + subagent: ActiveAcpSubagent, + status: OrchestrationV2Subagent["status"], + resultOverride?: string | null, + ) { + yield* mutateCarryoverSubagentStatus(subagent, status, resultOverride); + const now = subagent.task.updatedAt; + const nativeTaskId = subagent.task.nativeTaskRef?.nativeId ?? subagent.task.id; + const nativeItemRef = { + driver, + nativeId: nativeTaskId, + strength: "strong" as const, + }; + const parentProviderThreadId = subagent.parentProviderThreadId; + const result = subagent.task.result; + const completedAt = subagent.task.completedAt; + yield* emitProviderEvent({ + type: "node.updated", + driver, + node: { + id: subagent.task.id, + threadId: subagent.task.threadId, + runId: subagent.task.runId, + parentNodeId: subagent.task.parentNodeId, + rootNodeId: subagent.task.parentNodeId, + kind: "subagent", + status, + countsForRun: false, + providerThreadId: parentProviderThreadId, + providerTurnId: subagent.providerTurnId, + nativeItemRef, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: subagent.task.startedAt, + completedAt, + }, + }); + yield* emitProviderEvent({ + type: "node.updated", + driver, + node: { + id: subagent.childRootNodeId, + threadId: subagent.childThreadId, + runId: null, + parentNodeId: null, + rootNodeId: subagent.childRootNodeId, + kind: "root_turn", + status, + countsForRun: false, + providerThreadId: subagent.task.providerThreadId, + providerTurnId: null, + nativeItemRef, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: subagent.task.startedAt, + completedAt, + }, + }); + yield* emitProviderEvent({ + type: "subagent.updated", + driver, + subagent: subagent.task, + }); + yield* emitProviderEvent({ + type: "turn_item.updated", + driver, + turnItem: { + id: subagent.turnItemId, + threadId: subagent.task.threadId, + runId: subagent.task.runId, + nodeId: subagent.task.id, + providerThreadId: parentProviderThreadId, + providerTurnId: subagent.providerTurnId, + nativeItemRef, + parentItemId: null, + ordinal: subagent.turnItemOrdinal, + status, + title: subagent.task.title, + startedAt: subagent.task.startedAt, + completedAt, + updatedAt: now, + type: "subagent", + subagentId: subagent.task.id, + origin: "provider_native", + driver, + providerInstanceId: subagent.task.providerInstanceId, + childThreadId: subagent.childThreadId, + prompt: subagent.task.prompt, + result, + }, + }); + if (acpSubagentStatusIsTerminal(status)) { + subagent.terminalStatusProjected = true; + } + }); + + /** + * Update a carryover subagent matched by native task id or child session + * id. Post-settle completions must flip the pin in hasPendingBackgroundWork + * without waiting for a new user turn to consume carryover. + * When `project` is false, only the in-memory carryover status advances + * (the next attach projects the terminal state). + * If status already matches but projection was deferred, a later + * project:true caller still projects once via terminalStatusProjected. + */ + const updateCarryoverSubagentStatus = Effect.fnUntraced(function* ( + nativeIdOrChildSessionId: string, + status: OrchestrationV2Subagent["status"], + result?: string | null, + options?: { readonly project?: boolean }, + ) { + const carryover = yield* Ref.get(carryoverSubagents); + if (carryover === null) return false; + const match = carryover.subagents.find((subagent) => { + const nativeId = subagent.task.nativeTaskRef?.nativeId ?? String(subagent.task.id); + return ( + nativeId === nativeIdOrChildSessionId || + subagent.childSessionId === nativeIdOrChildSessionId + ); + }); + if (match === undefined) return false; + const shouldProject = options?.project !== false; + if (match.task.status === status) { + // Already at this status: only project if requested and not yet done. + if (!shouldProject || match.terminalStatusProjected) { + return true; + } + yield* projectCarryoverSubagentStatus(match, status, result); + return true; + } + // Only advance nonterminal entries; do not resurrect a terminal one. + if (match.task.status !== "running" && match.task.status !== "pending") { + return false; + } + if (!shouldProject) { + yield* mutateCarryoverSubagentStatus(match, status, result); + } else { + yield* projectCarryoverSubagentStatus(match, status, result); + } + return true; + }); + + applyFinalizedActiveTurnSubagentTerminal = Effect.fnUntraced(function* ( + context: ActiveAcpTurn, + notification: EffectAcpSchema.SessionNotification, + ) { + if (flavor.extractSubagentUpdate === undefined) return false; + + const applyTerminal = Effect.fnUntraced(function* ( + subagent: ActiveAcpSubagent, + update: AcpAdapterV2SubagentUpdate, + ) { + if (!acpSubagentStatusIsTerminal(update.status)) return false; + if (acpSubagentStatusIsTerminal(subagent.task.status)) { + if (subagent.terminalStatusProjected || context.finalizedStatus !== "completed") { + return true; + } + } + if (context.finalizedStatus === "completed") { + yield* emitSubagent(context, update); + } else { + yield* mutateCarryoverSubagentStatus(subagent, update.status, update.result); + } + return true; + }); + + const sessionUpdate = notification.update; + if ( + sessionUpdate.sessionUpdate === "tool_call" || + sessionUpdate.sessionUpdate === "tool_call_update" + ) { + for (const event of parseSessionUpdateEvent(notification).events) { + if (event._tag !== "ToolCallUpdated") continue; + const toolCall = flavor.normalizeToolCall?.(event.toolCall) ?? event.toolCall; + const subagentUpdate = flavor.extractSubagentUpdate(toolCall); + if ( + subagentUpdate === undefined || + !acpSubagentStatusIsTerminal(subagentUpdate.status) + ) { + continue; + } + const subagent = + context.subagents.get(subagentUpdate.nativeTaskId) ?? + (subagentUpdate.childSessionId === null + ? undefined + : context.subagentsBySessionId.get(subagentUpdate.childSessionId)) ?? + context.subagentsBySessionId.get(notification.sessionId); + if (subagent === undefined) continue; + const nativeTaskId = + subagent.task.nativeTaskRef?.nativeId ?? String(subagent.task.id); + if ( + subagentUpdate.nativeTaskId !== nativeTaskId && + (subagentUpdate.childSessionId === null || + (subagentUpdate.childSessionId !== subagent.childSessionId && + subagentUpdate.childSessionId !== notification.sessionId)) + ) { + continue; + } + if (yield* applyTerminal(subagent, subagentUpdate)) return true; + } + } + + if ( + sessionUpdate.sessionUpdate === "user_message_chunk" && + sessionUpdate.content.type === "text" && + flavor.extractSubagentEndNotice !== undefined + ) { + const notice = flavor.extractSubagentEndNotice(sessionUpdate.content.text); + const subagent = + notice === undefined + ? undefined + : context.subagentsBySessionId.get(notice.childSessionId); + if (notice !== undefined && subagent !== undefined) { + return yield* applyTerminal(subagent, { + nativeTaskId: subagent.task.nativeTaskRef?.nativeId ?? String(subagent.task.id), + prompt: subagent.task.prompt, + title: subagent.task.title, + model: subagent.task.model, + status: notice.status, + childSessionId: notice.childSessionId, + result: null, + suppressNormalTool: true, + }); + } + } + + return false; + }); + + const projectDeferredCarryoverTerminals = Effect.fnUntraced(function* ( + subagents: ReadonlyArray, + ) { + for (const subagent of subagents) { + if ( + subagent.task.status === "running" || + subagent.task.status === "pending" || + subagent.terminalStatusProjected + ) { + continue; + } + yield* projectCarryoverSubagentStatus( + subagent, + subagent.task.status, + subagent.task.result, + ); + } + }); + /** * Direct Stop after a soft steer clears carryover without an active turn. * Emit the same interrupted terminal events terminalizeOpenRunOwnedItems * would have, context-free (no ActiveAcpTurn). */ const terminalizeCarryoverSubagents = Effect.fnUntraced(function* ( - carryover: { - readonly sessionId: string; - readonly subagents: ReadonlyArray; - } | null, + carryover: AcpCarryoverSubagents | null, ) { if (carryover === null) return; - const now = yield* DateTime.now; for (const subagent of carryover.subagents) { - if (subagent.task.status !== "running" && subagent.task.status !== "pending") { + if (acpSubagentStatusIsTerminal(subagent.task.status)) { + if (!subagent.terminalStatusProjected) { + yield* projectCarryoverSubagentStatus( + subagent, + subagent.task.status, + subagent.task.result, + ); + } continue; } - const nativeTaskId = subagent.task.nativeTaskRef?.nativeId ?? subagent.task.id; - const nativeItemRef = { - driver, - nativeId: nativeTaskId, - strength: "strong" as const, - }; - const parentProviderThreadId = subagent.parentProviderThreadId; - const result = subagent.assistantText || subagent.task.result; - subagent.task = { - ...subagent.task, - status: "interrupted", - result, - completedAt: now, - updatedAt: now, - }; - yield* emitProviderEvent({ - type: "node.updated", - driver, - node: { - id: subagent.task.id, - threadId: subagent.task.threadId, - runId: subagent.task.runId, - parentNodeId: subagent.task.parentNodeId, - rootNodeId: subagent.task.parentNodeId, - kind: "subagent", - status: "interrupted", - countsForRun: false, - providerThreadId: parentProviderThreadId, - providerTurnId: subagent.providerTurnId, - nativeItemRef, - runtimeRequestId: null, - checkpointScopeId: null, - startedAt: subagent.task.startedAt, - completedAt: now, - }, - }); - yield* emitProviderEvent({ - type: "node.updated", - driver, - node: { - id: subagent.childRootNodeId, - threadId: subagent.childThreadId, - runId: null, - parentNodeId: null, - rootNodeId: subagent.childRootNodeId, - kind: "root_turn", - status: "interrupted", - countsForRun: false, - providerThreadId: subagent.task.providerThreadId, - providerTurnId: null, - nativeItemRef, - runtimeRequestId: null, - checkpointScopeId: null, - startedAt: subagent.task.startedAt, - completedAt: now, - }, - }); - yield* emitProviderEvent({ - type: "subagent.updated", - driver, - subagent: subagent.task, - }); - yield* emitProviderEvent({ - type: "turn_item.updated", - driver, - turnItem: { - id: subagent.turnItemId, - threadId: subagent.task.threadId, - runId: subagent.task.runId, - nodeId: subagent.task.id, - providerThreadId: parentProviderThreadId, - providerTurnId: subagent.providerTurnId, - nativeItemRef, - parentItemId: null, - ordinal: subagent.turnItemOrdinal, - status: "interrupted", - title: subagent.task.title, - startedAt: subagent.task.startedAt, - completedAt: now, - updatedAt: now, - type: "subagent", - subagentId: subagent.task.id, - origin: "provider_native", - driver, - providerInstanceId: subagent.task.providerInstanceId, - childThreadId: subagent.childThreadId, - prompt: subagent.task.prompt, - result, - }, - }); + yield* projectCarryoverSubagentStatus(subagent, "interrupted"); } }); @@ -4072,6 +4497,7 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV ) { if (context.finalized) return; const settledStatus = context.interrupted ? "interrupted" : status; + context.finalizedStatus = settledStatus; context.finalized = true; if (options?.drainTrailingChunks === true) { yield* drainTrailingRootTurnChunks(); @@ -4140,14 +4566,18 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV threadDisposition: "reusable", }, ); - const liveSubagents = [...context.subagents.values()].filter( - (subagent) => subagent.task.status === "running" || subagent.task.status === "pending", + const subagentsRequiringCarryover = [...context.subagents.values()].filter( + acpSubagentHasPendingBackgroundWork, ); // Direct Stop must not carry residual subagents into a later run. - if (liveSubagents.length > 0 && !directStopQuarantine) { + if (subagentsRequiringCarryover.length > 0 && !directStopQuarantine) { const sessionId = yield* Ref.get(activeSessionId); if (sessionId !== null) { - yield* Ref.set(carryoverSubagents, { sessionId, subagents: liveSubagents }); + yield* Ref.set(carryoverSubagents, { + sessionId, + rootTerminalStatus: settledStatus, + subagents: subagentsRequiringCarryover, + }); } } yield* Ref.set(activeTurn, null); @@ -4166,8 +4596,8 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV }); // Debounce already proved root-session quiescence; open segment handles // without an explicit close should not block settlement. - const hasRunningSubagent = [...context.subagents.values()].some( - (subagent) => subagent.task.status === "running", + const hasActiveSubagent = [...context.subagents.values()].some((subagent) => + acpSubagentStatusBlocksTurnSettlement(subagent.task.status), ); if ( !acpRootTurnIsIdle({ @@ -4178,7 +4608,7 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV hasRunningTool, hasPendingRuntimeRequest, hasToolHistory: context.tools.size > 0, - hasRunningSubagent, + hasActiveSubagent, hasOutput: context.assistant.nextSegment > 0, }) ) { @@ -4363,10 +4793,17 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV postSettleContinuationEnabled && turnInput.message.createdBy === "agent" && turnInput.message.creationSource === "provider"; - // Drop a sticky continuation offer when any new turn starts so idle - // pin and further offers cannot wed on a completed or failed dispatch. + // A user turn supersedes an empty offer, but not an offer that owns + // buffered wake traffic. ProviderContinuationService queues that + // continuation behind the user run so it can drain afterwards. yield* continuationPermit.withPermit( Effect.gen(function* () { + const preserveBufferedContinuation = acpTurnStartShouldPreserveContinuation({ + continuationRequested: yield* Ref.get(continuationRequested), + isContinuationTurn, + wakeBufferLength: (yield* Ref.get(wakeBuffer)).length, + }); + if (preserveBufferedContinuation) return; yield* Ref.update(continuationGeneration, (value) => value + 1); yield* Ref.set(continuationRequested, false); }), @@ -4397,6 +4834,7 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV plan: null, interrupted: false, finalized: false, + finalizedStatus: null, settleScheduleGeneration: 0, promptSettled: false, promptSettledStatus: null, @@ -4404,7 +4842,9 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV backgroundFinalizeGeneration: 0, }; const carryover = yield* Ref.getAndSet(carryoverSubagents, null); + let rehydratedCarryoverSubagents: ReadonlyArray = []; if (carryover !== null && carryover.sessionId === requestedSessionId) { + rehydratedCarryoverSubagents = carryover.subagents; for (const subagent of carryover.subagents) { const nativeId = subagent.task.nativeTaskRef?.nativeId ?? null; if (nativeId !== null) { @@ -4455,25 +4895,41 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV createdAt: startedAt, updatedAt: startedAt, }); + // Every attach ends the deferred-terminal contract, but only the + // queued provider continuation owns wake traffic. A user turn may + // start before that continuation and must leave its buffer intact. + yield* projectDeferredCarryoverTerminals(rehydratedCarryoverSubagents); + // Keep projected terminals addressable through the wake drain so + // emitSubagent's monotonic guard can reject replayed spawn frames. + // Finalize carries only entries with pending background work, so a + // terminal-and-projected lineage still expires with this turn. if (isContinuationTurn) { - const drained = yield* Ref.modify(wakeBuffer, (current) => { - const next: Array = []; - return [current.slice(), next] as const; - }); yield* Ref.set(continuationRequested, false); + const drainedWakeCount = yield* Ref.modify(wakeBuffer, (current) => { + const next: Array = []; + return [ + current.filter((notification) => notification.sessionId === requestedSessionId), + next, + ] as const; + }).pipe( + Effect.tap((drained) => + Effect.forEach(drained, handleSessionUpdate, { + concurrency: 1, + discard: true, + }), + ), + Effect.map((drained) => drained.length), + ); // Treat attach mode as prompt-settled so deferred finalize / quiet // windows can complete the continuation after wake traffic drains. context.promptSettled = true; context.promptSettledStatus = "completed"; - if (drained.length === 0) { + if (drainedWakeCount === 0) { yield* finalizeTurn(context, "completed", undefined, { drainTrailingChunks: true, }); return; } - for (const notification of drained) { - yield* handleSessionUpdate(notification); - } if (!context.finalized) { if (hasDeferredBackgroundWork(context)) { yield* rearmDeferredFinalize(context); @@ -4640,6 +5096,27 @@ export function makeAcpAdapterV2(options: AcpAdapterV2Options): ProviderAdapterV if ((yield* Ref.get(wakeBuffer)).length > 0) return true; if (yield* Ref.get(continuationRequested)) return true; if ((yield* Ref.get(runningBackgroundTaskIds)).size > 0) return true; + // Projected post-settle Grok subagents can outlive the root + // turn via carryover; keep the ACP process pinned until they + // terminalize or teardown clears the carryover. + // Also pin while a terminal status is held only in memory + // (project:false) so idle release cannot drop the session + // before the continuation drain (or a later project:true path) + // delivers the turn_item terminal. + const active = yield* Ref.get(activeTurn); + if ( + active !== null && + [...active.subagents.values()].some(acpSubagentHasPendingBackgroundWork) + ) { + return true; + } + const carryover = yield* Ref.get(carryoverSubagents); + if ( + carryover !== null && + carryover.subagents.some(acpSubagentHasPendingBackgroundWork) + ) { + return true; + } return false; }), } diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 8e22379ae8d..58e84b251c3 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -54,6 +54,7 @@ import { CLAUDE_READ_ONLY_T3_MCP_ALLOWED_TOOLS, CLAUDE_T3_MCP_TOOL_WILDCARD, ClaudeProviderCapabilitiesV2, + ClaudeAgentSdkQueryRunnerError, claudeEffectiveQueryPolicyKey, claudeMcpQueryOverrides, claudeQueryMessages, @@ -122,6 +123,8 @@ function makeClaudeTestTurnInput(input: { readonly providerTurnOrdinal?: number; readonly messageCreatedBy?: ProviderAdapterV2TurnInput["message"]["createdBy"]; readonly messageCreationSource?: ProviderAdapterV2TurnInput["message"]["creationSource"]; + readonly modelSelection?: ModelSelection; + readonly runtimePolicy?: ProviderAdapterV2RuntimePolicy; }): ProviderAdapterV2TurnInput { return { appThread: makeClaudeTestAppThread(input), @@ -139,8 +142,8 @@ function makeClaudeTestTurnInput(input: { text: input.text, attachments: input.attachments, }, - modelSelection: CLAUDE_TEST_MODEL_SELECTION, - runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + modelSelection: input.modelSelection ?? CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: input.runtimePolicy ?? CLAUDE_TEST_RUNTIME_POLICY, }; } @@ -1119,6 +1122,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { const WAKE_NATIVE_SESSION = "native-thread-claude-wake"; const WAKE_TASK_ID = "task-wake-build"; const WAKE_SUMMARY = "Background build completed successfully"; + const WAKE_ASSISTANT_TEXT = "The background build has finished."; const WAKE_RESULT_TEXT = "The background build finished; everything passed."; function claudeSdkFrame(frame: unknown): SDKMessage { @@ -1183,6 +1187,16 @@ describe("ClaudeAdapterV2 background wake turns", () => { uuid: "00000000-0000-4000-8000-000000000103", session_id: WAKE_NATIVE_SESSION, }); + const wakeAssistant = claudeSdkFrame({ + type: "assistant", + message: { + role: "assistant", + content: [{ type: "text", text: WAKE_ASSISTANT_TEXT }], + }, + parent_tool_use_id: null, + uuid: "00000000-0000-4000-8000-000000000107", + session_id: WAKE_NATIVE_SESSION, + }); const wakeResult = makeResultFrame({ uuid: "00000000-0000-4000-8000-000000000104", result: WAKE_RESULT_TEXT, @@ -1290,6 +1304,528 @@ describe("ClaudeAdapterV2 background wake turns", () => { }; }); + const providerThreadRosterEvents = (events: ReadonlyArray) => + events.filter( + (event): event is Extract => + event.type === "provider_thread.updated", + ); + + it.effect( + "projects an authoritative background_tasks_changed roster on the provider thread", + () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + const rosterSnapshot = claudeSdkFrame({ + type: "system", + subtype: "background_tasks_changed", + tasks: [ + { + task_id: WAKE_TASK_ID, + description: "npm run build", + task_type: "local_bash", + }, + ], + uuid: "00000000-0000-4000-8000-000000000201", + session_id: WAKE_NATIVE_SESSION, + }); + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-roster-snapshot"), + text: "Run the build in the background.", + attachments: [], + }), + ); + yield* Queue.offer(harness.sdkMessages, rosterSnapshot); + yield* Queue.offer(harness.sdkMessages, turnOneResult); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "first turn terminal"); + + const rosterEvents = providerThreadRosterEvents(harness.events).filter( + (event) => (event.providerThread.pendingBackgroundTasks?.length ?? 0) > 0, + ); + assert.isAtLeast(rosterEvents.length, 1); + assert.deepEqual(rosterEvents.at(-1)?.providerThread.pendingBackgroundTasks ?? [], [ + { + taskId: WAKE_TASK_ID, + description: "npm run build", + taskType: "local_bash", + }, + ]); + assert.isTrue(yield* harness.hasPendingBackgroundWork); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect( + "uses task_started as an incremental roster fallback and clears on empty snapshot", + () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + const emptyRoster = claudeSdkFrame({ + type: "system", + subtype: "background_tasks_changed", + tasks: [], + uuid: "00000000-0000-4000-8000-000000000202", + session_id: WAKE_NATIVE_SESSION, + }); + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-roster-fallback"), + text: "Run the build in the background.", + attachments: [], + }), + ); + yield* Queue.offer(harness.sdkMessages, wakeTaskStarted); + yield* Queue.offer(harness.sdkMessages, turnOneResult); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "first turn terminal"); + + const afterStart = providerThreadRosterEvents(harness.events).filter( + (event) => (event.providerThread.pendingBackgroundTasks?.length ?? 0) > 0, + ); + assert.isAtLeast(afterStart.length, 1); + assert.equal( + (afterStart.at(-1)?.providerThread.pendingBackgroundTasks ?? [])[0]?.taskId, + WAKE_TASK_ID, + ); + + yield* Queue.offer(harness.sdkMessages, emptyRoster); + yield* awaitUntil( + () => + providerThreadRosterEvents(harness.events).some( + (event) => + event.providerThread.status === "idle" && + (event.providerThread.pendingBackgroundTasks?.length ?? 0) === 0, + ), + "empty roster clear", + ); + assert.isFalse(yield* harness.hasPendingBackgroundWork); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("clears the roster when a turn fails", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + const failedResult = claudeSdkFrame({ + type: "result", + subtype: "error_during_execution", + duration_ms: 10, + duration_api_ms: 10, + is_error: true, + num_turns: 1, + result: "boom", + stop_reason: "end_turn", + total_cost_usd: 0, + usage: { + input_tokens: 1, + output_tokens: 1, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }, + modelUsage: {}, + permission_denials: [], + errors: ["boom"], + uuid: "00000000-0000-4000-8000-000000000203", + session_id: WAKE_NATIVE_SESSION, + }); + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-roster-fail"), + text: "Run the build in the background.", + attachments: [], + }), + ); + yield* Queue.offer(harness.sdkMessages, wakeTaskStarted); + yield* awaitUntil( + () => + providerThreadRosterEvents(harness.events).some( + (event) => (event.providerThread.pendingBackgroundTasks?.length ?? 0) > 0, + ), + "roster after task_started", + ); + yield* Queue.offer(harness.sdkMessages, failedResult); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "failed terminal"); + assert.equal(harness.terminalEvents()[0]?.status, "failed"); + + const afterFailure = providerThreadRosterEvents(harness.events).at(-1); + assert.deepEqual(afterFailure?.providerThread.pendingBackgroundTasks ?? [], []); + assert.isFalse(yield* harness.hasPendingBackgroundWork); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("clears the native-thread roster when a turn is interrupted", () => + Effect.scoped( + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const attachmentsDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-claude-v2-roster-interrupt-", + }); + const sdkMessages = yield* Queue.unbounded(); + const events: Array = []; + const adapter = makeClaudeAdapterV2({ + instanceId: CLAUDE_DEFAULT_INSTANCE_ID, + settings: DEFAULT_CLAUDE_SETTINGS, + environment: {}, + attachmentsDir, + fileSystem, + idAllocator, + continuationRequests: { offer: () => Effect.void }, + queryRunner: { + allocateSessionId: Effect.succeed(WAKE_NATIVE_SESSION), + open: () => + Effect.succeed({ + messages: Stream.fromQueue(sdkMessages), + offer: () => Effect.void, + setModel: () => Effect.void, + interrupt: Effect.void, + // End the message stream so interruptTurn's closed wait resolves + // via stream exit finalize (interrupted status clears roster). + close: Queue.shutdown(sdkMessages), + }), + forkSession: () => Effect.die("unused forkSession"), + assertComplete: Effect.void, + }, + }); + const threadId = ThreadId.make("thread-claude-roster-interrupt"); + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-claude-roster-interrupt"), + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + yield* runtime.events.pipe( + Stream.runForEach((event) => + Effect.sync(() => { + events.push(event); + }), + ), + Effect.forkScoped, + ); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die("Claude adapter runtime must expose hasPendingBackgroundWork."); + } + const now = yield* DateTime.now; + + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-roster-interrupt"), + text: "Run the build in the background.", + attachments: [], + }), + ); + yield* Queue.offer(sdkMessages, wakeTaskStarted); + yield* awaitUntil( + () => + providerThreadRosterEvents(events).some( + (event) => (event.providerThread.pendingBackgroundTasks?.length ?? 0) > 0, + ), + "roster after task_started", + ); + + const providerTurnId = events.find( + (event): event is Extract => + event.type === "provider_turn.updated", + )?.providerTurn.id; + assert.isDefined(providerTurnId); + yield* runtime.interruptTurn({ + providerThread, + providerTurnId: providerTurnId!, + }); + yield* awaitUntil( + () => + events.some( + (event) => event.type === "turn.terminal" && event.status === "interrupted", + ), + "interrupted terminal", + ); + + const afterInterrupt = providerThreadRosterEvents(events).at(-1); + assert.deepEqual(afterInterrupt?.providerThread.pendingBackgroundTasks ?? [], []); + assert.isFalse(yield* runtime.hasPendingBackgroundWork); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect( + "clears the replaced sibling native thread roster when openQuery switches processes", + () => + Effect.scoped( + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const attachmentsDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-claude-v2-sibling-replace-", + }); + const nativeIds = ["native-thread-roster-a", "native-thread-roster-b"] as const; + let allocateIndex = 0; + // Real two-process model: each openQuery owns its own message queue. + // A shared queue would mask sibling process death on replacement. + const processQueues: Array<{ + readonly nativeThreadId: string; + readonly queue: Queue.Queue; + }> = []; + const events: Array = []; + const adapter = makeClaudeAdapterV2({ + instanceId: CLAUDE_DEFAULT_INSTANCE_ID, + settings: DEFAULT_CLAUDE_SETTINGS, + environment: {}, + attachmentsDir, + fileSystem, + idAllocator, + continuationRequests: { + offer: () => Effect.void, + }, + queryRunner: { + allocateSessionId: Effect.sync(() => { + const next = + nativeIds[allocateIndex] ?? `native-thread-roster-extra-${allocateIndex}`; + allocateIndex += 1; + return next; + }), + open: (openInput) => + Effect.gen(function* () { + const nativeThreadId = openInput.options.sessionId ?? openInput.options.resume; + if (typeof nativeThreadId !== "string" || nativeThreadId.length === 0) { + return yield* Effect.die("openQuery must supply a native session id"); + } + const queue = yield* Queue.unbounded(); + processQueues.push({ nativeThreadId, queue }); + return { + messages: Stream.fromQueue(queue), + offer: () => Effect.void, + setModel: () => Effect.void, + interrupt: Effect.void, + close: Queue.shutdown(queue), + }; + }), + forkSession: () => Effect.die("unused forkSession"), + assertComplete: Effect.void, + }, + }); + const appThreadA = ThreadId.make("thread-claude-roster-a"); + const appThreadB = ThreadId.make("thread-claude-roster-b"); + const runtime = yield* adapter.openSession({ + threadId: appThreadA, + providerSessionId: ProviderSessionId.make("provider-session-claude-sibling-replace"), + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + const providerThreadA = yield* runtime.ensureThread({ + threadId: appThreadA, + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + const providerThreadB = yield* runtime.ensureThread({ + threadId: appThreadB, + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + assert.notEqual( + providerThreadA.nativeThreadRef?.nativeId, + providerThreadB.nativeThreadRef?.nativeId, + ); + yield* runtime.events.pipe( + Stream.runForEach((event) => + Effect.sync(() => { + events.push(event); + }), + ), + Effect.forkScoped, + ); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "Claude adapter runtime must expose hasPendingBackgroundWork.", + ); + } + if (runtime.hasPendingBackgroundWorkForThread === undefined) { + return yield* Effect.die( + "Claude adapter runtime must expose hasPendingBackgroundWorkForThread.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const hasPendingBackgroundWorkForThread = runtime.hasPendingBackgroundWorkForThread; + const now = yield* DateTime.now; + const taskA = "task-roster-a"; + const taskB = "task-roster-b"; + + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: appThreadA, + providerThread: providerThreadA, + now, + attemptId: RunAttemptId.make("attempt-roster-iso-a"), + text: "Background work on A.", + attachments: [], + }), + ); + assert.equal(processQueues.length, 1); + const processA = processQueues[0]!; + yield* Queue.offer( + processA.queue, + claudeSdkFrame({ + type: "system", + subtype: "task_started", + task_id: taskA, + description: "work on A", + task_type: "local_bash", + uuid: "00000000-0000-4000-8000-000000000301", + session_id: nativeIds[0], + }), + ); + yield* Queue.offer( + processA.queue, + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000302", + result: "A settled with background work.", + }), + ); + yield* awaitUntil( + () => + events.some( + (event) => + event.type === "turn.terminal" && + event.providerThreadId === providerThreadA.id && + event.status === "completed", + ), + "thread A terminal", + ); + const rosterAAfterSettle = providerThreadRosterEvents(events).findLast( + (event) => event.providerThread.id === providerThreadA.id, + )?.providerThread.pendingBackgroundTasks; + assert.deepEqual(rosterAAfterSettle ?? [], [ + { taskId: taskA, description: "work on A", taskType: "local_bash" }, + ]); + assert.isTrue(yield* hasPendingBackgroundWork); + assert.isTrue(yield* hasPendingBackgroundWorkForThread(providerThreadA)); + assert.isFalse(yield* hasPendingBackgroundWorkForThread(providerThreadB)); + + // Starting B closes A's only live query. A can never emit a roster + // clear from a dead process, so openQuery must idle-clear A. + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: appThreadB, + providerThread: { ...providerThreadB, status: "active" }, + now, + attemptId: RunAttemptId.make("attempt-roster-iso-b"), + text: "Background work on B.", + attachments: [], + }), + ); + assert.equal(processQueues.length, 2); + yield* awaitUntil( + () => + providerThreadRosterEvents(events).some( + (event) => + event.providerThread.id === providerThreadA.id && + event.providerThread.status === "idle" && + (event.providerThread.pendingBackgroundTasks?.length ?? 0) === 0, + ), + "sibling A roster cleared idle on process replacement", + ); + assert.isFalse(yield* hasPendingBackgroundWorkForThread(providerThreadA)); + + const processB = processQueues[1]!; + yield* Queue.offer( + processB.queue, + claudeSdkFrame({ + type: "system", + subtype: "task_started", + task_id: taskB, + description: "work on B", + task_type: "local_bash", + uuid: "00000000-0000-4000-8000-000000000303", + session_id: nativeIds[1], + }), + ); + yield* awaitUntil( + () => + providerThreadRosterEvents(events).some( + (event) => + event.providerThread.id === providerThreadB.id && + (event.providerThread.pendingBackgroundTasks?.length ?? 0) > 0, + ), + "thread B roster populated", + ); + assert.isTrue(yield* hasPendingBackgroundWorkForThread(providerThreadB)); + assert.isTrue(yield* hasPendingBackgroundWork); + // Starting B's process clears only B's process-scoped level; A stays + // empty from the sibling replacement clear above. + assert.isFalse(yield* hasPendingBackgroundWorkForThread(providerThreadA)); + + yield* Queue.offer( + processB.queue, + claudeSdkFrame({ + type: "result", + subtype: "error_during_execution", + duration_ms: 10, + duration_api_ms: 10, + is_error: true, + num_turns: 1, + result: "B failed", + stop_reason: "end_turn", + total_cost_usd: 0, + usage: { + input_tokens: 1, + output_tokens: 1, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }, + modelUsage: {}, + permission_denials: [], + errors: ["B failed"], + uuid: "00000000-0000-4000-8000-000000000304", + session_id: nativeIds[1], + }), + ); + yield* awaitUntil( + () => + events.some( + (event) => + event.type === "turn.terminal" && + event.providerThreadId === providerThreadB.id && + event.status === "failed", + ), + "thread B failed terminal", + ); + + const rosterBAfterFail = providerThreadRosterEvents(events).findLast( + (event) => event.providerThread.id === providerThreadB.id, + )?.providerThread.pendingBackgroundTasks; + assert.deepEqual(rosterBAfterFail ?? [], []); + assert.isFalse(yield* hasPendingBackgroundWorkForThread(providerThreadA)); + assert.isFalse(yield* hasPendingBackgroundWorkForThread(providerThreadB)); + assert.isFalse(yield* hasPendingBackgroundWork); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + it.effect("buffers wake output and requests a single continuation run", () => Effect.scoped( Effect.gen(function* () { @@ -1314,6 +1850,12 @@ describe("ClaudeAdapterV2 background wake turns", () => { assert.lengthOf(harness.continuationRequests, 0); yield* Queue.offer(harness.sdkMessages, wakeNotification); + let quietYields = 0; + yield* awaitUntil(() => quietYields++ >= 50, "notification-only quiet window"); + assert.lengthOf(harness.continuationRequests, 0); + assert.isFalse(yield* harness.hasPendingBackgroundWork); + + yield* Queue.offer(harness.sdkMessages, wakeAssistant); yield* awaitUntil(() => harness.continuationRequests.length === 1, "continuation request"); assert.equal(harness.continuationRequests[0]?.threadId, harness.threadId); assert.equal(harness.continuationRequests[0]?.providerThreadId, harness.providerThread.id); @@ -1330,6 +1872,57 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + it.effect("does not offer a continuation for notification-only opaque work", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-notification-only"), + text: "Start opaque background work.", + attachments: [], + }), + ); + yield* Queue.offer(harness.sdkMessages, wakeTaskStarted); + yield* Queue.offer(harness.sdkMessages, turnOneResult); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "first turn terminal"); + assert.isTrue(yield* harness.hasPendingBackgroundWork); + + yield* Queue.offer(harness.sdkMessages, wakeNotification); + let quietYields = 0; + yield* awaitUntil(() => quietYields++ >= 100, "notification-only quiet window"); + assert.lengthOf(harness.continuationRequests, 0); + assert.lengthOf(harness.terminalEvents(), 1); + assert.isFalse(yield* harness.hasPendingBackgroundWork); + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-notification-only-continuation"), + text: "Background task completed.", + attachments: [], + providerTurnOrdinal: 2, + messageCreatedBy: "agent", + messageCreationSource: "provider", + }), + ); + yield* awaitUntil( + () => harness.terminalEvents().length === 2, + "notification-only continuation terminal", + ); + assert.equal(harness.terminalEvents()[1]?.status, "completed"); + assert.lengthOf(harness.offeredMessages, 1); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + it.effect("drains buffered wake messages into a continuation turn", () => Effect.scoped( Effect.gen(function* () { @@ -1381,7 +1974,11 @@ describe("ClaudeAdapterV2 background wake turns", () => { ); // The background task never renders as a subagent node. assert.isFalse( - harness.events.some((event) => JSON.stringify(event).includes(WAKE_TASK_ID)), + harness.events.some( + (event) => + event.type !== "provider_thread.updated" && + JSON.stringify(event).includes(WAKE_TASK_ID), + ), ); assert.isFalse(yield* harness.hasPendingBackgroundWork); }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), @@ -1468,7 +2065,11 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); assert.isFalse( - harness.events.some((event) => JSON.stringify(event).includes(WAKE_TASK_ID)), + harness.events.some( + (event) => + event.type !== "provider_thread.updated" && + JSON.stringify(event).includes(WAKE_TASK_ID), + ), ); assert.isFalse(yield* harness.hasPendingBackgroundWork); }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), @@ -1652,10 +2253,10 @@ describe("ClaudeAdapterV2 background wake turns", () => { session_id: WAKE_NATIVE_SESSION, }), ); + yield* Queue.offer(harness.sdkMessages, wakeResult); yield* awaitUntil(() => harness.continuationRequests.length === 1, "continuation request"); assert.isNull(harness.continuationRequests[0]?.detail); - yield* Queue.offer(harness.sdkMessages, wakeResult); yield* harness.runtime.startTurn( makeClaudeTestTurnInput({ threadId: harness.threadId, @@ -2448,6 +3049,1243 @@ describe("ClaudeAdapterV2 background wake turns", () => { }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), ), ); + + it.effect( + "orders nonempty level, empty level, notification, and continuation drain without subagent projection", + () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + const nonemptyRoster = claudeSdkFrame({ + type: "system", + subtype: "background_tasks_changed", + tasks: [ + { + task_id: WAKE_TASK_ID, + description: "npm run build", + task_type: "local_bash", + }, + ], + uuid: "00000000-0000-4000-8000-000000000600", + session_id: WAKE_NATIVE_SESSION, + }); + const emptyRoster = claudeSdkFrame({ + type: "system", + subtype: "background_tasks_changed", + tasks: [], + uuid: "00000000-0000-4000-8000-000000000601", + session_id: WAKE_NATIVE_SESSION, + }); + const duplicateNotification = claudeSdkFrame({ + type: "system", + subtype: "task_notification", + task_id: WAKE_TASK_ID, + status: "completed", + output_file: "/tmp/task-wake-build-dup.log", + summary: "duplicate should not re-buffer", + uuid: "00000000-0000-4000-8000-000000000605", + session_id: WAKE_NATIVE_SESSION, + }); + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-level-before-edge-a"), + text: "Run the build in the background.", + attachments: [], + }), + ); + // 1) Nonempty authoritative level admits local_bash to Waiting + + // wake eligibility. + yield* Queue.offer(harness.sdkMessages, nonemptyRoster); + yield* awaitUntil( + () => + providerThreadRosterEvents(harness.events).some( + (event) => (event.providerThread.pendingBackgroundTasks?.length ?? 0) > 0, + ), + "nonempty level populated Waiting roster", + ); + assert.deepEqual( + providerThreadRosterEvents(harness.events).at(-1)?.providerThread + .pendingBackgroundTasks ?? [], + [ + { + taskId: WAKE_TASK_ID, + description: "npm run build", + taskType: "local_bash", + }, + ], + ); + yield* Queue.offer(harness.sdkMessages, turnOneResult); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "first turn terminal"); + assert.isTrue(yield* harness.hasPendingBackgroundWork); + + // 2) Empty level clears Waiting but keeps wake eligibility so the + // later notification can still offer exactly one continuation. + yield* Queue.offer(harness.sdkMessages, emptyRoster); + yield* awaitUntil( + () => + providerThreadRosterEvents(harness.events).some( + (event) => + event.providerThread.status === "idle" && + (event.providerThread.pendingBackgroundTasks?.length ?? 0) === 0, + ), + "empty level cleared Waiting roster", + ); + assert.isFalse(yield* harness.hasPendingBackgroundWork); + assert.lengthOf(harness.continuationRequests, 0); + + // 3) First idle notification buffers and consumes eligibility. The + // following native assistant frame proves Claude began a wake turn. + yield* Queue.offer(harness.sdkMessages, wakeNotification); + yield* Queue.offer(harness.sdkMessages, wakeAssistant); + yield* awaitUntil( + () => harness.continuationRequests.length === 1, + "continuation after level-before-edge", + ); + assert.equal(harness.continuationRequests[0]?.detail, WAKE_SUMMARY); + + // A duplicate notification must not re-buffer or re-offer. + yield* Queue.offer(harness.sdkMessages, duplicateNotification); + let settleYields = 0; + yield* awaitUntil(() => settleYields++ >= 50, "duplicate notification settle"); + assert.lengthOf(harness.continuationRequests, 1); + + // 4) Continuation drain classifies the buffered notification as + // local_bash (replay tombstone) and never fabricates a subagent. + yield* Queue.offer(harness.sdkMessages, wakeResult); + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-level-before-edge-b"), + text: "Background task completed.", + attachments: [], + providerTurnOrdinal: 2, + messageCreatedBy: "agent", + messageCreationSource: "provider", + }), + ); + yield* awaitUntil(() => harness.terminalEvents().length === 2, "continuation terminal"); + assert.equal(harness.terminalEvents()[1]?.status, "completed"); + assert.lengthOf(harness.continuationRequests, 1); + assert.isTrue( + harness.events.some( + (event) => + event.type === "message.updated" && event.message.text === WAKE_ASSISTANT_TEXT, + ), + ); + assert.isFalse( + harness.events.some( + (event) => + event.type === "subagent.updated" || + (event.type === "node.updated" && event.node.kind === "subagent"), + ), + ); + assert.isFalse( + harness.events.some( + (event) => + event.type !== "provider_thread.updated" && + JSON.stringify(event).includes(WAKE_TASK_ID), + ), + ); + assert.isFalse(yield* harness.hasPendingBackgroundWork); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("resets Waiting roster and wake eligibility when the CLI process is replaced", () => + Effect.scoped( + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const attachmentsDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-claude-v2-process-reset-", + }); + const processQueues: Array> = []; + const events: Array = []; + const continuationRequests: Array = []; + const adapter = makeClaudeAdapterV2({ + instanceId: CLAUDE_DEFAULT_INSTANCE_ID, + settings: DEFAULT_CLAUDE_SETTINGS, + environment: {}, + attachmentsDir, + fileSystem, + idAllocator, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + queryRunner: { + allocateSessionId: Effect.succeed(WAKE_NATIVE_SESSION), + open: () => + Effect.gen(function* () { + const sdkMessages = yield* Queue.unbounded(); + processQueues.push(sdkMessages); + return { + messages: Stream.fromQueue(sdkMessages), + offer: () => Effect.void, + setModel: () => Effect.void, + interrupt: Effect.void, + // End this process stream so openQuery can replace it. + close: Queue.shutdown(sdkMessages), + }; + }), + forkSession: () => Effect.die("unused forkSession"), + assertComplete: Effect.void, + }, + }); + const threadId = ThreadId.make("thread-claude-process-reset"); + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-claude-process-reset"), + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + yield* runtime.events.pipe( + Stream.runForEach((event) => + Effect.sync(() => { + events.push(event); + }), + ), + Effect.forkScoped, + ); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die("Claude adapter runtime must expose hasPendingBackgroundWork."); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const now = yield* DateTime.now; + + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-process-reset-a"), + text: "Run the build in the background.", + attachments: [], + }), + ); + assert.equal(processQueues.length, 1); + const firstProcess = processQueues[0]!; + yield* Queue.offer(firstProcess, wakeTaskStarted); + yield* Queue.offer(firstProcess, turnOneResult); + yield* awaitUntil( + () => events.some((event) => event.type === "turn.terminal"), + "first turn terminal", + ); + assert.isTrue(yield* hasPendingBackgroundWork); + + const alternateModel = { + ...CLAUDE_TEST_MODEL_SELECTION, + model: "claude-haiku-4-5-20251001", + } satisfies ModelSelection; + // ProviderTurnStartService marks the thread active before startTurn; + // the process-reset clear must preserve that status. + const activeProviderThread = { + ...providerThread, + status: "active" as const, + } satisfies OrchestrationV2ProviderThread; + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread: activeProviderThread, + now, + attemptId: RunAttemptId.make("attempt-claude-process-reset-b"), + text: "Continue after process restart.", + attachments: [], + providerTurnOrdinal: 2, + modelSelection: alternateModel, + }), + ); + assert.equal(processQueues.length, 2); + + // Process-scoped level resets to empty on CLI (re)start while the + // starting turn's provider thread remains active (not idle). + yield* awaitUntil( + () => + providerThreadRosterEvents(events).some( + (event) => + event.providerThread.status === "active" && + (event.providerThread.pendingBackgroundTasks?.length ?? 0) === 0 && + // Prefer the post-replace clear over the initial empty thread. + event.providerThread.updatedAt !== undefined, + ), + "roster cleared on process replace while remaining active", + ); + // After replace, the in-memory Waiting probe must be false even if a + // late empty-level event was already present before background work. + assert.isFalse(yield* hasPendingBackgroundWork); + const emptyActiveRosterEvents = providerThreadRosterEvents(events).filter( + (event) => + event.providerThread.status === "active" && + (event.providerThread.pendingBackgroundTasks?.length ?? 0) === 0, + ); + assert.isAtLeast(emptyActiveRosterEvents.length, 1); + assert.deepEqual( + emptyActiveRosterEvents.at(-1)?.providerThread.pendingBackgroundTasks ?? [], + [], + ); + assert.equal(emptyActiveRosterEvents.at(-1)?.providerThread.status, "active"); + + // A late notification from the previous process must not wake after + // eligibility was reset with the process. Offer on the new process + // stream (the old queue is shut down). + const secondProcess = processQueues[1]!; + yield* Queue.offer(secondProcess, wakeNotification); + let settleYields = 0; + yield* awaitUntil(() => settleYields++ >= 50, "stale notification settle"); + assert.lengthOf(continuationRequests, 0); + + yield* Queue.offer( + secondProcess, + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000602", + result: "Process restart turn finished.", + }), + ); + yield* awaitUntil( + () => events.filter((event) => event.type === "turn.terminal").length === 2, + "second turn terminal", + ); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("admits only local_bash from a mixed background_tasks_changed snapshot", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + const SUBAGENT_TASK_ID = "task-mixed-snapshot-subagent"; + const SUBAGENT_TOOL_USE_ID = "toolu-mixed-snapshot-subagent"; + const mixedSnapshot = claudeSdkFrame({ + type: "system", + subtype: "background_tasks_changed", + tasks: [ + { + task_id: WAKE_TASK_ID, + description: "npm run build", + task_type: "local_bash", + }, + { + task_id: SUBAGENT_TASK_ID, + description: "Agent review", + task_type: "local_agent", + }, + { + task_id: "task-mixed-foreground-agent", + description: "Backgrounded foreground agent", + task_type: "local_agent", + }, + ], + uuid: "00000000-0000-4000-8000-000000000603", + session_id: WAKE_NATIVE_SESSION, + }); + const subagentTaskStarted = claudeSdkFrame({ + type: "system", + subtype: "task_started", + task_id: SUBAGENT_TASK_ID, + tool_use_id: SUBAGENT_TOOL_USE_ID, + description: "Agent review", + subagent_type: "general-purpose", + task_type: "local_agent", + prompt: "Review the change.", + uuid: "00000000-0000-4000-8000-000000000604", + session_id: WAKE_NATIVE_SESSION, + }); + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-mixed-snapshot"), + text: "Background a bash task and a subagent.", + attachments: [], + }), + ); + yield* Queue.offer(harness.sdkMessages, subagentTaskStarted); + yield* awaitUntil( + () => + harness.events.some( + (event) => + event.type === "subagent.updated" && + event.subagent.nativeTaskRef?.nativeId === SUBAGENT_TASK_ID, + ), + "subagent projected normally", + ); + yield* Queue.offer(harness.sdkMessages, mixedSnapshot); + yield* awaitUntil( + () => + providerThreadRosterEvents(harness.events).some( + (event) => (event.providerThread.pendingBackgroundTasks?.length ?? 0) > 0, + ), + "roster after mixed snapshot", + ); + + const roster = providerThreadRosterEvents(harness.events).at(-1)?.providerThread + .pendingBackgroundTasks; + assert.deepEqual(roster ?? [], [ + { + taskId: WAKE_TASK_ID, + description: "npm run build", + taskType: "local_bash", + }, + ]); + // Subagent lifecycle stays on the subagent path, not the Waiting roster. + assert.isTrue( + harness.events.some( + (event) => + event.type === "subagent.updated" && + event.subagent.nativeTaskRef?.nativeId === SUBAGENT_TASK_ID && + event.subagent.status === "running", + ), + ); + assert.isFalse((roster ?? []).some((task) => task.taskId === SUBAGENT_TASK_ID)); + + yield* Queue.offer(harness.sdkMessages, turnOneResult); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "turn terminal"); + assert.isTrue(yield* harness.hasPendingBackgroundWork); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect( + "preserves buffered local_bash notification classification across model/policy query replacement", + () => + Effect.scoped( + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const attachmentsDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-claude-v2-buffer-replace-", + }); + const processQueues: Array> = []; + const events: Array = []; + const continuationRequests: Array = []; + const adapter = makeClaudeAdapterV2({ + instanceId: CLAUDE_DEFAULT_INSTANCE_ID, + settings: DEFAULT_CLAUDE_SETTINGS, + environment: {}, + attachmentsDir, + fileSystem, + idAllocator, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + queryRunner: { + allocateSessionId: Effect.succeed(WAKE_NATIVE_SESSION), + open: () => + Effect.gen(function* () { + const sdkMessages = yield* Queue.unbounded(); + processQueues.push(sdkMessages); + return { + messages: Stream.fromQueue(sdkMessages), + offer: () => Effect.void, + setModel: () => Effect.void, + interrupt: Effect.void, + close: Queue.shutdown(sdkMessages), + }; + }), + forkSession: () => Effect.die("unused forkSession"), + assertComplete: Effect.void, + }, + }); + const threadId = ThreadId.make("thread-claude-buffer-replace"); + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-claude-buffer-replace"), + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + yield* runtime.events.pipe( + Stream.runForEach((event) => + Effect.sync(() => { + events.push(event); + }), + ), + Effect.forkScoped, + ); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "Claude adapter runtime must expose hasPendingBackgroundWork.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const now = yield* DateTime.now; + + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-buffer-replace-a"), + text: "Run the build in the background.", + attachments: [], + }), + ); + assert.equal(processQueues.length, 1); + const firstProcess = processQueues[0]!; + yield* Queue.offer(firstProcess, wakeTaskStarted); + yield* Queue.offer(firstProcess, turnOneResult); + yield* awaitUntil( + () => events.some((event) => event.type === "turn.terminal"), + "first turn terminal", + ); + assert.isTrue(yield* hasPendingBackgroundWork); + + // Idle completion notification buffers before any continuation runs. + yield* Queue.offer(firstProcess, wakeNotification); + let quietYields = 0; + yield* awaitUntil(() => quietYields++ >= 50, "notification-only quiet window"); + assert.lengthOf(continuationRequests, 0); + + // User turn changes model, replacing the query while the wake buffer + // stays queued for the later provider continuation. + const alternateModel = { + ...CLAUDE_TEST_MODEL_SELECTION, + model: "claude-haiku-4-5-20251001", + } satisfies ModelSelection; + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread: { ...providerThread, status: "active" }, + now, + attemptId: RunAttemptId.make("attempt-claude-buffer-replace-user"), + text: "Switch model while background work completes.", + attachments: [], + providerTurnOrdinal: 2, + modelSelection: alternateModel, + }), + ); + assert.equal(processQueues.length, 2); + const secondProcess = processQueues[1]!; + yield* Queue.offer( + secondProcess, + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000701", + result: "User turn finished after model switch.", + }), + ); + yield* awaitUntil( + () => events.filter((event) => event.type === "turn.terminal").length === 2, + "user turn terminal after replace", + ); + // The terminal notification remains buffered for classification, but + // notification-only traffic no longer pins pending work. + assert.isFalse(yield* hasPendingBackgroundWork); + assert.lengthOf(continuationRequests, 0); + + // Continuation drains the buffered local_bash notification with no + // fabricated subagent/node and attributes the wake result text. + yield* Queue.offer(secondProcess, wakeResult); + yield* awaitUntil(() => continuationRequests.length === 1, "continuation after result"); + assert.equal(continuationRequests[0]?.detail, WAKE_SUMMARY); + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread: { ...providerThread, status: "active" }, + now, + attemptId: RunAttemptId.make("attempt-claude-buffer-replace-cont"), + text: "Background task completed.", + attachments: [], + providerTurnOrdinal: 3, + modelSelection: alternateModel, + messageCreatedBy: "agent", + messageCreationSource: "provider", + }), + ); + yield* awaitUntil( + () => events.filter((event) => event.type === "turn.terminal").length === 3, + "continuation terminal after buffered drain", + ); + assert.isTrue( + events.some( + (event) => + event.type === "message.updated" && event.message.text === WAKE_RESULT_TEXT, + ), + ); + assert.isFalse( + events.some( + (event) => + event.type === "subagent.updated" || + (event.type === "node.updated" && event.node.kind === "subagent"), + ), + ); + // Must not re-project the opaque task id as anything but roster history. + assert.isFalse( + events.some( + (event) => + event.type !== "provider_thread.updated" && + JSON.stringify(event).includes(WAKE_TASK_ID), + ), + ); + assert.isFalse(yield* hasPendingBackgroundWork); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect( + "does not opaque-misclassify a buffered subagent notification across model/policy query replacement", + () => + Effect.scoped( + Effect.gen(function* () { + const SUBAGENT_TASK_ID = "task-buffer-replace-subagent"; + const SUBAGENT_TOOL_USE_ID = "toolu-buffer-replace-subagent"; + const SUBAGENT_SUMMARY = "SUB_BUFFER_REPLACE_DONE"; + const subagentTaskStarted = claudeSdkFrame({ + type: "system", + subtype: "task_started", + task_id: SUBAGENT_TASK_ID, + tool_use_id: SUBAGENT_TOOL_USE_ID, + description: "Background research", + subagent_type: "general-purpose", + task_type: "local_agent", + prompt: "Research then return SUB_BUFFER_REPLACE_DONE.", + uuid: "00000000-0000-4000-8000-000000000801", + session_id: WAKE_NATIVE_SESSION, + }); + const subagentNotification = claudeSdkFrame({ + type: "system", + subtype: "task_notification", + task_id: SUBAGENT_TASK_ID, + tool_use_id: SUBAGENT_TOOL_USE_ID, + status: "completed", + output_file: "/tmp/task-buffer-replace-subagent.output", + summary: SUBAGENT_SUMMARY, + uuid: "00000000-0000-4000-8000-000000000802", + session_id: WAKE_NATIVE_SESSION, + }); + const subagentAsyncAck = claudeSdkFrame({ + type: "user", + message: { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: SUBAGENT_TOOL_USE_ID, + content: [{ type: "text", text: "Async agent launched successfully." }], + }, + ], + }, + parent_tool_use_id: null, + uuid: "00000000-0000-4000-8000-000000000803", + session_id: WAKE_NATIVE_SESSION, + tool_use_result: { + isAsync: true, + status: "async_launched", + agentId: SUBAGENT_TASK_ID, + prompt: "Research then return SUB_BUFFER_REPLACE_DONE.", + }, + }); + + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const attachmentsDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-claude-v2-subagent-buffer-replace-", + }); + const processQueues: Array> = []; + const events: Array = []; + const continuationRequests: Array = []; + const adapter = makeClaudeAdapterV2({ + instanceId: CLAUDE_DEFAULT_INSTANCE_ID, + settings: DEFAULT_CLAUDE_SETTINGS, + environment: {}, + attachmentsDir, + fileSystem, + idAllocator, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + queryRunner: { + allocateSessionId: Effect.succeed(WAKE_NATIVE_SESSION), + open: () => + Effect.gen(function* () { + const sdkMessages = yield* Queue.unbounded(); + processQueues.push(sdkMessages); + return { + messages: Stream.fromQueue(sdkMessages), + offer: () => Effect.void, + setModel: () => Effect.void, + interrupt: Effect.void, + close: Queue.shutdown(sdkMessages), + }; + }), + forkSession: () => Effect.die("unused forkSession"), + assertComplete: Effect.void, + }, + }); + const threadId = ThreadId.make("thread-claude-subagent-buffer-replace"); + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-claude-subagent-buffer-replace", + ), + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + yield* runtime.events.pipe( + Stream.runForEach((event) => + Effect.sync(() => { + events.push(event); + }), + ), + Effect.forkScoped, + ); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "Claude adapter runtime must expose hasPendingBackgroundWork.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const subagentEvents = () => + events.filter( + (event): event is Extract => + event.type === "subagent.updated", + ); + const now = yield* DateTime.now; + + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-subagent-buffer-replace-a"), + text: "Spawn a background subagent and stop.", + attachments: [], + }), + ); + assert.equal(processQueues.length, 1); + const firstProcess = processQueues[0]!; + yield* Queue.offer(firstProcess, subagentTaskStarted); + yield* awaitUntil(() => subagentEvents().length >= 1, "subagent node created"); + assert.equal(subagentEvents()[0]?.subagent.status, "running"); + yield* Queue.offer(firstProcess, subagentAsyncAck); + yield* Queue.offer( + firstProcess, + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000804", + result: "Spawned the subagent in the background.", + }), + ); + yield* awaitUntil( + () => events.some((event) => event.type === "turn.terminal"), + "first turn terminal", + ); + assert.isTrue(yield* hasPendingBackgroundWork); + + // Session-registered subagent completion buffers; no opaque tombstone. + yield* Queue.offer(firstProcess, subagentNotification); + yield* awaitUntil(() => continuationRequests.length === 1, "continuation after notify"); + assert.equal(continuationRequests[0]?.detail, SUBAGENT_SUMMARY); + + // Model-changing user turn replaces the query while continuation stays + // queued. Process reset must not invent opaque classification for the + // buffered subagent notification. + const alternateModel = { + ...CLAUDE_TEST_MODEL_SELECTION, + model: "claude-haiku-4-5-20251001", + } satisfies ModelSelection; + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread: { ...providerThread, status: "active" }, + now, + attemptId: RunAttemptId.make("attempt-claude-subagent-buffer-replace-user"), + text: "Switch model while the subagent completes.", + attachments: [], + providerTurnOrdinal: 2, + modelSelection: alternateModel, + }), + ); + assert.equal(processQueues.length, 2); + const secondProcess = processQueues[1]!; + yield* Queue.offer( + secondProcess, + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000805", + result: "User turn finished after model switch.", + }), + ); + yield* awaitUntil( + () => events.filter((event) => event.type === "turn.terminal").length === 2, + "user turn terminal after replace", + ); + assert.isTrue(yield* hasPendingBackgroundWork); + assert.lengthOf(continuationRequests, 1); + + yield* Queue.offer( + secondProcess, + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000806", + result: "The subagent finished with SUB_BUFFER_REPLACE_DONE.", + }), + ); + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread: { ...providerThread, status: "active" }, + now, + attemptId: RunAttemptId.make("attempt-claude-subagent-buffer-replace-cont"), + text: "Background task completed.", + attachments: [], + providerTurnOrdinal: 3, + modelSelection: alternateModel, + messageCreatedBy: "agent", + messageCreationSource: "provider", + }), + ); + yield* awaitUntil( + () => events.filter((event) => event.type === "turn.terminal").length === 3, + "continuation terminal after buffered subagent drain", + ); + + const finalSubagent = subagentEvents().at(-1)?.subagent; + assert.equal(finalSubagent?.status, "completed"); + assert.equal(finalSubagent?.result, SUBAGENT_SUMMARY); + assert.equal(finalSubagent?.runId, subagentEvents()[0]?.subagent.runId); + const subagentNodeEvents = events.filter( + (event): event is Extract => + event.type === "node.updated" && + event.node.kind === "subagent" && + event.node.nativeItemRef?.nativeId === SUBAGENT_TASK_ID, + ); + assert.equal(subagentNodeEvents.at(-1)?.node.status, "completed"); + assert.isFalse(yield* hasPendingBackgroundWork); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect( + "clears process-scoped roster when same-native-thread replacement open fails after close", + () => + Effect.scoped( + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const attachmentsDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-claude-v2-replace-open-fail-", + }); + let openCount = 0; + const processQueues: Array> = []; + const events: Array = []; + const adapter = makeClaudeAdapterV2({ + instanceId: CLAUDE_DEFAULT_INSTANCE_ID, + settings: DEFAULT_CLAUDE_SETTINGS, + environment: {}, + attachmentsDir, + fileSystem, + idAllocator, + continuationRequests: { + offer: () => Effect.void, + }, + queryRunner: { + allocateSessionId: Effect.succeed(WAKE_NATIVE_SESSION), + open: () => { + openCount += 1; + if (openCount === 2) { + return Effect.fail( + new ClaudeAgentSdkQueryRunnerError({ + method: "open", + cause: "forced replacement open failure", + }), + ); + } + return Effect.gen(function* () { + const sdkMessages = yield* Queue.unbounded(); + processQueues.push(sdkMessages); + return { + messages: Stream.fromQueue(sdkMessages), + offer: () => Effect.void, + setModel: () => Effect.void, + interrupt: Effect.void, + close: Queue.shutdown(sdkMessages), + }; + }); + }, + forkSession: () => Effect.die("unused forkSession"), + assertComplete: Effect.void, + }, + }); + const threadId = ThreadId.make("thread-claude-replace-open-fail"); + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-claude-replace-open-fail"), + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + yield* runtime.events.pipe( + Stream.runForEach((event) => + Effect.sync(() => { + events.push(event); + }), + ), + Effect.forkScoped, + ); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "Claude adapter runtime must expose hasPendingBackgroundWork.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const now = yield* DateTime.now; + + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-replace-open-fail-a"), + text: "Run the build in the background.", + attachments: [], + }), + ); + assert.equal(processQueues.length, 1); + yield* Queue.offer(processQueues[0]!, wakeTaskStarted); + yield* Queue.offer(processQueues[0]!, turnOneResult); + yield* awaitUntil( + () => events.some((event) => event.type === "turn.terminal"), + "first turn terminal", + ); + assert.isTrue(yield* hasPendingBackgroundWork); + + const alternateModel = { + ...CLAUDE_TEST_MODEL_SELECTION, + model: "claude-haiku-4-5-20251001", + } satisfies ModelSelection; + const failedStart = yield* runtime + .startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread: { ...providerThread, status: "active" }, + now, + attemptId: RunAttemptId.make("attempt-claude-replace-open-fail-b"), + text: "Replace process but fail open.", + attachments: [], + providerTurnOrdinal: 2, + modelSelection: alternateModel, + }), + ) + .pipe(Effect.exit); + assert.isTrue(Exit.isFailure(failedStart)); + // Old process was closed before the failed open: roster must not stick. + yield* awaitUntil( + () => + providerThreadRosterEvents(events).some( + (event) => + event.providerThread.status === "idle" && + (event.providerThread.pendingBackgroundTasks?.length ?? 0) === 0, + ), + "roster cleared after failed same-thread replacement open", + ); + assert.isFalse(yield* hasPendingBackgroundWork); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect( + "clears buffered wake and continuation state when same-native-thread replacement open fails", + () => + Effect.scoped( + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const attachmentsDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-claude-v2-replace-open-fail-wake-", + }); + let openCount = 0; + const processQueues: Array> = []; + const events: Array = []; + const continuationRequests: Array = []; + const adapter = makeClaudeAdapterV2({ + instanceId: CLAUDE_DEFAULT_INSTANCE_ID, + settings: DEFAULT_CLAUDE_SETTINGS, + environment: {}, + attachmentsDir, + fileSystem, + idAllocator, + continuationRequests: { + offer: (request) => + Effect.sync(() => { + continuationRequests.push(request); + }), + }, + queryRunner: { + allocateSessionId: Effect.succeed(WAKE_NATIVE_SESSION), + open: () => { + openCount += 1; + if (openCount === 2) { + return Effect.fail( + new ClaudeAgentSdkQueryRunnerError({ + method: "open", + cause: "forced replacement open failure", + }), + ); + } + return Effect.gen(function* () { + const sdkMessages = yield* Queue.unbounded(); + processQueues.push(sdkMessages); + return { + messages: Stream.fromQueue(sdkMessages), + offer: () => Effect.void, + setModel: () => Effect.void, + interrupt: Effect.void, + close: Queue.shutdown(sdkMessages), + }; + }); + }, + forkSession: () => Effect.die("unused forkSession"), + assertComplete: Effect.void, + }, + }); + const threadId = ThreadId.make("thread-claude-replace-open-fail-wake"); + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make( + "provider-session-claude-replace-open-fail-wake", + ), + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + yield* runtime.events.pipe( + Stream.runForEach((event) => + Effect.sync(() => { + events.push(event); + }), + ), + Effect.forkScoped, + ); + if (runtime.hasPendingBackgroundWork === undefined) { + return yield* Effect.die( + "Claude adapter runtime must expose hasPendingBackgroundWork.", + ); + } + const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; + const now = yield* DateTime.now; + + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-replace-open-fail-wake-a"), + text: "Run the build in the background.", + attachments: [], + }), + ); + yield* Queue.offer(processQueues[0]!, wakeTaskStarted); + yield* Queue.offer(processQueues[0]!, turnOneResult); + yield* awaitUntil( + () => events.filter((event) => event.type === "turn.terminal").length === 1, + "first turn terminal", + ); + yield* Queue.offer(processQueues[0]!, wakeNotification); + yield* Queue.offer(processQueues[0]!, wakeAssistant); + yield* awaitUntil(() => continuationRequests.length === 1, "first continuation request"); + assert.isTrue(yield* hasPendingBackgroundWork); + + const alternateModel = { + ...CLAUDE_TEST_MODEL_SELECTION, + model: "claude-haiku-4-5-20251001", + } satisfies ModelSelection; + const failedStart = yield* runtime + .startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread: { ...providerThread, status: "active" }, + now, + attemptId: RunAttemptId.make("attempt-claude-replace-open-fail-wake-b"), + text: "Replace process but fail open.", + attachments: [], + providerTurnOrdinal: 2, + modelSelection: alternateModel, + }), + ) + .pipe(Effect.exit); + assert.isTrue(Exit.isFailure(failedStart)); + assert.isFalse(yield* hasPendingBackgroundWork); + + yield* runtime.startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread: { ...providerThread, status: "active" }, + now, + attemptId: RunAttemptId.make("attempt-claude-replace-open-fail-wake-c"), + text: "Retry after the failed replacement.", + attachments: [], + providerTurnOrdinal: 2, + modelSelection: alternateModel, + }), + ); + const retryProcess = processQueues[1]!; + const retryTaskId = "task-wake-build-after-retry"; + yield* Queue.offer( + retryProcess, + claudeSdkFrame({ + type: "system", + subtype: "task_started", + task_id: retryTaskId, + description: "npm run build after retry", + task_type: "local_bash", + uuid: "00000000-0000-4000-8000-000000000901", + session_id: WAKE_NATIVE_SESSION, + }), + ); + yield* Queue.offer( + retryProcess, + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000902", + result: "Kicked off the retry build in the background.", + }), + ); + yield* awaitUntil( + () => events.filter((event) => event.type === "turn.terminal").length === 2, + "retry turn terminal", + ); + yield* Queue.offer( + retryProcess, + claudeSdkFrame({ + type: "system", + subtype: "task_notification", + task_id: retryTaskId, + status: "completed", + output_file: "/tmp/task-wake-build-after-retry.log", + summary: "Retry build completed successfully", + uuid: "00000000-0000-4000-8000-000000000903", + session_id: WAKE_NATIVE_SESSION, + }), + ); + yield* Queue.offer( + retryProcess, + claudeSdkFrame({ + type: "assistant", + message: { + role: "assistant", + content: [{ type: "text", text: "The retry build has finished." }], + }, + parent_tool_use_id: null, + uuid: "00000000-0000-4000-8000-000000000904", + session_id: WAKE_NATIVE_SESSION, + }), + ); + yield* awaitUntil( + () => continuationRequests.length === 2, + "continuation request after retry", + ); + assert.equal(continuationRequests[1]?.detail, "Retry build completed successfully"); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("does not invent process reset state on a first-ever failed open", () => + Effect.scoped( + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const idAllocator = yield* IdAllocatorV2; + const attachmentsDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-claude-v2-first-open-fail-", + }); + const events: Array = []; + const adapter = makeClaudeAdapterV2({ + instanceId: CLAUDE_DEFAULT_INSTANCE_ID, + settings: DEFAULT_CLAUDE_SETTINGS, + environment: {}, + attachmentsDir, + fileSystem, + idAllocator, + continuationRequests: { + offer: () => Effect.void, + }, + queryRunner: { + allocateSessionId: Effect.succeed(WAKE_NATIVE_SESSION), + open: () => + Effect.fail( + new ClaudeAgentSdkQueryRunnerError({ + method: "open", + cause: "forced first open failure", + }), + ), + forkSession: () => Effect.die("unused forkSession"), + assertComplete: Effect.void, + }, + }); + const threadId = ThreadId.make("thread-claude-first-open-fail"); + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("provider-session-claude-first-open-fail"), + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + const providerThread = yield* runtime.ensureThread({ + threadId, + modelSelection: CLAUDE_TEST_MODEL_SELECTION, + runtimePolicy: CLAUDE_TEST_RUNTIME_POLICY, + }); + yield* runtime.events.pipe( + Stream.runForEach((event) => + Effect.sync(() => { + events.push(event); + }), + ), + Effect.forkScoped, + ); + const now = yield* DateTime.now; + const failedStart = yield* runtime + .startTurn( + makeClaudeTestTurnInput({ + threadId, + providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-first-open-fail"), + text: "First open fails.", + attachments: [], + }), + ) + .pipe(Effect.exit); + assert.isTrue(Exit.isFailure(failedStart)); + // No live process ever existed: do not emit a fabricated empty roster. + assert.lengthOf(providerThreadRosterEvents(events), 0); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); }); describe("ClaudeAdapterV2 query message stream", () => { diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index c681d89ab7a..f27b2b8c116 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -27,6 +27,7 @@ import { type OrchestrationV2ExecutionNode, type OrchestrationV2ProviderCapabilities, type OrchestrationV2ProviderFailure, + type OrchestrationV2PendingBackgroundTask, type OrchestrationV2ProviderSession, type OrchestrationV2ProviderThread, type OrchestrationV2ProviderTurn, @@ -1393,6 +1394,15 @@ function commandInputFromClaudeTool(toolName: string, input: ClaudeNativeToolInp ); } +// Opaque non-subagent background work admitted onto the Waiting roster. +// Subagents project through the normal subagent lifecycle and must not be +// double-counted when background_tasks_changed includes them. +const CLAUDE_OPAQUE_BACKGROUND_TASK_TYPES = new Set(["local_bash"]); + +function isClaudeOpaqueBackgroundTaskType(taskType: string | null | undefined): boolean { + return typeof taskType === "string" && CLAUDE_OPAQUE_BACKGROUND_TASK_TYPES.has(taskType); +} + function claudeTaskTypeFromSdkMessage(message: SDKMessage): string | null { if (typeof message !== "object" || message === null) { return null; @@ -1402,7 +1412,45 @@ function claudeTaskTypeFromSdkMessage(message: SDKMessage): string | null { } function isClaudeNonSubagentTask(message: SDKMessage): boolean { - return claudeTaskTypeFromSdkMessage(message) === "local_bash"; + return isClaudeOpaqueBackgroundTaskType(claudeTaskTypeFromSdkMessage(message)); +} + +function isClaudeBackgroundTasksChangedMessage(message: SDKMessage): boolean { + return ( + message.type === "system" && + // Undeclared SDK subtype: full roster snapshot of live background tasks. + (message.subtype as string) === "background_tasks_changed" + ); +} + +function claudePendingBackgroundTasksFromRoster( + roster: ReadonlyMap, +): ReadonlyArray { + return Array.from(roster.values()); +} + +function parseClaudeBackgroundTaskEntry( + entry: unknown, +): OrchestrationV2PendingBackgroundTask | null { + if (entry === null || typeof entry !== "object") { + return null; + } + const taskId = Reflect.get(entry, "task_id"); + if (typeof taskId !== "string" || taskId.length === 0) { + return null; + } + const taskType = Reflect.get(entry, "task_type"); + // Mirror the incremental path: only opaque non-subagent types currently + // supported for Waiting. Subagent/agent entries stay on the subagent path. + if (!isClaudeOpaqueBackgroundTaskType(typeof taskType === "string" ? taskType : null)) { + return null; + } + const description = Reflect.get(entry, "description"); + return { + taskId, + ...(typeof description === "string" && description.trim().length > 0 ? { description } : {}), + taskType, + }; } function fileNameFromClaudeTool(toolName: string, input: ClaudeNativeToolInput): string { @@ -2008,7 +2056,32 @@ export function makeClaudeAdapterV2( { readonly threadId: ThreadId; readonly providerThreadId: ProviderThreadId } >(), ); - const pendingBackgroundTaskIds = yield* Ref.make(new Set()); + // Authoritative + incremental background-task roster for post-settle + // Waiting UI. Outer key is native Claude session id so concurrent + // provider threads on one runtime cannot share or clear each other. + const pendingBackgroundTasksByNativeThread = yield* Ref.make( + new Map>(), + ); + // Wake eligibility is separate from the Waiting roster. It survives + // empty background_tasks_changed levels (SDK: empty level can precede + // task_notification) and is consumed when the first idle notification + // is buffered/offered so duplicates cannot re-buffer. A short-lived + // replay tombstone then covers continuation drain classification so + // local_bash is never projected as a subagent; the tombstone is + // cleared after that drained notification is processed. Both sets + // clear on CLI process open/replacement and failed/interrupted turns. + const wakeEligibleBackgroundTasksByNativeThread = yield* Ref.make( + new Map>(), + ); + const opaqueBackgroundTaskReplayTombstonesByNativeThread = yield* Ref.make( + new Map>(), + ); + // Last known provider-thread payload per native session, used to emit + // roster-only provider_thread.updated events between turns without + // resurrecting an active status after root settlement. + const lastProviderThreadByNativeThread = yield* Ref.make( + new Map(), + ); // Subagent registry that survives turn settle: a background subagent // (Agent with run_in_background) can complete after the root turn // ended, and its task_notification must both count as wake evidence @@ -2028,6 +2101,359 @@ export function makeClaudeAdapterV2( const emitProviderEvent = (event: ProviderAdapterV2Event) => Queue.offer(events, event).pipe(Effect.asVoid); + const rememberProviderThread = (providerThread: OrchestrationV2ProviderThread) => + Effect.gen(function* () { + const nativeThreadId = providerThread.nativeThreadRef?.nativeId; + if (nativeThreadId === undefined || nativeThreadId === null) { + return; + } + yield* Ref.update(lastProviderThreadByNativeThread, (current) => + new Map(current).set(nativeThreadId, providerThread), + ); + }); + + const rosterForNativeThread = ( + all: ReadonlyMap>, + nativeThreadId: string, + ): Map => + all.get(nativeThreadId) ?? new Map(); + + const hasPendingBackgroundTaskOnNativeThread = (nativeThreadId: string, taskId: string) => + Ref.get(pendingBackgroundTasksByNativeThread).pipe( + Effect.map((all) => rosterForNativeThread(all, nativeThreadId).has(taskId)), + ); + + const taskIdSetForNativeThread = ( + all: ReadonlyMap>, + nativeThreadId: string, + ): Set => all.get(nativeThreadId) ?? new Set(); + + const addTaskIdsToNativeThreadSet = ( + ref: Ref.Ref>>, + nativeThreadId: string, + taskIds: ReadonlyArray, + ) => + Ref.update(ref, (current) => { + if (taskIds.length === 0) { + return current; + } + const next = new Set(taskIdSetForNativeThread(current, nativeThreadId)); + let changed = false; + for (const taskId of taskIds) { + if (!next.has(taskId)) { + next.add(taskId); + changed = true; + } + } + return changed ? new Map(current).set(nativeThreadId, next) : current; + }); + + const clearTaskIdFromNativeThreadSet = ( + ref: Ref.Ref>>, + nativeThreadId: string, + taskId: string, + ) => + Ref.update(ref, (current) => { + const existing = taskIdSetForNativeThread(current, nativeThreadId); + if (!existing.has(taskId)) { + return current; + } + const next = new Set(existing); + next.delete(taskId); + const updated = new Map(current); + if (next.size === 0) { + updated.delete(nativeThreadId); + } else { + updated.set(nativeThreadId, next); + } + return updated; + }); + + const clearNativeThreadTaskIdSet = ( + ref: Ref.Ref>>, + nativeThreadId: string, + ) => + Ref.update(ref, (current) => { + if (!current.has(nativeThreadId)) { + return current; + } + const updated = new Map(current); + updated.delete(nativeThreadId); + return updated; + }); + + // First-notification wake offering only: not the Waiting roster and + // not the post-buffer replay tombstone. + const isWakeEligibleOpaqueBackgroundTaskOnNativeThread = ( + nativeThreadId: string, + taskId: string, + ) => + Ref.get(wakeEligibleBackgroundTasksByNativeThread).pipe( + Effect.map((all) => taskIdSetForNativeThread(all, nativeThreadId).has(taskId)), + ); + + const hasOpaqueBackgroundTaskReplayTombstoneOnNativeThread = ( + nativeThreadId: string, + taskId: string, + ) => + Ref.get(opaqueBackgroundTaskReplayTombstonesByNativeThread).pipe( + Effect.map((all) => taskIdSetForNativeThread(all, nativeThreadId).has(taskId)), + ); + + // Classify a task_notification as opaque local_bash (not a subagent): + // live roster, still-eligible first notification, or short-lived + // replay tombstone left after the idle notification was buffered. + const isKnownOpaqueBackgroundTaskOnNativeThread = ( + nativeThreadId: string, + taskId: string, + ) => + Effect.gen(function* () { + if (yield* hasPendingBackgroundTaskOnNativeThread(nativeThreadId, taskId)) { + return true; + } + if (yield* isWakeEligibleOpaqueBackgroundTaskOnNativeThread(nativeThreadId, taskId)) { + return true; + } + return yield* hasOpaqueBackgroundTaskReplayTombstoneOnNativeThread( + nativeThreadId, + taskId, + ); + }); + + // Admit onto wake eligibility only. Replay tombstones are created when + // the first idle notification is buffered, not at task start. + const markWakeEligibleOpaqueBackgroundTasks = ( + nativeThreadId: string, + taskIds: ReadonlyArray, + ) => + addTaskIdsToNativeThreadSet( + wakeEligibleBackgroundTasksByNativeThread, + nativeThreadId, + taskIds, + ); + + // After the first idle opaque notification is buffered/offered: stop + // further wake buffering for this task id, but keep a replay tombstone + // until the continuation drain classifies the buffered notification. + const consumeWakeEligibilityForBufferedNotification = ( + nativeThreadId: string, + taskId: string, + ) => + Effect.gen(function* () { + yield* clearTaskIdFromNativeThreadSet( + wakeEligibleBackgroundTasksByNativeThread, + nativeThreadId, + taskId, + ); + yield* addTaskIdsToNativeThreadSet( + opaqueBackgroundTaskReplayTombstonesByNativeThread, + nativeThreadId, + [taskId], + ); + }); + + const clearOpaqueBackgroundTaskReplayTombstone = (nativeThreadId: string, taskId: string) => + clearTaskIdFromNativeThreadSet( + opaqueBackgroundTaskReplayTombstonesByNativeThread, + nativeThreadId, + taskId, + ); + + const emitProviderThreadRoster = Effect.fnUntraced(function* (input: { + readonly nativeThreadId: string; + readonly providerThread: OrchestrationV2ProviderThread; + readonly status?: OrchestrationV2ProviderThread["status"]; + }) { + const roster = rosterForNativeThread( + yield* Ref.get(pendingBackgroundTasksByNativeThread), + input.nativeThreadId, + ); + const now = yield* DateTime.now; + const providerThread: OrchestrationV2ProviderThread = { + ...input.providerThread, + providerSessionId: session.id, + ...(input.status === undefined ? {} : { status: input.status }), + pendingBackgroundTasks: claudePendingBackgroundTasksFromRoster(roster), + updatedAt: now, + }; + yield* rememberProviderThread(providerThread); + yield* emitProviderEvent({ + type: "provider_thread.updated", + driver: CLAUDE_PROVIDER, + providerThread, + }); + }); + + const replacePendingBackgroundTasks = ( + nativeThreadId: string, + tasks: ReadonlyArray, + ) => + Effect.gen(function* () { + yield* Ref.update(pendingBackgroundTasksByNativeThread, (current) => { + const updated = new Map(current); + if (tasks.length === 0) { + updated.delete(nativeThreadId); + } else { + updated.set( + nativeThreadId, + new Map(tasks.map((task) => [task.taskId, task] as const)), + ); + } + return updated; + }); + // Empty level must not drop wake eligibility: notification may + // still be in flight. Non-empty level admits new task ids to + // wake eligibility only (replay tombstones are edge-created). + if (tasks.length > 0) { + yield* markWakeEligibleOpaqueBackgroundTasks( + nativeThreadId, + tasks.map((task) => task.taskId), + ); + } + }); + + const upsertPendingBackgroundTask = ( + nativeThreadId: string, + task: OrchestrationV2PendingBackgroundTask, + ) => + Effect.gen(function* () { + yield* Ref.update(pendingBackgroundTasksByNativeThread, (current) => { + const roster = new Map(rosterForNativeThread(current, nativeThreadId)); + roster.set(task.taskId, task); + return new Map(current).set(nativeThreadId, roster); + }); + yield* markWakeEligibleOpaqueBackgroundTasks(nativeThreadId, [task.taskId]); + }); + + const clearPendingBackgroundTask = (nativeThreadId: string, taskId: string) => + Ref.modify(pendingBackgroundTasksByNativeThread, (current) => { + const roster = rosterForNativeThread(current, nativeThreadId); + if (!roster.has(taskId)) { + return [false, current] as const; + } + const nextRoster = new Map(roster); + nextRoster.delete(taskId); + const updated = new Map(current); + if (nextRoster.size === 0) { + updated.delete(nativeThreadId); + } else { + updated.set(nativeThreadId, nextRoster); + } + return [true, updated] as const; + }); + + const clearPendingBackgroundTasksForNativeThread = (nativeThreadId: string) => + Ref.update(pendingBackgroundTasksByNativeThread, (current) => { + if (!current.has(nativeThreadId)) { + return current; + } + const updated = new Map(current); + updated.delete(nativeThreadId); + return updated; + }); + + // Drop idle wake traffic for a dead native process so it cannot pin + // session-wide pending work after sibling query replacement. + const clearWakeStateForNativeThread = (nativeThreadId: string) => + Effect.gen(function* () { + yield* Ref.update(wakeBuffers, (current) => { + if (!current.has(nativeThreadId)) { + return current; + } + const updated = new Map(current); + updated.delete(nativeThreadId); + return updated; + }); + yield* Ref.update(requestedContinuations, (current) => { + if (!current.has(nativeThreadId)) { + return current; + } + const updated = new Set(current); + updated.delete(nativeThreadId); + return updated; + }); + }); + + // Process-scoped level: SDK emits nothing at CLI start, so both the + // Waiting roster and wake eligibility reset when a live query opens + // or is replaced for this native thread. Opaque replay tombstones that + // already covered buffered task_notification frames are restored so a + // model/policy query replacement still classifies those local_bash + // completions on continuation drain. Buffer membership alone must not + // invent opaque classification: session-registered subagent + // notifications share the same buffer. + const resetBackgroundTaskStateForNativeThreadProcess = Effect.fnUntraced(function* ( + nativeThreadId: string, + options?: { + // openQuery during startTurn: activeTurn is not installed yet, but + // ProviderTurnStartService already marked the provider thread active. + readonly status?: OrchestrationV2ProviderThread["status"]; + }, + ) { + const hadRoster = + rosterForNativeThread( + yield* Ref.get(pendingBackgroundTasksByNativeThread), + nativeThreadId, + ).size > 0; + const remembered = (yield* Ref.get(lastProviderThreadByNativeThread)).get(nativeThreadId); + const hadPersistedRoster = (remembered?.pendingBackgroundTasks?.length ?? 0) > 0; + const priorOpaqueTombstones = taskIdSetForNativeThread( + yield* Ref.get(opaqueBackgroundTaskReplayTombstonesByNativeThread), + nativeThreadId, + ); + const bufferedTaskNotificationIds = new Set(); + const buffered = (yield* Ref.get(wakeBuffers)).get(nativeThreadId); + if (buffered !== undefined) { + for (const message of buffered.messages) { + if (message.type === "system" && message.subtype === "task_notification") { + bufferedTaskNotificationIds.add(message.task_id); + } + } + } + const preservedOpaqueTombstones = [...priorOpaqueTombstones].filter((taskId) => + bufferedTaskNotificationIds.has(taskId), + ); + yield* clearPendingBackgroundTasksForNativeThread(nativeThreadId); + yield* clearNativeThreadTaskIdSet( + wakeEligibleBackgroundTasksByNativeThread, + nativeThreadId, + ); + yield* clearNativeThreadTaskIdSet( + opaqueBackgroundTaskReplayTombstonesByNativeThread, + nativeThreadId, + ); + if (preservedOpaqueTombstones.length > 0) { + yield* addTaskIdsToNativeThreadSet( + opaqueBackgroundTaskReplayTombstonesByNativeThread, + nativeThreadId, + preservedOpaqueTombstones, + ); + } + if (!hadRoster && !hadPersistedRoster) { + return; + } + if (remembered === undefined) { + return; + } + // Prefer an explicit starting-turn status so a successful openQuery + // replacement clear cannot emit idle over an already-active thread. + // Otherwise: between turns never resurrect active from process reset; + // with a live activeTurn context, upgrade idle → active. + const activeContext = yield* Ref.get(activeTurn); + const status = + options?.status ?? + (activeContext === null + ? ("idle" as const) + : remembered.status === "idle" + ? ("active" as const) + : remembered.status); + yield* emitProviderThreadRoster({ + nativeThreadId, + providerThread: remembered, + status, + }); + }); + const resolveItemOrdinal = Effect.fnUntraced(function* ( context: ActiveClaudeTurnContext, nativeItemId: string, @@ -2878,26 +3304,55 @@ export function makeClaudeAdapterV2( completedAt: input.completedAt, }), }), - ...(input.status === "completed" && - input.context.input.providerThread.nativeConversationHeadRef !== null - ? [ - emitProviderEvent({ - type: "provider_thread.updated" as const, - driver: CLAUDE_PROVIDER, - providerThread: { - ...input.context.input.providerThread, - providerSessionId: session.id, - nativeConversationHeadRef: null, - status: "active" as const, - firstRunOrdinal: - input.context.input.providerThread.firstRunOrdinal ?? - input.context.input.runOrdinal, - lastRunOrdinal: input.context.input.runOrdinal, - updatedAt: input.completedAt, - }, - }), - ] - : []), + // Surface this native thread's roster before the root turn + // terminals so writeFinalRunEvents preserves it. Failed or + // interrupted turns drop only this thread's roster so sibling + // native threads keep their Waiting state. + Effect.gen(function* () { + const nativeThreadId = + input.context.input.providerThread.nativeThreadRef?.nativeId ?? null; + if (nativeThreadId !== null) { + if (input.status !== "completed") { + yield* clearPendingBackgroundTasksForNativeThread(nativeThreadId); + yield* clearNativeThreadTaskIdSet( + wakeEligibleBackgroundTasksByNativeThread, + nativeThreadId, + ); + yield* clearNativeThreadTaskIdSet( + opaqueBackgroundTaskReplayTombstonesByNativeThread, + nativeThreadId, + ); + } + } + const roster = + nativeThreadId === null + ? new Map() + : rosterForNativeThread( + yield* Ref.get(pendingBackgroundTasksByNativeThread), + nativeThreadId, + ); + const clearConversationHead = + input.status === "completed" && + input.context.input.providerThread.nativeConversationHeadRef !== null; + const providerThread: OrchestrationV2ProviderThread = { + ...input.context.input.providerThread, + providerSessionId: session.id, + ...(clearConversationHead ? { nativeConversationHeadRef: null } : {}), + firstRunOrdinal: + input.context.input.providerThread.firstRunOrdinal ?? + input.context.input.runOrdinal, + lastRunOrdinal: input.context.input.runOrdinal, + pendingBackgroundTasks: claudePendingBackgroundTasksFromRoster(roster), + status: input.status === "completed" ? "active" : "idle", + updatedAt: input.completedAt, + }; + yield* rememberProviderThread(providerThread); + yield* emitProviderEvent({ + type: "provider_thread.updated" as const, + driver: CLAUDE_PROVIDER, + providerThread, + }); + }), emitProviderEvent(terminalEvent), ], { concurrency: 1 }, @@ -2992,16 +3447,6 @@ export function makeClaudeAdapterV2( } }); - const clearPendingBackgroundTask = (taskId: string) => - Ref.modify(pendingBackgroundTaskIds, (current) => { - if (!current.has(taskId)) { - return [false, current] as const; - } - const updated = new Set(current); - updated.delete(taskId); - return [true, updated] as const; - }); - const bufferWakeMessage = Effect.fnUntraced(function* (wakeInput: { readonly nativeThreadId: string; readonly message: SDKMessage; @@ -3010,13 +3455,17 @@ export function makeClaudeAdapterV2( const isNotification = message.type === "system" && message.subtype === "task_notification"; // Only notifications for tracked tasks count as wake evidence: a - // pending local_bash background task, or a session-registered - // subagent that is still running (Agent with run_in_background - // settling after the root turn). A stray notification for an - // unknown task is dropped as before instead of triggering a - // spurious continuation. + // wake-eligible local_bash task (eligibility set, not the Waiting + // roster), or a session-registered subagent that is still running + // (Agent with run_in_background settling after the root turn). A + // stray notification for an unknown task is dropped as before + // instead of triggering a spurious continuation. const isPendingTaskNotification = - isNotification && (yield* Ref.get(pendingBackgroundTaskIds)).has(message.task_id); + isNotification && + (yield* isWakeEligibleOpaqueBackgroundTaskOnNativeThread( + wakeInput.nativeThreadId, + message.task_id, + )); const isPendingSubagentNotification = isNotification && !isPendingTaskNotification && @@ -3080,12 +3529,31 @@ export function makeClaudeAdapterV2( }); return updated; }); - // Request a continuation run once per wake, when the wake turn has - // either announced the finished task or fully settled. Earlier - // messages only buffer; the continuation turn drains them. + // First idle opaque notification: consume wake eligibility so a + // duplicate cannot re-buffer, and leave a short-lived replay + // tombstone for continuation-drain classification. + if (isPendingTaskNotification) { + yield* consumeWakeEligibilityForBufferedNotification( + wakeInput.nativeThreadId, + message.task_id, + ); + } + // A terminal task notification can clear the Waiting roster without + // Claude dequeuing it into a native model turn. Buffer it for replay, + // but do not open an opaque-task continuation until native user, + // assistant, or result output proves that Claude actually began the + // wake turn. Subagent notifications retain their existing immediate + // offer because their projected lifecycle owns the continuation. + const buffered = (yield* Ref.get(wakeBuffers)).get(wakeInput.nativeThreadId); + const hasBufferedNotification = + buffered?.messages.some( + (entry) => entry.type === "system" && entry.subtype === "task_notification", + ) ?? false; + const isNativeOpaqueWakeFrame = + hasBufferedNotification && (message.type === "assistant" || message.type === "user"); if ( - !isPendingTaskNotification && !isPendingSubagentNotification && + !isNativeOpaqueWakeFrame && message.type !== "result" ) { return; @@ -3124,6 +3592,90 @@ export function makeClaudeAdapterV2( }); }); + const applyBackgroundTaskRosterMessage = Effect.fnUntraced(function* (input: { + readonly nativeThreadId: string; + readonly message: SDKMessage; + readonly activeContext: ActiveClaudeTurnContext | null; + }) { + const message = input.message; + let rosterChanged = false; + + if (isClaudeBackgroundTasksChangedMessage(message)) { + const roster = Reflect.get(message, "tasks"); + if (!Array.isArray(roster)) { + return false; + } + const nextTasks: OrchestrationV2PendingBackgroundTask[] = []; + for (const entry of roster) { + const task = parseClaudeBackgroundTaskEntry(entry); + if (task !== null) { + nextTasks.push(task); + } + } + yield* replacePendingBackgroundTasks(input.nativeThreadId, nextTasks); + rosterChanged = true; + } else if (message.type === "system" && message.subtype === "task_started") { + // Incremental fallback when background_tasks_changed is absent. + // Subagent tasks project as subagent turn items; only non-subagent + // background work (e.g. local_bash) lives on the provider-thread roster. + if (!isClaudeNonSubagentTask(message)) { + return false; + } + const description = + typeof message.description === "string" && message.description.trim().length > 0 + ? message.description + : undefined; + const taskType = claudeTaskTypeFromSdkMessage(message) ?? undefined; + yield* upsertPendingBackgroundTask(input.nativeThreadId, { + taskId: message.task_id, + ...(description === undefined ? {} : { description }), + ...(taskType === undefined ? {} : { taskType }), + }); + rosterChanged = true; + } else if (message.type === "system" && message.subtype === "task_notification") { + const removed = yield* clearPendingBackgroundTask( + input.nativeThreadId, + message.task_id, + ); + // Waiting roster clears on the notification edge. Wake eligibility + // is consumed when the first idle notification is buffered; clear + // here too for same-turn active notifications that never entered + // the idle buffer path. Replay tombstones are not cleared here. + yield* clearTaskIdFromNativeThreadSet( + wakeEligibleBackgroundTasksByNativeThread, + input.nativeThreadId, + message.task_id, + ); + rosterChanged = removed; + } + + if (!rosterChanged) { + return false; + } + + const baseThread = + input.activeContext?.input.providerThread ?? + (yield* Ref.get(lastProviderThreadByNativeThread)).get(input.nativeThreadId); + if (baseThread === undefined) { + return true; + } + + // Between turns, never resurrect active status from a late empty + // roster update. During an active turn, preserve the thread status. + const status = + input.activeContext === null + ? ("idle" as const) + : baseThread.status === "idle" + ? ("active" as const) + : baseThread.status; + yield* emitProviderThreadRoster({ + nativeThreadId: input.nativeThreadId, + providerThread: baseThread, + status, + }); + return true; + }); + const handleSdkMessage = Effect.fnUntraced(function* (input: { readonly query: ClaudeAgentSdkQuerySession; readonly message: SDKMessage; @@ -3136,7 +3688,23 @@ export function makeClaudeAdapterV2( const message = input.message; const context = yield* Ref.get(activeTurn); if (context === null) { - yield* bufferWakeMessage({ nativeThreadId: liveQuery.nativeThreadId, message }); + // task_notification must buffer wake evidence while still tracked + // on the roster; clearing first would drop the wake pin. + if (message.type === "system" && message.subtype === "task_notification") { + yield* bufferWakeMessage({ nativeThreadId: liveQuery.nativeThreadId, message }); + yield* applyBackgroundTaskRosterMessage({ + nativeThreadId: liveQuery.nativeThreadId, + message, + activeContext: null, + }); + } else { + yield* applyBackgroundTaskRosterMessage({ + nativeThreadId: liveQuery.nativeThreadId, + message, + activeContext: null, + }); + yield* bufferWakeMessage({ nativeThreadId: liveQuery.nativeThreadId, message }); + } return; } @@ -3144,12 +3712,23 @@ export function makeClaudeAdapterV2( context.nativeMessageCursor = message.uuid; } + if (isClaudeBackgroundTasksChangedMessage(message)) { + yield* applyBackgroundTaskRosterMessage({ + nativeThreadId: liveQuery.nativeThreadId, + message, + activeContext: context, + }); + return; + } + if (message.type === "system" && message.subtype === "task_started") { if (isClaudeNonSubagentTask(message)) { context.ignoredTaskIds.add(message.task_id); - yield* Ref.update(pendingBackgroundTaskIds, (current) => - new Set(current).add(message.task_id), - ); + yield* applyBackgroundTaskRosterMessage({ + nativeThreadId: liveQuery.nativeThreadId, + message, + activeContext: context, + }); } else { yield* updateClaudeSubagentNode({ context, @@ -3165,7 +3744,8 @@ export function makeClaudeAdapterV2( if (message.type === "system" && message.subtype === "task_progress") { const progress = message.description.trim(); - const isBackgroundTask = (yield* Ref.get(pendingBackgroundTaskIds)).has( + const isBackgroundTask = yield* hasPendingBackgroundTaskOnNativeThread( + liveQuery.nativeThreadId, message.task_id, ); if ( @@ -3184,9 +3764,19 @@ export function makeClaudeAdapterV2( } if (message.type === "system" && message.subtype === "task_notification") { - // A wake-replay turn has empty ignoredTaskIds, so the session-level - // background registry is the durable ignore signal across turns. - const wasBackgroundTask = yield* clearPendingBackgroundTask(message.task_id); + // A wake-replay turn has empty ignoredTaskIds, so opaque-task + // tracking (live roster, wake eligibility, or the short-lived + // post-buffer replay tombstone) classifies local_bash before any + // subagent handling. + const wasBackgroundTask = yield* isKnownOpaqueBackgroundTaskOnNativeThread( + liveQuery.nativeThreadId, + message.task_id, + ); + yield* applyBackgroundTaskRosterMessage({ + nativeThreadId: liveQuery.nativeThreadId, + message, + activeContext: context, + }); if (!wasBackgroundTask && !context.ignoredTaskIds.has(message.task_id)) { yield* updateClaudeSubagentNode({ context, @@ -3201,6 +3791,14 @@ export function makeClaudeAdapterV2( : "failed", }); } + // Replay tombstone only needs to outlive buffering until this + // drained/live classification runs; drop it so it cannot leak. + if (wasBackgroundTask) { + yield* clearOpaqueBackgroundTaskReplayTombstone( + liveQuery.nativeThreadId, + message.task_id, + ); + } } for (const toolUse of claudeToolUseBlocksFromAssistantMessage(message)) { @@ -3455,8 +4053,20 @@ export function makeClaudeAdapterV2( return existing; } + // openQuery owns one live process. Closing it for another native + // thread kills that sibling's CLI; it can never emit a roster clear, + // so drop its process-scoped Waiting/wake state immediately. Closing + // for the same native thread leaves a non-authoritative roster until + // the replacement open succeeds or fails below. + const closedExistingNativeThreadId = existing !== null ? existing.nativeThreadId : null; if (existing !== null) { yield* existing.query.close.pipe(Effect.ignore); + if (existing.nativeThreadId !== nativeThreadId) { + yield* clearWakeStateForNativeThread(existing.nativeThreadId); + yield* resetBackgroundTaskStateForNativeThreadProcess(existing.nativeThreadId, { + status: "idle", + }); + } } const openedWithResume = (yield* Ref.get(openedNativeThreads)).has(nativeThreadId); @@ -3468,26 +4078,45 @@ export function makeClaudeAdapterV2( const hasPersistedProviderTurn = turnInput.providerTurnOrdinal > 1; const shouldResume = resumeSessionAt !== undefined || openedWithResume || hasPersistedProviderTurn; - const querySession = yield* queryRunner.open({ - threadId: turnInput.threadId, - providerSessionId: input.providerSessionId, - options: makeClaudeQueryOptions({ - modelSelection: turnInput.modelSelection, - nativeThreadId, - resume: shouldResume, - ...(resumeSessionAt === undefined ? {} : { resumeSessionAt }), - cwd: turnInput.runtimePolicy.cwd, - settings: adapterOptions.settings, - environment: adapterOptions.environment, - tools: queryPolicy.tools ?? CLAUDE_CODE_PRESET_TOOLS, - ...mcpOverrides, - permissionMode: queryPolicy.permissionMode, - ...(queryPolicy.allowDangerouslySkipPermissions === undefined - ? {} - : { allowDangerouslySkipPermissions: queryPolicy.allowDangerouslySkipPermissions }), - ...(shouldInstallClaudePermissionCallback(queryPolicy) ? { canUseTool } : {}), - }), - }); + const querySession = yield* queryRunner + .open({ + threadId: turnInput.threadId, + providerSessionId: input.providerSessionId, + options: makeClaudeQueryOptions({ + modelSelection: turnInput.modelSelection, + nativeThreadId, + resume: shouldResume, + ...(resumeSessionAt === undefined ? {} : { resumeSessionAt }), + cwd: turnInput.runtimePolicy.cwd, + settings: adapterOptions.settings, + environment: adapterOptions.environment, + tools: queryPolicy.tools ?? CLAUDE_CODE_PRESET_TOOLS, + ...mcpOverrides, + permissionMode: queryPolicy.permissionMode, + ...(queryPolicy.allowDangerouslySkipPermissions === undefined + ? {} + : { + allowDangerouslySkipPermissions: queryPolicy.allowDangerouslySkipPermissions, + }), + ...(shouldInstallClaudePermissionCallback(queryPolicy) ? { canUseTool } : {}), + }), + }) + .pipe( + Effect.tapError(() => + // Same-native-thread replacement: the old process is already + // dead, so its process-scoped roster is not authoritative. + // First-ever failed open (no prior live query) must not invent + // native-session reset events. + closedExistingNativeThreadId === nativeThreadId + ? Effect.gen(function* () { + yield* clearWakeStateForNativeThread(nativeThreadId); + yield* resetBackgroundTaskStateForNativeThreadProcess(nativeThreadId, { + status: "idle", + }); + }) + : Effect.void, + ), + ); // Marked only after a successful open: a failed create must not // leave the runtime believing the native session exists, or the // retry would resume a session that was never created. @@ -3499,6 +4128,16 @@ export function makeClaudeAdapterV2( updated.add(nativeThreadId); return updated; }); + // Level is per CLI process: reset Waiting roster and wake + // eligibility whenever this native thread's process starts or is + // replaced. Membership repopulates on the next snapshot/edge. + // openQuery only runs from startTurn after ProviderTurnStartService + // marked the provider thread active, and before activeTurn is set. + // Buffered local_bash task_notification classification is preserved + // across this reset (see resetBackgroundTaskStateForNativeThreadProcess). + yield* resetBackgroundTaskStateForNativeThreadProcess(nativeThreadId, { + status: "active", + }); const closed = yield* Deferred.make(); const context: ClaudeLiveQueryContext = { nativeThreadId, @@ -3554,6 +4193,7 @@ export function makeClaudeAdapterV2( }); return updated; }); + yield* rememberProviderThread(turnInput.providerThread); const context: ActiveClaudeTurnContext = { input: turnInput, nativeTurnId, @@ -3632,6 +4272,16 @@ export function makeClaudeAdapterV2( // replaying it before the rest would drop them back into the wake // buffer and request another continuation. const resultMessages = drained.filter((entry) => entry.type === "result"); + const opaqueReplayTombstones = taskIdSetForNativeThread( + yield* Ref.get(opaqueBackgroundTaskReplayTombstonesByNativeThread), + nativeThreadId, + ); + const hasOpaqueTaskNotification = drained.some( + (entry) => + entry.type === "system" && + entry.subtype === "task_notification" && + opaqueReplayTombstones.has(entry.task_id), + ); for (const entry of drained) { if (entry.type !== "result") { yield* handleSdkMessage({ query: querySession.query, message: entry }); @@ -3640,6 +4290,14 @@ export function makeClaudeAdapterV2( const lastResult = resultMessages.at(-1); if (lastResult !== undefined) { yield* handleSdkMessage({ query: querySession.query, message: lastResult }); + return; + } + const hasNativeWakeFrame = drained.some( + (entry) => entry.type === "user" || entry.type === "assistant", + ); + if (hasOpaqueTaskNotification && !hasNativeWakeFrame) { + const completedAt = yield* DateTime.now; + yield* finalizeActiveTurn({ context, status: "completed", completedAt }); } }, (effect, turnInput) => @@ -3812,8 +4470,11 @@ export function makeClaudeAdapterV2( providerSession: session, events: Stream.fromEffectRepeat(Queue.take(events)), hasPendingBackgroundWork: Effect.gen(function* () { - if ((yield* Ref.get(pendingBackgroundTaskIds)).size > 0) { - return true; + // Session capability: any native thread with pending work pins idle. + for (const roster of (yield* Ref.get(pendingBackgroundTasksByNativeThread)).values()) { + if (roster.size > 0) { + return true; + } } for (const subagent of (yield* Ref.get(sessionSubagentsByTaskId)).values()) { if (subagent.task.status === "running") { @@ -3822,12 +4483,34 @@ export function makeClaudeAdapterV2( } const buffers = yield* Ref.get(wakeBuffers); for (const entry of buffers.values()) { - if (entry.messages.length > 0) { + if ( + entry.messages.some( + (message) => + message.type === "user" || + message.type === "assistant" || + message.type === "result", + ) + ) { return true; } } return false; }), + hasPendingBackgroundWorkForThread: (providerThread) => + Effect.gen(function* () { + const nativeThreadId = providerThread.nativeThreadRef?.nativeId; + if (nativeThreadId === undefined || nativeThreadId === null) { + return false; + } + // Root-run stop gate: only this native thread's roster. Session + // subagents and wake buffers stay on the session-wide probe. + return ( + rosterForNativeThread( + yield* Ref.get(pendingBackgroundTasksByNativeThread), + nativeThreadId, + ).size > 0 + ); + }), ensureThread: Effect.fn("ClaudeAdapterV2.ensureThread")( function* (threadInput: ProviderAdapterV2EnsureThreadInput) { const createdAt = yield* DateTime.now; diff --git a/apps/server/src/orchestration-v2/Adapters/GrokAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/GrokAdapterV2.test.ts index 27f8a603801..e6d62050052 100644 --- a/apps/server/src/orchestration-v2/Adapters/GrokAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/GrokAdapterV2.test.ts @@ -13,6 +13,7 @@ import { acpRootTurnIsIdle, acpRootTurnSettleDebounceMs, acpRootTurnShouldRearmRecoveryTimers, + acpSubagentStatusBlocksTurnSettlement, acpSupportsImagePrompts, } from "./AcpAdapterV2.ts"; import { @@ -159,6 +160,20 @@ describe("acpRootTurn recovery timer re-arm", () => { }); }); +describe("acpSubagentStatusBlocksTurnSettlement", () => { + it("blocks settlement for pending and running subagents", () => { + assert.isTrue(acpSubagentStatusBlocksTurnSettlement("pending")); + assert.isTrue(acpSubagentStatusBlocksTurnSettlement("running")); + }); + + it("does not block settlement for terminal subagents", () => { + assert.isFalse(acpSubagentStatusBlocksTurnSettlement("cancelled")); + assert.isFalse(acpSubagentStatusBlocksTurnSettlement("completed")); + assert.isFalse(acpSubagentStatusBlocksTurnSettlement("failed")); + assert.isFalse(acpSubagentStatusBlocksTurnSettlement("interrupted")); + }); +}); + describe("acpRootTurnIsIdle", () => { const quiet = { finalized: false, @@ -168,7 +183,7 @@ describe("acpRootTurnIsIdle", () => { hasRunningTool: false, hasPendingRuntimeRequest: false, hasToolHistory: false, - hasRunningSubagent: false, + hasActiveSubagent: false, hasOutput: true, } as const; @@ -185,7 +200,7 @@ describe("acpRootTurnIsIdle", () => { }); it("is false while a native subagent task is still running", () => { - assert.isFalse(acpRootTurnIsIdle({ ...quiet, hasRunningSubagent: true })); + assert.isFalse(acpRootTurnIsIdle({ ...quiet, hasActiveSubagent: true })); }); it("is false when only reasoning or tools have streamed", () => { diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index 5e226553e73..9ef784009bf 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -3,6 +3,7 @@ import { type OrchestrationV2Run, OrchestrationV2DomainEvent, OrchestrationV2StoredEvent, + ProviderThreadId, RunAttemptId, RunId, ThreadId, @@ -99,6 +100,26 @@ export interface EventSinkV2Shape { }, EventSinkV2Error >; + /** + * Atomically commit only when the provider thread is still owned by the + * expected run attempt and ordinal. Used for late post-terminal + * provider_thread updates so a completed or superseded attempt cannot clobber + * a newer attempt that already claimed the thread. + */ + readonly writeIfProviderThreadOwner: (input: { + readonly commandId?: CommandId; + readonly providerThreadId: ProviderThreadId; + readonly runId: RunId; + readonly activeAttemptId: RunAttemptId; + readonly expectedLastRunOrdinal: number; + readonly events: ReadonlyArray; + }) => Effect.Effect< + { + readonly committed: boolean; + readonly storedEvents: ReadonlyArray; + }, + EventSinkV2Error + >; readonly commitCommand: (input: { readonly commandId: CommandId; readonly threadId: ThreadId; @@ -299,6 +320,62 @@ const baseLayer: Layer.Layer< }, ); + const writeIfProviderThreadOwnerEffect = Effect.fn( + "orchestrationV2.EventSink.writeIfProviderThreadOwner", + )(function* (input: Parameters[0]) { + yield* Effect.annotateCurrentSpan({ + "orchestration_v2.command_id": input.commandId ?? null, + "orchestration_v2.event_count": input.events.length, + "orchestration_v2.provider_thread_id": input.providerThreadId, + "orchestration_v2.run_id": input.runId, + "orchestration_v2.active_attempt_id": input.activeAttemptId, + "orchestration_v2.expected_last_run_ordinal": input.expectedLastRunOrdinal, + }); + + const result = yield* sql.withTransaction( + Effect.gen(function* () { + const rows = yield* sql<{ + readonly active_attempt_id: string | null; + readonly last_run_ordinal: number | null; + }>` + SELECT + json_extract(r.payload_json, '$.activeAttemptId') AS active_attempt_id, + p.last_run_ordinal + FROM orchestration_v2_projection_provider_threads p + JOIN orchestration_v2_projection_runs r + ON r.run_id = ${input.runId} + AND r.thread_id = p.thread_id + WHERE p.provider_thread_id = ${input.providerThreadId} + LIMIT 1 + `; + const current = rows[0]; + if ( + current === undefined || + current.active_attempt_id !== input.activeAttemptId || + current.last_run_ordinal !== input.expectedLastRunOrdinal + ) { + return { + committed: false as const, + storedEvents: [] as ReadonlyArray, + }; + } + + const normalized = yield* normalizeEvents(input.events); + const storedEvents = yield* eventStore.append({ + ...(input.commandId === undefined ? {} : { commandId: input.commandId }), + events: normalized, + }); + yield* applyStoredEvents(storedEvents); + return { committed: true as const, storedEvents }; + }), + ); + if (result.committed) { + yield* eventStore.publishCommitted(result.storedEvents); + yield* PubSub.publishAll(liveEvents, result.storedEvents); + } + return result; + }); + const existingCommandResult = (commandId: CommandId) => Effect.gen(function* () { const existing = yield* commandReceipts.getByCommandId(commandId); @@ -497,6 +574,17 @@ const baseLayer: Layer.Layer< }), ), ), + writeIfProviderThreadOwner: (input) => + writeIfProviderThreadOwnerEffect(input).pipe( + Effect.mapError( + (cause) => + new EventSinkWriteError({ + eventCount: input.events.length, + ...(input.commandId === undefined ? {} : { commandId: input.commandId }), + cause, + }), + ), + ), commitCommand: (input) => commitCommandEffect(input).pipe( Effect.mapError( diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index d218701a4a3..0057f8eafb4 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -685,6 +685,229 @@ it.layer(TestLayer)("orchestration V2 foundation persistence", (it) => { }), ); + it.effect("guards post-terminal provider-thread writes by attempt and run ordinal", () => + Effect.gen(function* () { + const eventSink = yield* EventSinkV2; + const projectionStore = yield* ProjectionStoreV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread:foundation-provider-thread-owner"); + const runId = RunId.make("run:foundation-provider-thread-owner"); + const attemptId = RunAttemptId.make("attempt:foundation-provider-thread-owner"); + const replacementAttemptId = RunAttemptId.make( + "attempt:foundation-provider-thread-owner:replacement", + ); + const rootNodeId = NodeId.make("node:foundation-provider-thread-owner"); + const providerThreadId = ProviderThreadId.make( + "provider-thread:foundation-provider-thread-owner", + ); + const thread = makeThread(threadId, now); + const run: OrchestrationV2Run = { + id: runId, + threadId, + ordinal: 1, + providerInstanceId, + modelSelection, + providerThreadId, + userMessageId: MessageId.make("message:foundation-provider-thread-owner"), + rootNodeId, + activeAttemptId: attemptId, + status: "completed", + queuePosition: null, + requestedAt: now, + startedAt: now, + completedAt: now, + checkpointId: null, + contextHandoffId: null, + }; + const baseProviderThread = { + id: providerThreadId, + driver: providerDriver, + providerInstanceId, + providerSessionId: null, + appThreadId: threadId, + ownerNodeId: null, + nativeThreadRef: null, + nativeConversationHeadRef: null, + status: "idle" as const, + firstRunOrdinal: 1, + lastRunOrdinal: 1, + handoffIds: [] as const, + forkedFrom: null, + pendingBackgroundTasks: [ + { taskId: "bg-owner", description: "sleep 20", taskType: "local_bash" }, + ], + createdAt: now, + updatedAt: now, + }; + + yield* eventSink.write({ + events: [ + threadCreatedEvent({ + id: "event:foundation-provider-thread-owner:thread", + thread, + now, + }), + { + id: EventId.make("event:foundation-provider-thread-owner:run"), + type: "run.created", + threadId, + runId, + nodeId: rootNodeId, + providerInstanceId, + occurredAt: now, + payload: run, + }, + { + id: EventId.make("event:foundation-provider-thread-owner:provider-thread"), + type: "provider-thread.updated", + threadId, + driver: providerDriver, + providerInstanceId, + occurredAt: now, + payload: baseProviderThread, + }, + ], + }); + + const ownedClear = yield* eventSink.writeIfProviderThreadOwner({ + providerThreadId, + runId, + activeAttemptId: attemptId, + expectedLastRunOrdinal: 1, + events: [ + { + id: EventId.make("event:foundation-provider-thread-owner:clear"), + type: "provider-thread.updated", + threadId, + driver: providerDriver, + providerInstanceId, + occurredAt: now, + payload: { + ...baseProviderThread, + pendingBackgroundTasks: [], + updatedAt: now, + }, + }, + ], + }); + assert.isTrue(ownedClear.committed); + assert.equal(ownedClear.storedEvents.length, 1); + + const afterReplacement = yield* DateTime.now; + yield* eventSink.write({ + events: [ + { + id: EventId.make("event:foundation-provider-thread-owner:replacement-attempt"), + type: "run.updated", + threadId, + runId, + nodeId: rootNodeId, + providerInstanceId, + occurredAt: afterReplacement, + payload: { + ...run, + activeAttemptId: replacementAttemptId, + status: "running", + }, + }, + ], + }); + + const supersededAttemptWrite = yield* eventSink.writeIfProviderThreadOwner({ + providerThreadId, + runId, + activeAttemptId: attemptId, + expectedLastRunOrdinal: 1, + events: [ + { + id: EventId.make("event:foundation-provider-thread-owner:superseded-attempt"), + type: "provider-thread.updated", + threadId, + driver: providerDriver, + providerInstanceId, + occurredAt: afterReplacement, + payload: { + ...baseProviderThread, + status: "active", + pendingBackgroundTasks: [ + { + taskId: "bg-superseded", + description: "should not land", + taskType: "local_bash", + }, + ], + updatedAt: afterReplacement, + }, + }, + ], + }); + assert.isFalse(supersededAttemptWrite.committed); + assert.deepEqual(supersededAttemptWrite.storedEvents, []); + + yield* eventSink.write({ + events: [ + { + id: EventId.make("event:foundation-provider-thread-owner:replacement-completed"), + type: "run.updated", + threadId, + runId, + nodeId: rootNodeId, + providerInstanceId, + occurredAt: afterReplacement, + payload: { + ...run, + activeAttemptId: replacementAttemptId, + }, + }, + { + id: EventId.make("event:foundation-provider-thread-owner:newer-run"), + type: "provider-thread.updated", + threadId, + driver: providerDriver, + providerInstanceId, + occurredAt: afterReplacement, + payload: { + ...baseProviderThread, + lastRunOrdinal: 2, + status: "active", + pendingBackgroundTasks: [], + updatedAt: afterReplacement, + }, + }, + ], + }); + + const staleOrdinalWrite = yield* eventSink.writeIfProviderThreadOwner({ + providerThreadId, + runId, + activeAttemptId: replacementAttemptId, + expectedLastRunOrdinal: 1, + events: [ + { + id: EventId.make("event:foundation-provider-thread-owner:stale-ordinal"), + type: "provider-thread.updated", + threadId, + driver: providerDriver, + providerInstanceId, + occurredAt: afterReplacement, + payload: baseProviderThread, + }, + ], + }); + assert.isFalse(staleOrdinalWrite.committed); + assert.deepEqual(staleOrdinalWrite.storedEvents, []); + + const projection = yield* projectionStore.getThreadProjection(threadId); + const providerThread = projection.providerThreads.find( + (candidate) => candidate.id === providerThreadId, + ); + assert.isDefined(providerThread); + assert.equal(providerThread?.lastRunOrdinal, 2); + assert.equal(providerThread?.status, "active"); + assert.deepEqual(providerThread?.pendingBackgroundTasks ?? [], []); + }), + ); + it.effect("interrupts a running process-bound effect when it is cancelled", () => Effect.gen(function* () { const outbox = yield* EffectOutboxV2; diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index 4ca21930612..45d5f3a7c07 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -283,6 +283,9 @@ it.layer(TestLayer)("ProjectionStoreV2", (it) => { const assistantMessageId = MessageId.make("message:projection-rollback-prune:assistant"); const userTurnItemId = TurnItemId.make("turn-item:projection-rollback-prune:user"); const assistantTurnItemId = TurnItemId.make("turn-item:projection-rollback-prune:assistant"); + const backgroundTurnItemId = TurnItemId.make( + "turn-item:projection-rollback-prune:background", + ); yield* projectionStore.apply({ id: EventId.make("event:projection-rollback-prune:thread-created"), @@ -567,6 +570,33 @@ it.layer(TestLayer)("ProjectionStoreV2", (it) => { streaming: false, }, }); + yield* projectionStore.apply({ + id: EventId.make("event:projection-rollback-prune:background-item"), + type: "turn-item.updated", + threadId, + runId, + nodeId: rootNodeId, + driver, + occurredAt: now, + payload: { + id: backgroundTurnItemId, + threadId, + runId, + nodeId: rootNodeId, + providerThreadId, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 300, + status: "running", + title: "rolled back background command", + startedAt: now, + completedAt: null, + updatedAt: now, + type: "command_execution", + input: "sleep 60", + }, + }); yield* projectionStore.apply({ id: EventId.make("event:projection-rollback-prune:run-rolled-back"), type: "run.updated", @@ -635,8 +665,16 @@ it.layer(TestLayer)("ProjectionStoreV2", (it) => { ); assert.lengthOf(projection.providerTurns, 1); assert.lengthOf(projection.messages, 2); - assert.lengthOf(projection.turnItems, 2); + assert.lengthOf(projection.turnItems, 3); assert.lengthOf(projection.visibleTurnItems, 0); + + // A rolled-back run's background item is abandoned, not pending. The + // shell must not report it as Waiting, or the sidebar shows Waiting for + // work nothing will ever finish. + const shell = yield* projectionStore.getShellSnapshot(); + const rolledBackShellThread = shell.threads.find((entry) => entry.id === threadId); + assert.isDefined(rolledBackShellThread); + assert.deepEqual(rolledBackShellThread?.pendingBackgroundTasks ?? [], []); }), ); diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index c6a446039e1..9dba4e3a0da 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -34,6 +34,7 @@ import { isOrchestrationV2SupersededInterrupt, isOrchestrationV2TurnItemVisible, } from "@t3tools/shared/orchestrationV2Timeline"; +import { derivePendingBackgroundWork } from "@t3tools/shared/orchestrationV2PendingBackgroundWork"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -753,6 +754,13 @@ export function threadShellFromProjection( (left, right) => DateTime.toEpochMillis(right.updatedAt) - DateTime.toEpochMillis(left.updatedAt), )[0] ?? null; + const pendingBackgroundTasks = derivePendingBackgroundWork({ + latestRun, + providerThreads: projection.providerThreads, + turnItems: projection.turnItems, + activeProviderThreadId: projection.thread.activeProviderThreadId, + runs: projection.runs, + }); return { createdBy: projection.thread.createdBy, creationSource: projection.thread.creationSource, @@ -792,6 +800,7 @@ export function threadShellFromProjection( hasActionableProposedPlan: projection.plans.some( (plan) => plan.kind === "proposed_plan" && plan.status === "active", ), + pendingBackgroundTasks: [...pendingBackgroundTasks], itemCount: activeLocalTurnItems(projection).length, visibleItemCount: projection.visibleTurnItems.length, createdAt: projection.thread.createdAt, @@ -823,6 +832,7 @@ type ShellThreadState = { readonly latestVisibleMessage: OrchestrationV2ConversationMessage | null; readonly latestUserMessageAt: DateTime.Utc | null; readonly hasActionableProposedPlan: boolean; + readonly pendingBackgroundTasks: OrchestrationV2ThreadShell["pendingBackgroundTasks"]; readonly itemCount: number; readonly updatedAt: OrchestrationV2ThreadProjection["updatedAt"]; readonly runOrdinalById: ReadonlyMap; @@ -952,6 +962,7 @@ function shellFromState(input: { }, latestUserMessageAt: input.state.latestUserMessageAt, hasActionableProposedPlan: input.state.hasActionableProposedPlan, + pendingBackgroundTasks: input.state.pendingBackgroundTasks, itemCount: input.state.itemCount, visibleItemCount: input.visibleItemCount, createdAt: input.state.thread.createdAt, @@ -2052,7 +2063,14 @@ export const layer: Layer.Layer = sql .withTransaction( Effect.gen(function* () { - const [threadRows, runRows, itemCountRows, sequenceRows] = yield* Effect.all([ + const [ + threadRows, + runRows, + itemCountRows, + sequenceRows, + providerThreadRows, + pendingTurnItemRows, + ] = yield* Effect.all([ sql` SELECT t.thread_id, @@ -2136,6 +2154,23 @@ export const layer: Layer.Layer = FROM orchestration_events WHERE application_event_version = 2 AND aggregate_kind = 'thread' + `, + sql` + SELECT thread_id, payload_json + FROM orchestration_v2_projection_provider_threads + WHERE thread_id IS NOT NULL + `, + sql` + SELECT i.thread_id, i.payload_json + FROM orchestration_v2_projection_turn_items i + LEFT JOIN orchestration_v2_projection_runs r + ON r.run_id = i.run_id + WHERE i.type IN ('command_execution', 'dynamic_tool', 'subagent') + AND i.status NOT IN ('completed', 'interrupted', 'failed', 'cancelled') + -- A rolled-back run's items are abandoned, not pending. Without + -- this the shell reports Waiting for work no one will finish, + -- matching the item_count query's exclusion above. + AND (i.run_id IS NULL OR r.status <> 'rolled_back') `, ]); @@ -2157,6 +2192,33 @@ export const layer: Layer.Layer = itemCountsByThreadId.set(threadId, existing); } + const providerThreadsByThreadId = new Map< + ThreadId, + Array + >(); + for (const row of providerThreadRows) { + const providerThread = yield* decodeProviderThreadPayload(row.payload_json); + const threadId = + row.thread_id.length > 0 + ? ThreadId.make(row.thread_id) + : providerThread.appThreadId; + if (threadId === null) { + continue; + } + const existing = providerThreadsByThreadId.get(threadId) ?? []; + existing.push(providerThread); + providerThreadsByThreadId.set(threadId, existing); + } + + const pendingTurnItemsByThreadId = new Map>(); + for (const row of pendingTurnItemRows) { + const turnItem = yield* decodeTurnItemPayload(row.payload_json); + const threadId = ThreadId.make(row.thread_id); + const existing = pendingTurnItemsByThreadId.get(threadId) ?? []; + existing.push(turnItem); + pendingTurnItemsByThreadId.set(threadId, existing); + } + const states = yield* Effect.forEach(threadRows, (row) => Effect.gen(function* () { const thread = yield* decodeThreadPayload(row.payload_json); @@ -2168,10 +2230,28 @@ export const layer: Layer.Layer = row.latest_message_payload_json === null ? null : yield* decodeMessagePayload(row.latest_message_payload_json); + const latestRunId = + row.latest_run_id === null ? null : RunId.make(row.latest_run_id); + const latestRunStatus = shellStatusFromStoredRunStatus(row.latest_run_status); + const pendingBackgroundTasks = [ + ...derivePendingBackgroundWork({ + latestRun: + latestRunId === null || latestRunStatus === "idle" + ? null + : { + id: latestRunId, + ordinal: 0, + status: latestRunStatus, + }, + providerThreads: providerThreadsByThreadId.get(thread.id) ?? [], + turnItems: pendingTurnItemsByThreadId.get(thread.id) ?? [], + activeProviderThreadId: thread.activeProviderThreadId, + }), + ]; return { thread, - latestRunId: row.latest_run_id === null ? null : RunId.make(row.latest_run_id), - latestRunStatus: shellStatusFromStoredRunStatus(row.latest_run_status), + latestRunId, + latestRunStatus, activeRunId: row.active_run_id === null ? null : RunId.make(row.active_run_id), pendingRuntimeRequest, latestVisibleMessage, @@ -2180,6 +2260,7 @@ export const layer: Layer.Layer = ? null : DateTime.makeUnsafe(row.latest_user_message_at), hasActionableProposedPlan: row.has_actionable_proposed_plan === 1, + pendingBackgroundTasks, itemCount: row.item_count, updatedAt: thread.updatedAt, runOrdinalById: diff --git a/apps/server/src/orchestration-v2/ProviderAdapter.ts b/apps/server/src/orchestration-v2/ProviderAdapter.ts index d8f3595bcf3..dadfacc7ec6 100644 --- a/apps/server/src/orchestration-v2/ProviderAdapter.ts +++ b/apps/server/src/orchestration-v2/ProviderAdapter.ts @@ -482,6 +482,15 @@ export interface ProviderAdapterV2SessionRuntime { * here so the session manager defers idle release while it is pending. */ readonly hasPendingBackgroundWork?: Effect.Effect; + /** + * Per-provider-thread pending work for root-run ingestion stop gates. When + * present, RunExecutionService uses only this probe (never the session-wide + * hasPendingBackgroundWork) so sibling native threads cannot pin an + * unrelated root subscription open. + */ + readonly hasPendingBackgroundWorkForThread?: ( + providerThread: OrchestrationV2ProviderThread, + ) => Effect.Effect; readonly ensureThread: ( input: ProviderAdapterV2EnsureThreadInput, ) => Effect.Effect; diff --git a/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts b/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts index f3bbe364385..af0b0375530 100644 --- a/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderContinuationService.test.ts @@ -100,6 +100,25 @@ describe("ProviderContinuationService", () => { }); }); + it.effect("queues a continuation behind an active user run", () => { + return Effect.gen(function* () { + const dispatched = yield* Queue.unbounded(); + yield* Effect.gen(function* () { + const requests = yield* ProviderContinuationRequests; + yield* requests.offer(request()); + const command = yield* Queue.take(dispatched); + assert.deepEqual((command as { readonly dispatchMode?: unknown }).dispatchMode, { + type: "queue_after_active", + }); + }).pipe( + Effect.provide( + testLayer({ dispatched, getThreadProjection: () => Effect.succeed(projection) }), + ), + Effect.scoped, + ); + }); + }); + it.effect("drops a request invalidated before dispatch", () => { return Effect.gen(function* () { const dispatched = yield* Queue.unbounded(); diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts index ea2d2123e31..48171de6b0a 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts @@ -6,8 +6,13 @@ import { type OrchestrationV2AppThread, type OrchestrationV2DomainEvent, type OrchestrationV2ProviderThread, + type OrchestrationV2Run, + type OrchestrationV2TurnItem, ProviderDriverKind, ProviderInstanceId, + RunAttemptId, + RunId, + TurnItemId, } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -28,6 +33,12 @@ import { layer as providerEventIngestorLayer, } from "./ProviderEventIngestor.ts"; import { makeProviderFailure } from "./ProviderFailure.ts"; +import { + makeProviderEventRoutingState, + type ProviderEventRouteIdentity, + routeProviderEvent, + selectInheritedBackgroundTurnItems, +} from "./RunExecutionService.ts"; const TestDatabaseLayer = SqlitePersistenceMemory; const TestStoresLayer = Layer.merge(eventStoreLayer, projectionStoreLayer).pipe( @@ -232,6 +243,263 @@ layer("ProviderEventIngestorV2", (it) => { }), ); + it.effect("persists an interrupted run's inherited terminal through the live run router", () => + Effect.gen(function* () { + const now = yield* DateTime.now; + const eventSink = yield* EventSinkV2; + const projectionStore = yield* ProjectionStoreV2; + const ingestor = yield* ProviderEventIngestorV2; + const idAllocator = yield* IdAllocatorV2; + const threadEvent = yield* threadCreatedEvent(now); + const priorRunId = RunId.make("run:provider-event-inherited:prior"); + const currentRunId = RunId.make("run:provider-event-inherited:current"); + const itemId = TurnItemId.make("turn-item:provider-event-inherited"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId: threadEvent.threadId, + }); + const providerThreadId = idAllocator.derive.providerThread({ + driver: CODEX_DRIVER, + nativeThreadId: "native-thread-inherited", + }); + const providerTurnId = idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: "native-turn-inherited", + }); + const runningItem = { + id: itemId, + threadId: threadEvent.threadId, + runId: priorRunId, + nodeId: NodeId.make("node:provider-event-inherited"), + providerThreadId, + providerTurnId, + nativeItemRef: null, + parentItemId: null, + ordinal: 101, + status: "running", + title: "Inherited background command", + startedAt: now, + completedAt: null, + updatedAt: now, + type: "command_execution", + input: "sleep 60", + } satisfies OrchestrationV2TurnItem; + const terminalItem = { + ...runningItem, + status: "completed" as const, + completedAt: now, + updatedAt: now, + }; + + yield* eventSink.write({ events: [threadEvent] }); + yield* ingestor.ingestNormalized({ + providerSessionId, + providerInstanceId: modelSelection.instanceId, + threadId: threadEvent.threadId, + runId: priorRunId, + event: { type: "turn_item.updated", driver: CODEX_DRIVER, turnItem: runningItem }, + }); + + const identity: ProviderEventRouteIdentity = { + threadId: threadEvent.threadId, + runId: currentRunId, + attemptId: RunAttemptId.make("attempt:provider-event-inherited:current"), + providerThreadId, + }; + const inheritedBackgroundTurnItems = selectInheritedBackgroundTurnItems({ + threadId: threadEvent.threadId, + currentProviderThreadId: providerThreadId, + currentRunOrdinal: 2, + runs: [ + { + id: priorRunId, + threadId: threadEvent.threadId, + ordinal: 1, + status: "interrupted", + } as OrchestrationV2Run, + { + id: currentRunId, + threadId: threadEvent.threadId, + ordinal: 2, + status: "running", + } as OrchestrationV2Run, + ], + turnItems: [runningItem], + }); + const routeState = makeProviderEventRoutingState({ + identity, + inheritedBackgroundTurnItems, + providerTurnId: null, + }); + const terminalEvent = { + type: "turn_item.updated", + driver: CODEX_DRIVER, + turnItem: terminalItem, + } as const; + const [accepted] = routeProviderEvent(terminalEvent, identity, routeState); + assert.isTrue(accepted); + + const stored = yield* ingestor.ingestNormalized({ + providerSessionId, + providerInstanceId: modelSelection.instanceId, + threadId: threadEvent.threadId, + runId: currentRunId, + event: terminalEvent, + }); + const projection = yield* projectionStore.getThreadProjection(threadEvent.threadId); + const persisted = projection.turnItems.find((item) => item.id === itemId); + + assert.equal(stored.length, 1); + assert.equal(stored[0]?.event.type, "turn-item.updated"); + assert.equal(persisted?.runId, priorRunId); + assert.equal(persisted?.threadId, threadEvent.threadId); + assert.equal(persisted?.status, "completed"); + }), + ); + + it.effect("persists a completed run's late background terminal exactly once", () => + Effect.gen(function* () { + const now = yield* DateTime.now; + const eventSink = yield* EventSinkV2; + const eventStore = yield* EventStoreV2; + const ingestor = yield* ProviderEventIngestorV2; + const idAllocator = yield* IdAllocatorV2; + const threadEvent = yield* threadCreatedEvent(now); + const priorRunId = RunId.make("run:provider-event-completed:prior"); + const currentRunId = RunId.make("run:provider-event-completed:current"); + const itemId = TurnItemId.make("turn-item:provider-event-completed"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId: threadEvent.threadId, + }); + const providerThreadId = idAllocator.derive.providerThread({ + driver: CODEX_DRIVER, + nativeThreadId: "native-thread-completed", + }); + const providerTurnId = idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: "native-turn-completed", + }); + const runningItem = { + id: itemId, + threadId: threadEvent.threadId, + runId: priorRunId, + nodeId: NodeId.make("node:provider-event-completed"), + providerThreadId, + providerTurnId, + nativeItemRef: null, + parentItemId: null, + ordinal: 101, + status: "running", + title: "Completed run background command", + startedAt: now, + completedAt: null, + updatedAt: now, + type: "command_execution", + input: "sleep 60", + } satisfies OrchestrationV2TurnItem; + const terminalEvent = { + type: "turn_item.updated", + driver: CODEX_DRIVER, + turnItem: { + ...runningItem, + status: "completed" as const, + completedAt: now, + updatedAt: now, + }, + } as const; + + yield* eventSink.write({ events: [threadEvent] }); + yield* ingestor.ingestNormalized({ + providerSessionId, + providerInstanceId: modelSelection.instanceId, + threadId: threadEvent.threadId, + runId: priorRunId, + event: { type: "turn_item.updated", driver: CODEX_DRIVER, turnItem: runningItem }, + }); + + const priorIdentity: ProviderEventRouteIdentity = { + threadId: threadEvent.threadId, + runId: priorRunId, + attemptId: RunAttemptId.make("attempt:provider-event-completed:prior"), + providerThreadId, + }; + const currentIdentity: ProviderEventRouteIdentity = { + threadId: threadEvent.threadId, + runId: currentRunId, + attemptId: RunAttemptId.make("attempt:provider-event-completed:current"), + providerThreadId, + }; + const inheritedBackgroundTurnItems = selectInheritedBackgroundTurnItems({ + threadId: threadEvent.threadId, + currentProviderThreadId: providerThreadId, + currentRunOrdinal: 2, + runs: [ + { + id: priorRunId, + threadId: threadEvent.threadId, + ordinal: 1, + status: "completed", + } as OrchestrationV2Run, + { + id: currentRunId, + threadId: threadEvent.threadId, + ordinal: 2, + status: "running", + } as OrchestrationV2Run, + ], + turnItems: [runningItem], + }); + const routers = [ + { + identity: priorIdentity, + state: makeProviderEventRoutingState({ + identity: priorIdentity, + providerTurnId: providerTurnId, + }), + }, + { + identity: currentIdentity, + state: makeProviderEventRoutingState({ + identity: currentIdentity, + inheritedBackgroundTurnItems, + providerTurnId: null, + }), + }, + ]; + const acceptedRouters = routers.filter( + ({ identity, state }) => routeProviderEvent(terminalEvent, identity, state)[0], + ); + + yield* Effect.forEach( + acceptedRouters, + ({ identity }) => + ingestor.ingestNormalized({ + providerSessionId, + providerInstanceId: modelSelection.instanceId, + threadId: threadEvent.threadId, + runId: identity.runId, + event: terminalEvent, + }), + { concurrency: 1 }, + ); + + const storedEvents = yield* eventStore + .read({ threadId: threadEvent.threadId }) + .pipe(Stream.runCollect); + const storedTerminals = Array.from(storedEvents).filter( + (stored) => + stored.event.type === "turn-item.updated" && + stored.event.payload.id === itemId && + stored.event.payload.status === "completed", + ); + + assert.equal(storedTerminals.length, 1); + assert.equal(acceptedRouters.length, 1); + assert.equal(acceptedRouters[0]?.identity.runId, priorRunId); + }), + ); + it.effect("persists a failed provider terminal as one expected error item", () => Effect.gen(function* () { const now = yield* DateTime.now; diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts index 91e4a974390..a09cf26ca6f 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts @@ -6,6 +6,7 @@ import { type OrchestrationV2Run, ProviderInstanceId, ProviderSessionId, + ProviderThreadId, RawEventId, RunAttemptId, RunId, @@ -81,6 +82,16 @@ export interface ProviderEventIngestorV2Shape { readonly activeAttemptId: RunAttemptId; readonly expectedStatus: OrchestrationV2Run["status"]; }; + /** + * Atomically reject provider-thread snapshots from an attempt that no + * longer owns the run or from a run that no longer owns the thread. + */ + readonly writeIfProviderThreadOwner?: { + readonly providerThreadId: ProviderThreadId; + readonly runId: RunId; + readonly activeAttemptId: RunAttemptId; + readonly expectedLastRunOrdinal: number; + }; }, ) => Effect.Effect, ProviderEventIngestorV2Error>; } @@ -279,6 +290,16 @@ export const layer: Layer.Layer { + const threadId = ThreadId.make("thread_recovery_background"); + const settledRunId = RunId.make("run_recovery_background_settled"); + const activeRunId = RunId.make("run_recovery_background_active"); + const activeAttemptId = RunAttemptId.make("attempt_recovery_background_active"); + const activeRootNodeId = NodeId.make("node_recovery_background_active"); + const idleProviderThreadId = ProviderThreadId.make("provider_thread_recovery_background_idle"); + const activeProviderThreadId = ProviderThreadId.make( + "provider_thread_recovery_background_active", + ); + const secondaryProviderThreadId = ProviderThreadId.make( + "provider_thread_recovery_background_secondary", + ); + const providerSessionId = ProviderSessionId.make("provider_session_recovery_background"); + const settledStaleItemId = TurnItemId.make("turn_item_recovery_background_stale"); + const activeRunItemId = TurnItemId.make("turn_item_recovery_background_active"); + const nullRunCommandItemId = TurnItemId.make("turn_item_recovery_background_null_run"); + const nullRunSubagentItemId = TurnItemId.make("turn_item_recovery_background_null_subagent"); + const claudeInstanceId = ProviderInstanceId.make("claude"); + const secondaryInstanceId = ProviderInstanceId.make("claude-secondary"); + const subagentInstanceId = ProviderInstanceId.make("claude-subagent"); + let committedInput: Parameters[0] | null = + null; + const projection = { + thread: { id: threadId }, + runtimeRequests: [], + providerSessions: [ + { + id: providerSessionId, + driver: ProviderDriverKind.make("claude"), + providerInstanceId: claudeInstanceId, + status: "ready", + }, + ], + providerThreads: [ + { + id: idleProviderThreadId, + driver: ProviderDriverKind.make("claude"), + // Index-0 is intentionally a different instance so misattribution + // to providerThreads[0] fails the assertions below. + providerInstanceId: claudeInstanceId, + status: "idle", + pendingBackgroundTasks: [{ taskId: "bg-settled", description: "sleep 30" }], + }, + { + id: activeProviderThreadId, + driver: ProviderDriverKind.make("claude"), + providerInstanceId: claudeInstanceId, + status: "active", + pendingBackgroundTasks: [{ taskId: "bg-active", description: "npm test" }], + }, + { + id: secondaryProviderThreadId, + driver: ProviderDriverKind.make("claude"), + providerInstanceId: secondaryInstanceId, + status: "idle", + pendingBackgroundTasks: [], + }, + ], + providerTurns: [], + runs: [ + { + id: settledRunId, + status: "completed", + providerInstanceId: claudeInstanceId, + }, + { + id: activeRunId, + status: "running", + providerInstanceId: claudeInstanceId, + }, + ], + attempts: [ + { + id: activeAttemptId, + runId: activeRunId, + rootNodeId: activeRootNodeId, + status: "running", + }, + ], + nodes: [{ id: activeRootNodeId, runId: activeRunId, status: "running" }], + subagents: [], + messages: [], + turnItems: [ + { + id: settledStaleItemId, + runId: settledRunId, + nodeId: null, + providerThreadId: idleProviderThreadId, + type: "command_execution", + status: "running", + }, + { + id: activeRunItemId, + runId: activeRunId, + nodeId: activeRootNodeId, + providerThreadId: activeProviderThreadId, + type: "dynamic_tool", + status: "running", + }, + { + // Missing run: must attribute via providerThreadId, not index 0. + id: nullRunCommandItemId, + runId: null, + nodeId: null, + providerThreadId: secondaryProviderThreadId, + type: "command_execution", + status: "running", + }, + { + // Missing run with a real matching provider thread whose instance + // differs from the subagent's own: own providerInstanceId must win. + id: nullRunSubagentItemId, + runId: null, + nodeId: null, + providerThreadId: secondaryProviderThreadId, + type: "subagent", + status: "running", + providerInstanceId: subagentInstanceId, + }, + ], + } as unknown as OrchestrationV2ThreadProjection; + const layer = ProviderRuntimeRecovery.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getShellSnapshot: () => + Effect.succeed({ + schemaVersion: 2, + snapshotSequence: 0, + threads: [{ id: threadId }], + archivedThreads: [], + } as never), + getThreadProjection: () => Effect.succeed(projection), + }), + Layer.mock(EventSink.EventSinkV2)({ + commitCommand: (input) => { + committedInput = input; + return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); + }, + }), + IdAllocator.layer, + Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ runOnce: Effect.succeed(false) }), + Layer.mock(EffectOutbox.EffectOutboxV2)({ + listByCommandId: () => Effect.succeed([]), + reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), + }), + ), + ), + ); + + return Effect.gen(function* () { + const summary = + yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup"); + assert.equal(summary.terminalizedRuns, 1); + const events = committedInput?.events ?? []; + + const turnItemCancels = events.filter( + (event) => event.type === "turn-item.updated" && event.payload.status === "cancelled", + ); + // Active-run item + settled-run stale + null-run command + null-run subagent. + assert.equal(turnItemCancels.length, 4); + assert.deepEqual( + turnItemCancels + .map((event) => event.type === "turn-item.updated" && event.payload.id) + .sort(), + [activeRunItemId, nullRunCommandItemId, nullRunSubagentItemId, settledStaleItemId].sort(), + ); + + const cancelById = (id: TurnItemId) => + turnItemCancels.find( + (event) => event.type === "turn-item.updated" && event.payload.id === id, + ); + assert.equal(cancelById(nullRunCommandItemId)?.providerInstanceId, secondaryInstanceId); + // Subagent own instance wins over the matching thread's secondary instance. + assert.notEqual(subagentInstanceId, secondaryInstanceId); + assert.equal(cancelById(nullRunSubagentItemId)?.providerInstanceId, subagentInstanceId); + // Settled-run item still prefers the run's provider instance when present. + assert.equal(cancelById(settledStaleItemId)?.providerInstanceId, claudeInstanceId); + + const providerThreadEvents = events.filter( + (event) => event.type === "provider-thread.updated", + ); + // Only threads with active status or nonempty rosters are rewritten. + assert.equal(providerThreadEvents.length, 2); + for (const event of providerThreadEvents) { + if (event.type !== "provider-thread.updated") continue; + assert.deepEqual(event.payload.pendingBackgroundTasks ?? [], []); + } + const idleThreadEvent = providerThreadEvents.find( + (event) => + event.type === "provider-thread.updated" && event.payload.id === idleProviderThreadId, + ); + assert.equal( + idleThreadEvent?.type === "provider-thread.updated" ? idleThreadEvent.payload.status : null, + "idle", + ); + const activeThreadEvent = providerThreadEvents.find( + (event) => + event.type === "provider-thread.updated" && event.payload.id === activeProviderThreadId, + ); + assert.equal( + activeThreadEvent?.type === "provider-thread.updated" + ? activeThreadEvent.payload.status + : null, + "idle", + ); + }).pipe(Effect.provide(layer)); + }, +); + +it.effect( + "terminalizes the linked subagent and node for a stale subagent item on a settled run", + () => { + const threadId = ThreadId.make("thread_recovery_subagent"); + const settledRunId = RunId.make("run_recovery_subagent_settled"); + const providerThreadId = ProviderThreadId.make("provider_thread_recovery_subagent"); + const staleSubagentNodeId = NodeId.make("node_recovery_subagent_stale"); + const doneSubagentNodeId = NodeId.make("node_recovery_subagent_done"); + const staleItemId = TurnItemId.make("turn_item_recovery_subagent_stale"); + const doneItemId = TurnItemId.make("turn_item_recovery_subagent_done"); + const claudeInstanceId = ProviderInstanceId.make("claude"); + let committedInput: Parameters[0] | null = + null; + const projection = { + thread: { id: threadId }, + runtimeRequests: [], + providerSessions: [], + providerThreads: [ + { + id: providerThreadId, + driver: ProviderDriverKind.make("claude"), + providerInstanceId: claudeInstanceId, + status: "idle", + pendingBackgroundTasks: [], + }, + ], + providerTurns: [], + // Settled run: the stale-item loop owns it, not the nonterminal loop. + runs: [{ id: settledRunId, status: "completed", providerInstanceId: claudeInstanceId }], + attempts: [], + nodes: [ + { id: staleSubagentNodeId, runId: settledRunId, status: "running" }, + { id: doneSubagentNodeId, runId: settledRunId, status: "completed" }, + ], + subagents: [ + { + id: staleSubagentNodeId, + runId: settledRunId, + driver: ProviderDriverKind.make("claude"), + providerInstanceId: claudeInstanceId, + status: "running", + }, + { + // Already finished with a real result: must never be overwritten. + id: doneSubagentNodeId, + runId: settledRunId, + driver: ProviderDriverKind.make("claude"), + providerInstanceId: claudeInstanceId, + status: "completed", + result: "done", + }, + ], + messages: [], + turnItems: [ + { + id: staleItemId, + runId: settledRunId, + nodeId: staleSubagentNodeId, + providerThreadId, + type: "subagent", + status: "running", + subagentId: staleSubagentNodeId, + providerInstanceId: claudeInstanceId, + }, + { + id: doneItemId, + runId: settledRunId, + nodeId: doneSubagentNodeId, + providerThreadId, + type: "subagent", + status: "completed", + subagentId: doneSubagentNodeId, + providerInstanceId: claudeInstanceId, + }, + ], + } as unknown as OrchestrationV2ThreadProjection; + const layer = ProviderRuntimeRecovery.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getShellSnapshot: () => + Effect.succeed({ + schemaVersion: 2, + snapshotSequence: 0, + threads: [{ id: threadId }], + archivedThreads: [], + } as never), + getThreadProjection: () => Effect.succeed(projection), + }), + Layer.mock(EventSink.EventSinkV2)({ + commitCommand: (input) => { + committedInput = input; + return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never); + }, + }), + IdAllocator.layer, + Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ runOnce: Effect.succeed(false) }), + Layer.mock(EffectOutbox.EffectOutboxV2)({ + listByCommandId: () => Effect.succeed([]), + reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), + }), + ), + ), + ); + + return Effect.gen(function* () { + yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup"); + const events = committedInput?.events ?? []; + + // Only the nonterminal subagent item is cancelled. + const turnItemCancels = events.filter( + (event) => event.type === "turn-item.updated" && event.payload.status === "cancelled", + ); + assert.equal(turnItemCancels.length, 1); + + // The linked subagent entity is terminalized alongside its turn item. + const subagentCancels = events.filter((event) => event.type === "subagent.updated"); + assert.equal(subagentCancels.length, 1); + const subagentCancel = subagentCancels[0]; + assert.equal( + subagentCancel?.type === "subagent.updated" ? subagentCancel.payload.id : null, + staleSubagentNodeId, + ); + assert.equal( + subagentCancel?.type === "subagent.updated" ? subagentCancel.payload.status : null, + "cancelled", + ); + + // So is its execution node, which no live process can terminalize. + const nodeCancels = events.filter((event) => event.type === "node.updated"); + assert.equal(nodeCancels.length, 1); + const nodeCancel = nodeCancels[0]; + assert.equal( + nodeCancel?.type === "node.updated" ? nodeCancel.payload.id : null, + staleSubagentNodeId, + ); + + // The already-completed subagent and node are left untouched. + assert.isFalse( + events.some( + (event) => event.type === "subagent.updated" && event.payload.id === doneSubagentNodeId, + ), + ); + assert.isFalse( + events.some( + (event) => event.type === "node.updated" && event.payload.id === doneSubagentNodeId, + ), + ); + }).pipe(Effect.provide(layer)); + }, +); diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts index 5dadaa7b01e..4cebdcb30fd 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts @@ -69,6 +69,58 @@ function nonterminalRuns(projection: OrchestrationV2ThreadProjection) { }); } +function isBackgroundCapableTurnItemType(type: string): boolean { + return type === "command_execution" || type === "dynamic_tool" || type === "subagent"; +} + +function isNonterminalTurnItemStatus(status: string): boolean { + return status === "pending" || status === "running" || status === "waiting"; +} + +function isNonterminalSubagentStatus(status: string): boolean { + return status === "pending" || status === "running" || status === "waiting"; +} + +function isNonterminalNodeStatus(status: string): boolean { + return status === "pending" || status === "running" || status === "waiting"; +} + +function providerThreadHasPendingBackgroundTasks( + providerThread: OrchestrationV2ThreadProjection["providerThreads"][number], +): boolean { + return (providerThread.pendingBackgroundTasks?.length ?? 0) > 0; +} + +/** + * Resolve providerInstanceId for a stale background-capable turn item whose + * run is missing/null (or not found). Prefer an existing run, then a subagent + * item's own instance id, then the item's provider thread, then a last-resort + * first provider thread. + */ +function resolveStaleBackgroundItemProviderInstanceId( + item: OrchestrationV2ThreadProjection["turnItems"][number], + projection: OrchestrationV2ThreadProjection, +): OrchestrationV2ThreadProjection["providerThreads"][number]["providerInstanceId"] | undefined { + if (item.runId !== null) { + const run = projection.runs.find((candidate) => candidate.id === item.runId); + if (run !== undefined) { + return run.providerInstanceId; + } + } + if (item.type === "subagent") { + return item.providerInstanceId; + } + if (item.providerThreadId !== null && item.providerThreadId !== undefined) { + const providerThread = projection.providerThreads.find( + (candidate) => candidate.id === item.providerThreadId, + ); + if (providerThread !== undefined) { + return providerThread.providerInstanceId; + } + } + return projection.providerThreads[0]?.providerInstanceId; +} + export const make = Effect.gen(function* () { const projections = yield* ProjectionStore.ProjectionStoreV2; const eventSink = yield* EventSink.EventSinkV2; @@ -253,9 +305,85 @@ export const make = Effect.gen(function* () { }); } } - for (const providerThread of projection.providerThreads.filter( - (candidate) => candidate.status === "active", - )) { + // Process loss also orphans background-capable turn items on already- + // settled runs (e.g. post-settle Waiting work). Skip items already + // cancelled above for recovered nonterminal runs to avoid duplicate + // cancellation events. + const recoveredNonterminalRunIds = new Set(runs.map((run) => run.id)); + for (const item of projection.turnItems ?? []) { + if (item.runId !== null && recoveredNonterminalRunIds.has(item.runId)) { + continue; + } + if (!isBackgroundCapableTurnItemType(item.type)) { + continue; + } + if (!isNonterminalTurnItemStatus(item.status)) { + continue; + } + const providerInstanceId = resolveStaleBackgroundItemProviderInstanceId(item, projection); + if (providerInstanceId === undefined) { + continue; + } + events.push({ + id: yield* allocateEventId(), + type: "turn-item.updated", + threadId: projection.thread.id, + ...(item.runId === null ? {} : { runId: item.runId }), + ...(item.nodeId === null || item.nodeId === undefined ? {} : { nodeId: item.nodeId }), + providerInstanceId, + occurredAt: now, + payload: { ...item, status: "cancelled", completedAt: now, updatedAt: now }, + }); + if (item.type !== "subagent") { + continue; + } + // Cancelling only the turn item would leave the linked subagent entity + // and its execution node non-terminal forever, since the dead provider + // process can no longer emit their terminal events. Match the exact + // linked ids so a subagent that already finished is never overwritten. + const staleSubagent = projection.subagents.find( + (candidate) => + candidate.id === item.subagentId && isNonterminalSubagentStatus(candidate.status), + ); + if (staleSubagent !== undefined) { + events.push({ + id: yield* allocateEventId(), + type: "subagent.updated", + threadId: projection.thread.id, + ...(item.runId === null ? {} : { runId: item.runId }), + nodeId: staleSubagent.id, + driver: staleSubagent.driver, + providerInstanceId: staleSubagent.providerInstanceId, + occurredAt: now, + payload: { ...staleSubagent, status: "cancelled", completedAt: now, updatedAt: now }, + }); + } + const staleSubagentNode = projection.nodes.find( + (candidate) => + candidate.id === item.subagentId && isNonterminalNodeStatus(candidate.status), + ); + if (staleSubagentNode !== undefined) { + events.push({ + id: yield* allocateEventId(), + type: "node.updated", + threadId: projection.thread.id, + ...(item.runId === null ? {} : { runId: item.runId }), + nodeId: staleSubagentNode.id, + providerInstanceId, + occurredAt: now, + payload: { ...staleSubagentNode, status: "cancelled", completedAt: now }, + }); + } + } + // All provider processes are gone on startup/shutdown: clear any + // persisted Waiting roster (including idle threads from settled roots) + // and idle active threads without resurrecting active status. + for (const providerThread of projection.providerThreads ?? []) { + const needsIdle = providerThread.status === "active"; + const needsRosterClear = providerThreadHasPendingBackgroundTasks(providerThread); + if (!needsIdle && !needsRosterClear) { + continue; + } events.push({ id: yield* allocateEventId(), type: "provider-thread.updated", @@ -263,7 +391,12 @@ export const make = Effect.gen(function* () { driver: providerThread.driver, providerInstanceId: providerThread.providerInstanceId, occurredAt: now, - payload: { ...providerThread, status: "idle", updatedAt: now }, + payload: { + ...providerThread, + status: needsIdle ? "idle" : providerThread.status, + pendingBackgroundTasks: [], + updatedAt: now, + }, }); } for (const session of projection.providerSessions.filter( diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 3b6f74bf5f7..16b0b1c3f9a 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -22,7 +22,11 @@ import { import { IdAllocatorV2 } from "./IdAllocator.ts"; import { ProjectionStoreV2 } from "./ProjectionStore.ts"; import { ProviderSessionManagerV2 } from "./ProviderSessionManager.ts"; -import { canRouteRelatedSubagent, RunExecutionServiceV2 } from "./RunExecutionService.ts"; +import { + canRouteRelatedSubagent, + RunExecutionServiceV2, + selectInheritedBackgroundTurnItems, +} from "./RunExecutionService.ts"; import { RuntimePolicyV2 } from "./RuntimePolicy.ts"; export class ProviderTurnStartError extends Schema.TaggedErrorClass()( @@ -417,6 +421,18 @@ export const layer: Layer.Layer< providerThread: runningProviderThread, attempt: runningAttempt, attemptId: attempt.id, + loadInheritedBackgroundTurnItems: () => + projectionStore.getThreadProjection(projection.thread.id).pipe( + Effect.map((current) => + selectInheritedBackgroundTurnItems({ + threadId: current.thread.id, + currentProviderThreadId: runningProviderThread.id, + currentRunOrdinal: run.ordinal, + runs: current.runs, + turnItems: current.turnItems, + }), + ), + ), relatedThreadIds: routableSubagents.flatMap((subagent) => subagent.childThreadId === null ? [] : [subagent.childThreadId], ), diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 28f2120e599..e7cb491d22f 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -47,6 +47,7 @@ import { type ProviderEventRouteIdentity, routeProviderEvent, RunExecutionServiceV2, + selectInheritedBackgroundTurnItems, } from "./RunExecutionService.ts"; const driver = ProviderDriverKind.make("codex"); @@ -187,6 +188,197 @@ it("does not route a superseded attempt through a reused provider thread", () => assert.isFalse(routeProviderEvent(oldTurnEvent, newAttempt, newState)[0]); }); +it("routes only exact same-thread background items inherited from settled runs", () => { + const threadId = ThreadId.make("thread:inherited-background-routing"); + const otherThreadId = ThreadId.make("thread:inherited-background-routing:other"); + const priorRunId = RunId.make("run:inherited-background-routing:prior"); + const currentRunId = RunId.make("run:inherited-background-routing:current"); + const itemId = TurnItemId.make("turn-item:inherited-background-routing"); + const identity: ProviderEventRouteIdentity = { + threadId, + runId: currentRunId, + attemptId: RunAttemptId.make("attempt:inherited-background-routing:current"), + providerThreadId: ProviderThreadId.make("provider-thread:inherited-background-routing:current"), + }; + const initial = makeProviderEventRoutingState({ + identity, + inheritedBackgroundTurnItems: [{ id: itemId, runId: priorRunId }], + providerTurnId: null, + }); + const inheritedRunning = { + type: "turn_item.updated", + driver, + turnItem: { + id: itemId, + threadId, + runId: priorRunId, + providerTurnId: null, + ordinal: 1, + type: "subagent", + status: "running", + }, + } as Extract; + const unrelatedRunItem = { + ...inheritedRunning, + turnItem: { + ...inheritedRunning.turnItem, + runId: RunId.make("run:inherited-background-routing:unrelated"), + }, + } as ProviderAdapterV2Event; + const unlistedPriorItem = { + ...inheritedRunning, + turnItem: { + ...inheritedRunning.turnItem, + id: TurnItemId.make("turn-item:inherited-background-routing:unrelated"), + }, + } as ProviderAdapterV2Event; + const unrelatedThreadItem = { + ...inheritedRunning, + turnItem: { ...inheritedRunning.turnItem, threadId: otherThreadId }, + } as ProviderAdapterV2Event; + const ordinaryItem = { + ...inheritedRunning, + turnItem: { ...inheritedRunning.turnItem, type: "reasoning" as const }, + } as ProviderAdapterV2Event; + const inheritedTerminal = { + ...inheritedRunning, + turnItem: { ...inheritedRunning.turnItem, status: "completed" as const }, + } as ProviderAdapterV2Event; + + const [runningAccepted, afterRunning] = routeProviderEvent(inheritedRunning, identity, initial); + assert.isTrue(runningAccepted); + assert.isFalse(routeProviderEvent(unrelatedRunItem, identity, afterRunning)[0]); + assert.isFalse(routeProviderEvent(unlistedPriorItem, identity, afterRunning)[0]); + assert.isFalse(routeProviderEvent(unrelatedThreadItem, identity, afterRunning)[0]); + assert.isFalse(routeProviderEvent(ordinaryItem, identity, afterRunning)[0]); + + const [terminalAccepted, afterTerminal] = routeProviderEvent( + inheritedTerminal, + identity, + afterRunning, + ); + assert.isTrue(terminalAccepted); + assert.isFalse( + routeProviderEvent(inheritedRunning, identity, afterTerminal)[0], + "a nonterminal replay must not resurrect an inherited terminal", + ); +}); + +it("selects only live background items from non-completed settled prior runs", () => { + const threadId = ThreadId.make("thread:inherited-background-selection"); + const otherThreadId = ThreadId.make("thread:inherited-background-selection:other"); + const currentProviderThreadId = ProviderThreadId.make( + "provider-thread:inherited-background-selection:current", + ); + const foreignProviderThreadId = ProviderThreadId.make( + "provider-thread:inherited-background-selection:foreign-session", + ); + const interruptedRunId = RunId.make("run:inherited-background-selection:interrupted"); + const failedRunId = RunId.make("run:inherited-background-selection:failed"); + const cancelledRunId = RunId.make("run:inherited-background-selection:cancelled"); + const completedRunId = RunId.make("run:inherited-background-selection:completed"); + const rolledBackRunId = RunId.make("run:inherited-background-selection:rolled-back"); + const currentRunId = RunId.make("run:inherited-background-selection:current"); + const inheritedItemId = TurnItemId.make("turn-item:inherited-background-selection:live"); + const failedItemId = TurnItemId.make("turn-item:inherited-background-selection:failed"); + const cancelledItemId = TurnItemId.make("turn-item:inherited-background-selection:cancelled"); + const makeRun = ( + id: RunId, + ordinal: number, + status: OrchestrationV2Run["status"], + runThreadId = threadId, + ) => + ({ + id, + threadId: runThreadId, + ordinal, + status, + }) as OrchestrationV2Run; + const makeItem = ( + id: TurnItemId, + runId: RunId, + status: OrchestrationV2TurnItem["status"], + type: OrchestrationV2TurnItem["type"] = "subagent", + itemThreadId = threadId, + providerThreadId = currentProviderThreadId, + ) => + ({ + id, + threadId: itemThreadId, + runId, + providerThreadId, + type, + status, + }) as OrchestrationV2TurnItem; + + const selected = selectInheritedBackgroundTurnItems({ + threadId, + currentProviderThreadId, + currentRunOrdinal: 6, + runs: [ + makeRun(interruptedRunId, 1, "interrupted"), + makeRun(failedRunId, 2, "failed"), + makeRun(cancelledRunId, 3, "cancelled"), + makeRun(completedRunId, 4, "completed"), + makeRun(rolledBackRunId, 5, "rolled_back"), + makeRun(currentRunId, 6, "running"), + makeRun( + RunId.make("run:inherited-background-selection:other"), + 1, + "interrupted", + otherThreadId, + ), + ], + turnItems: [ + makeItem(inheritedItemId, interruptedRunId, "running"), + makeItem(failedItemId, failedRunId, "running"), + makeItem(cancelledItemId, cancelledRunId, "running"), + makeItem( + TurnItemId.make("turn-item:inherited-background-selection:completed-run"), + completedRunId, + "running", + ), + makeItem( + TurnItemId.make("turn-item:inherited-background-selection:terminal"), + interruptedRunId, + "completed", + ), + makeItem( + TurnItemId.make("turn-item:inherited-background-selection:ordinary"), + interruptedRunId, + "running", + "reasoning", + ), + makeItem( + TurnItemId.make("turn-item:inherited-background-selection:rolled-back"), + rolledBackRunId, + "running", + ), + makeItem( + TurnItemId.make("turn-item:inherited-background-selection:other-thread"), + interruptedRunId, + "running", + "subagent", + otherThreadId, + ), + makeItem( + TurnItemId.make("turn-item:inherited-background-selection:foreign-provider"), + interruptedRunId, + "running", + "subagent", + threadId, + foreignProviderThreadId, + ), + ], + }); + + assert.deepEqual(selected, [ + { id: inheritedItemId, runId: interruptedRunId }, + { id: failedItemId, runId: failedRunId }, + { id: cancelledItemId, runId: cancelledRunId }, + ]); +}); + it("does not carry interrupted child ownership into later attempts", () => { assert.isFalse(canRouteRelatedSubagent("interrupted")); assert.isFalse(canRouteRelatedSubagent("failed")); @@ -543,55 +735,952 @@ it.effect("ingests the trailing subagent item completion after the subagent row }), ); -it.effect("keeps ingesting a child thread's late background item completion", () => - Effect.gen(function* () { - const observed = yield* runBackgroundItemScenario("bg-child-item", (ids) => [ - childThreadCreatedEvent(ids), - subagentEvent(ids, "running"), - childBackgroundTurnItemEvent(ids, "running", 1), - rootTerminalEvent(ids, "completed"), - subagentEvent(ids, "completed"), - childBackgroundTurnItemEvent(ids, "completed", 2), - ]); - assert.deepEqual(observed, [ - "subagent:running", - "turn_item:running", +it.effect("keeps ingesting a child thread's late background item completion", () => + Effect.gen(function* () { + const observed = yield* runBackgroundItemScenario("bg-child-item", (ids) => [ + childThreadCreatedEvent(ids), + subagentEvent(ids, "running"), + childBackgroundTurnItemEvent(ids, "running", 1), + rootTerminalEvent(ids, "completed"), + subagentEvent(ids, "completed"), + childBackgroundTurnItemEvent(ids, "completed", 2), + ]); + assert.deepEqual(observed, [ + "subagent:running", + "turn_item:running", + "root-finalized", + "subagent:completed", + "turn_item:completed", + ]); + }), +); + +it.effect("keeps ingesting until the last of several background items terminalizes", () => + Effect.gen(function* () { + const secondItemId = TurnItemId.make("turn-item:bg-multi:second"); + const observed = yield* runBackgroundItemScenario("bg-multi", (ids) => [ + backgroundTurnItemEvent(ids, "command_execution", "running", 1), + backgroundTurnItemEvent(ids, "dynamic_tool", "running", 2, secondItemId), + rootTerminalEvent(ids, "completed"), + backgroundTurnItemEvent(ids, "command_execution", "completed", 3), + backgroundTurnItemEvent(ids, "dynamic_tool", "completed", 4, secondItemId), + ]); + assert.deepEqual(observed, [ + "turn_item:running", + "turn_item:running", + "root-finalized", + "turn_item:completed", + "turn_item:completed", + ]); + }), +); + +it.effect("does not pin ingestion on background items when the root turn is interrupted", () => + Effect.gen(function* () { + const observed = yield* runBackgroundItemScenario("bg-interrupted", (ids) => [ + backgroundTurnItemEvent(ids, "command_execution", "running", 1), + rootTerminalEvent(ids, "interrupted"), + backgroundTurnItemEvent(ids, "command_execution", "completed", 2), + ]); + assert.deepEqual(observed, ["turn_item:running", "root-finalized"]); + }), +); + +it.effect("seeds inherited background items before their next update", () => + Effect.gen(function* () { + const key = "inherited-background-seeded"; + const priorRunId = RunId.make(`run:${key}:prior`); + const observed = yield* runBackgroundItemScenario( + key, + (ids) => [ + rootTerminalEvent(ids, "completed"), + backgroundTurnItemEventForRun(ids, priorRunId, "subagent", "completed", 1), + ], + { + loadInheritedBackgroundTurnItems: () => + Effect.succeed([{ id: TurnItemId.make(`turn-item:${key}`), runId: priorRunId }]), + }, + ); + + assert.deepEqual(observed, ["root-finalized", "turn_item:completed"]); + }), +); + +it.effect("releases the live run after an inherited background item terminalizes", () => + Effect.gen(function* () { + const key = "inherited-background-terminal"; + const priorRunId = RunId.make(`run:${key}:prior`); + const observed = yield* runBackgroundItemScenario( + key, + (ids) => [ + backgroundTurnItemEventForRun(ids, priorRunId, "subagent", "running", 1), + rootTerminalEvent(ids, "completed"), + backgroundTurnItemEventForRun(ids, priorRunId, "subagent", "completed", 2), + ], + { + loadInheritedBackgroundTurnItems: () => + Effect.succeed([{ id: TurnItemId.make(`turn-item:${key}`), runId: priorRunId }]), + }, + ); + + assert.deepEqual(observed, ["turn_item:running", "root-finalized", "turn_item:completed"]); + }), +); + +it.effect("does not hold the live stream open for a foreign provider's background item", () => + Effect.gen(function* () { + const key = "inherited-background-foreign-provider"; + const ids = backgroundScenarioIds(key); + const priorRunId = RunId.make(`run:${key}:prior`); + const foreignProviderThreadId = ProviderThreadId.make(`provider-thread:${key}:foreign-session`); + const foreignItem = { + id: ids.itemId, + threadId: ids.threadId, + runId: priorRunId, + providerThreadId: foreignProviderThreadId, + type: "subagent", + status: "running", + } as OrchestrationV2TurnItem; + const inherited = selectInheritedBackgroundTurnItems({ + threadId: ids.threadId, + currentProviderThreadId: ids.providerThreadId, + currentRunOrdinal: 2, + runs: [ + { + id: priorRunId, + threadId: ids.threadId, + ordinal: 1, + status: "interrupted", + } as OrchestrationV2Run, + ], + turnItems: [foreignItem], + }); + + const observed = yield* runBackgroundItemScenario( + key, + (scenarioIds) => [rootTerminalEvent(scenarioIds, "completed")], + { + keepEventStreamOpen: true, + loadInheritedBackgroundTurnItems: () => Effect.succeed(inherited), + }, + ); + + assert.deepEqual(inherited, []); + assert.deepEqual(observed, ["root-finalized"]); + }), +); + +it.effect("refreshes inherited background items after event subscription", () => + Effect.gen(function* () { + const key = "inherited-background-subscription-refresh"; + const ids = backgroundScenarioIds(key); + const priorRunId = RunId.make(`run:${key}:prior`); + const itemStatus = yield* Ref.make("running"); + const loadInheritedBackgroundTurnItems = () => + Ref.get(itemStatus).pipe( + Effect.map((status) => + selectInheritedBackgroundTurnItems({ + threadId: ids.threadId, + currentProviderThreadId: ids.providerThreadId, + currentRunOrdinal: 2, + runs: [ + { + id: priorRunId, + threadId: ids.threadId, + ordinal: 1, + status: "interrupted", + } as OrchestrationV2Run, + ], + turnItems: [ + { + id: ids.itemId, + threadId: ids.threadId, + runId: priorRunId, + providerThreadId: ids.providerThreadId, + type: "subagent", + status, + } as OrchestrationV2TurnItem, + ], + }), + ), + ); + + const observed = yield* runBackgroundItemScenario( + key, + (scenarioIds) => [rootTerminalEvent(scenarioIds, "completed")], + { + keepEventStreamOpen: true, + loadInheritedBackgroundTurnItems, + onSubscribe: Ref.set(itemStatus, "completed"), + }, + ); + + assert.deepEqual(yield* loadInheritedBackgroundTurnItems(), []); + assert.deepEqual(observed, ["root-finalized"]); + }), +); + +it.effect( + "keeps ingesting a late empty provider-thread roster while thread-scoped pending work is true", + () => + Effect.gen(function* () { + const key = "bg-roster-pending-work"; + const ids = backgroundScenarioIds(key); + const providerInstanceId = ProviderInstanceId.make("codex"); + const now = yield* DateTime.now; + const observed = yield* Ref.make>([]); + const pendingByProviderThreadId = yield* Ref.make(new Map([[ids.providerThreadId, true]])); + const scopedProbeArgs = yield* Ref.make>([]); + const ingestCalls = yield* Ref.make< + ReadonlyArray<{ + readonly activeAttemptId: RunAttemptId | null; + readonly eventType: string; + readonly hasWriteIfRunCurrent: boolean; + readonly hasWriteIfProviderThreadOwner: boolean; + readonly expectedLastRunOrdinal: number | null; + readonly runId: RunId | null; + readonly rosterLength: number | null; + }> + >([]); + const ingestionDone = yield* Deferred.make(); + const testLayer = runExecutionServiceLayer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(CheckpointServiceV2)({ captureBaseline: () => Effect.void }), + Layer.mock(EventSinkV2)({ + write: () => Effect.succeed([]), + writeWithEffects: (input) => + Effect.gen(function* () { + if ( + input.events.some( + (event) => event.type === "run.updated" && event.runId === ids.runId, + ) + ) { + yield* Ref.update(observed, (current) => [...current, "root-finalized"]); + } + return []; + }), + writeIfRunCurrent: () => Effect.succeed({ committed: true, storedEvents: [] }), + writeIfProviderThreadOwner: () => + Effect.succeed({ committed: true, storedEvents: [] }), + }), + idAllocatorLayer, + Layer.mock(ProviderEventIngestorV2)({ + ingestNormalized: (input) => + Effect.gen(function* () { + const event = input.event; + const rosterLength = + event.type === "provider_thread.updated" + ? (event.providerThread.pendingBackgroundTasks?.length ?? 0) + : null; + yield* Ref.update(ingestCalls, (current) => [ + ...current, + { + activeAttemptId: input.writeIfProviderThreadOwner?.activeAttemptId ?? null, + eventType: event.type, + hasWriteIfRunCurrent: input.writeIfRunCurrent !== undefined, + hasWriteIfProviderThreadOwner: input.writeIfProviderThreadOwner !== undefined, + expectedLastRunOrdinal: + input.writeIfProviderThreadOwner?.expectedLastRunOrdinal ?? null, + runId: input.writeIfProviderThreadOwner?.runId ?? null, + rosterLength, + }, + ]); + if (event.type === "provider_thread.updated" && rosterLength === 0) { + yield* Ref.update(pendingByProviderThreadId, (current) => { + const next = new Map(current); + next.set(event.providerThread.id, false); + return next; + }); + yield* Ref.update(observed, (current) => [...current, "roster-cleared"]); + } + if (event.type === "turn.terminal") { + yield* Ref.update(observed, (current) => [...current, "terminal"]); + } + return []; + }), + }), + ServerSettingsService.layerTest(), + ), + ), + ); + + const providerThreadBase = { + id: ids.providerThreadId, + driver, + providerInstanceId, + providerSessionId: ProviderSessionId.make(`session:${key}`), + appThreadId: ids.threadId, + ownerNodeId: null, + nativeThreadRef: null, + nativeConversationHeadRef: null, + status: "idle" as const, + firstRunOrdinal: 1, + lastRunOrdinal: 1, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }; + + yield* Effect.gen(function* () { + const runExecution = yield* RunExecutionServiceV2; + yield* runExecution.startRootRun({ + commandId: CommandId.make(`command:${key}`), + appThread: { id: ids.threadId } as OrchestrationV2AppThread, + providerSessionId: ProviderSessionId.make(`session:${key}`), + session: { + events: Stream.empty, + // Session-wide stays true forever; the root must consult the + // thread-scoped probe instead of being pinned by siblings. + hasPendingBackgroundWork: Effect.succeed(true), + hasPendingBackgroundWorkForThread: (providerThread: OrchestrationV2ProviderThread) => + Effect.gen(function* () { + yield* Ref.update(scopedProbeArgs, (current) => [...current, providerThread.id]); + return (yield* Ref.get(pendingByProviderThreadId)).get(providerThread.id) === true; + }), + subscribeEvents: Effect.succeed({ + events: Stream.fromIterable([ + { + type: "provider_thread.updated", + driver, + providerThread: { + ...providerThreadBase, + status: "active" as const, + pendingBackgroundTasks: [ + { taskId: "bg-1", description: "sleep 20", taskType: "local_bash" }, + ], + updatedAt: now, + }, + } as ProviderAdapterV2Event, + rootTerminalEvent(ids, "completed"), + { + type: "provider_thread.updated", + driver, + providerThread: { + ...providerThreadBase, + status: "idle" as const, + pendingBackgroundTasks: [], + updatedAt: now, + }, + } as ProviderAdapterV2Event, + ]), + close: Deferred.succeed(ingestionDone, undefined), + }), + startTurn: () => Effect.void, + } as unknown as ProviderAdapterV2SessionRuntime, + run: { + id: ids.runId, + threadId: ids.threadId, + ordinal: 1, + providerInstanceId, + } as OrchestrationV2Run, + rootNode: { id: ids.rootNodeId } as OrchestrationV2ExecutionNode, + checkpointScope: { + id: CheckpointScopeId.make(`checkpoint-scope:${key}`), + } as OrchestrationV2CheckpointScope, + providerThread: providerThreadBase as OrchestrationV2ProviderThread, + attempt: { + id: ids.attemptId, + providerTurnId: ids.rootProviderTurnId, + } as OrchestrationV2RunAttempt, + attemptId: ids.attemptId, + providerTurnOrdinal: 1, + message: { + messageId: MessageId.make(`message:${key}:user`), + text: "Start background work and settle.", + attachments: [], + createdBy: "user", + creationSource: "web", + }, + modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" }, + runtimePolicy: { + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + approvalPolicy: "never", + sandboxPolicy: { + type: "readOnly", + access: { type: "fullAccess" }, + networkAccess: false, + }, + }, + }); + }).pipe(Effect.provide(testLayer)); + + const closed = yield* Deferred.await(ingestionDone).pipe(Effect.timeoutOption("2 seconds")); + assert.isTrue(Option.isSome(closed), "event subscription did not release"); + assert.deepEqual(yield* Ref.get(observed), ["terminal", "root-finalized", "roster-cleared"]); + assert.isTrue((yield* Ref.get(scopedProbeArgs)).includes(ids.providerThreadId)); + + const calls = yield* Ref.get(ingestCalls); + const preTerminalRoster = calls.find( + (call) => call.eventType === "provider_thread.updated" && call.rosterLength === 1, + ); + const lateClear = calls.find( + (call) => call.eventType === "provider_thread.updated" && call.rosterLength === 0, + ); + assert.isDefined(preTerminalRoster); + assert.isTrue(preTerminalRoster?.hasWriteIfRunCurrent); + assert.isFalse(preTerminalRoster?.hasWriteIfProviderThreadOwner); + assert.isDefined(lateClear); + assert.isFalse( + lateClear?.hasWriteIfRunCurrent, + "late empty roster must not use stale writeIfRunCurrent running-gate", + ); + assert.isTrue( + lateClear?.hasWriteIfProviderThreadOwner, + "late empty roster must gate on provider-thread ownership", + ); + assert.equal(lateClear?.expectedLastRunOrdinal, 1); + assert.equal(lateClear?.runId, ids.runId); + assert.equal(lateClear?.activeAttemptId, ids.attemptId); + }), +); + +it.effect("drops late root provider-thread writes from a superseded attempt", () => + Effect.gen(function* () { + const key = "bg-roster-attempt-owner-lost"; + const ids = backgroundScenarioIds(key); + const replacementAttemptId = RunAttemptId.make(`attempt:${key}:replacement`); + const providerInstanceId = ProviderInstanceId.make("codex"); + const now = yield* DateTime.now; + const observed = yield* Ref.make>([]); + // Probe stays true forever so only ownership-loss can release the stream. + const ingestionDone = yield* Deferred.make(); + const ingestCalls = yield* Ref.make< + ReadonlyArray<{ + readonly activeAttemptId: RunAttemptId | null; + readonly eventType: string; + readonly hasWriteIfProviderThreadOwner: boolean; + readonly expectedLastRunOrdinal: number | null; + readonly rosterLength: number | null; + readonly committed: boolean; + }> + >([]); + const testLayer = runExecutionServiceLayer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(CheckpointServiceV2)({ captureBaseline: () => Effect.void }), + Layer.mock(EventSinkV2)({ + write: () => Effect.succeed([]), + writeWithEffects: (input) => + Effect.gen(function* () { + if ( + input.events.some( + (event) => event.type === "run.updated" && event.runId === ids.runId, + ) + ) { + yield* Ref.update(observed, (current) => [...current, "root-finalized"]); + } + return []; + }), + writeIfRunCurrent: () => Effect.succeed({ committed: true, storedEvents: [] }), + writeIfProviderThreadOwner: () => + Effect.succeed({ committed: false, storedEvents: [] }), + }), + idAllocatorLayer, + Layer.mock(ProviderEventIngestorV2)({ + ingestNormalized: (input) => + Effect.gen(function* () { + const event = input.event; + const rosterLength = + event.type === "provider_thread.updated" + ? (event.providerThread.pendingBackgroundTasks?.length ?? 0) + : null; + const ownerGate = input.writeIfProviderThreadOwner; + // Simulate the EventSink reject after a same-run replacement + // changes activeAttemptId without advancing lastRunOrdinal. + const rejectAsStaleOwner = + ownerGate !== undefined && ownerGate.activeAttemptId !== replacementAttemptId; + yield* Ref.update(ingestCalls, (current) => [ + ...current, + { + activeAttemptId: ownerGate?.activeAttemptId ?? null, + eventType: event.type, + hasWriteIfProviderThreadOwner: ownerGate !== undefined, + expectedLastRunOrdinal: ownerGate?.expectedLastRunOrdinal ?? null, + rosterLength, + committed: !rejectAsStaleOwner, + }, + ]); + if (event.type === "turn.terminal") { + yield* Ref.update(observed, (current) => [...current, "terminal"]); + } + if (event.type === "provider_thread.updated" && rejectAsStaleOwner) { + yield* Ref.update(observed, (current) => [...current, "stale-owner-rejected"]); + return []; + } + if (event.type === "provider_thread.updated") { + yield* Ref.update(observed, (current) => [...current, "roster-written"]); + } + return []; + }), + }), + ServerSettingsService.layerTest(), + ), + ), + ); + + const providerThreadBase = { + id: ids.providerThreadId, + driver, + providerInstanceId, + providerSessionId: ProviderSessionId.make(`session:${key}`), + appThreadId: ids.threadId, + ownerNodeId: null, + nativeThreadRef: null, + nativeConversationHeadRef: null, + status: "idle" as const, + firstRunOrdinal: 1, + lastRunOrdinal: 1, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }; + + yield* Effect.gen(function* () { + const runExecution = yield* RunExecutionServiceV2; + yield* runExecution.startRootRun({ + commandId: CommandId.make(`command:${key}`), + appThread: { id: ids.threadId } as OrchestrationV2AppThread, + providerSessionId: ProviderSessionId.make(`session:${key}`), + session: { + events: Stream.empty, + hasPendingBackgroundWork: Effect.succeed(true), + hasPendingBackgroundWorkForThread: () => Effect.succeed(true), + subscribeEvents: Effect.succeed({ + events: Stream.fromIterable([ + { + type: "provider_thread.updated", + driver, + providerThread: { + ...providerThreadBase, + status: "active" as const, + pendingBackgroundTasks: [ + { taskId: "bg-stale", description: "sleep 20", taskType: "local_bash" }, + ], + updatedAt: now, + }, + } as ProviderAdapterV2Event, + rootTerminalEvent(ids, "completed"), + // Late snapshot after a replacement attempt claimed the same + // run ordinal. Without attempt gating this would clobber it. + { + type: "provider_thread.updated", + driver, + providerThread: { + ...providerThreadBase, + status: "idle" as const, + lastRunOrdinal: 1, + pendingBackgroundTasks: [ + { taskId: "bg-stale", description: "sleep 20", taskType: "local_bash" }, + ], + updatedAt: now, + }, + } as ProviderAdapterV2Event, + ]), + close: Deferred.succeed(ingestionDone, undefined), + }), + startTurn: () => Effect.void, + } as unknown as ProviderAdapterV2SessionRuntime, + run: { + id: ids.runId, + threadId: ids.threadId, + ordinal: 1, + providerInstanceId, + } as OrchestrationV2Run, + rootNode: { id: ids.rootNodeId } as OrchestrationV2ExecutionNode, + checkpointScope: { + id: CheckpointScopeId.make(`checkpoint-scope:${key}`), + } as OrchestrationV2CheckpointScope, + providerThread: providerThreadBase as OrchestrationV2ProviderThread, + attempt: { + id: ids.attemptId, + providerTurnId: ids.rootProviderTurnId, + } as OrchestrationV2RunAttempt, + attemptId: ids.attemptId, + providerTurnOrdinal: 1, + message: { + messageId: MessageId.make(`message:${key}:user`), + text: "Background work that loses ownership.", + attachments: [], + createdBy: "user", + creationSource: "web", + }, + modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" }, + runtimePolicy: { + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + approvalPolicy: "never", + sandboxPolicy: { + type: "readOnly", + access: { type: "fullAccess" }, + networkAccess: false, + }, + }, + }); + }).pipe(Effect.provide(testLayer)); + + const closed = yield* Deferred.await(ingestionDone).pipe(Effect.timeoutOption("2 seconds")); + assert.isTrue( + Option.isSome(closed), + "subscription must release after ownership-loss reject even when probe stays true", + ); + assert.deepEqual(yield* Ref.get(observed), [ + "roster-written", + "terminal", "root-finalized", - "subagent:completed", - "turn_item:completed", + "stale-owner-rejected", ]); + + const calls = yield* Ref.get(ingestCalls); + const lateStale = calls.find( + (call) => + call.eventType === "provider_thread.updated" && + call.hasWriteIfProviderThreadOwner && + call.rosterLength === 1, + ); + assert.isDefined(lateStale); + assert.equal(lateStale?.expectedLastRunOrdinal, 1); + assert.equal(lateStale?.activeAttemptId, ids.attemptId); + assert.isFalse(lateStale?.committed); }), ); -it.effect("keeps ingesting until the last of several background items terminalizes", () => - Effect.gen(function* () { - const secondItemId = TurnItemId.make("turn-item:bg-multi:second"); - const observed = yield* runBackgroundItemScenario("bg-multi", (ids) => [ - backgroundTurnItemEvent(ids, "command_execution", "running", 1), - backgroundTurnItemEvent(ids, "dynamic_tool", "running", 2, secondItemId), - rootTerminalEvent(ids, "completed"), - backgroundTurnItemEvent(ids, "command_execution", "completed", 3), - backgroundTurnItemEvent(ids, "dynamic_tool", "completed", 4, secondItemId), - ]); - assert.deepEqual(observed, [ - "turn_item:running", - "turn_item:running", - "root-finalized", - "turn_item:completed", - "turn_item:completed", - ]); - }), +it.effect( + "keeps ingesting a late background turn-item completion after ownership-loss rejects roster writes", + () => + Effect.gen(function* () { + const key = "bg-item-after-owner-lost"; + const ids = backgroundScenarioIds(key); + const providerInstanceId = ProviderInstanceId.make("codex"); + const now = yield* DateTime.now; + const observed = yield* Ref.make>([]); + // Probe stays true forever; open background items must pin the stream + // past ownership-loss so late turn_item completions still land. + const ingestionDone = yield* Deferred.make(); + const testLayer = runExecutionServiceLayer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(CheckpointServiceV2)({ captureBaseline: () => Effect.void }), + Layer.mock(EventSinkV2)({ + write: () => Effect.succeed([]), + writeWithEffects: (input) => + Effect.gen(function* () { + if ( + input.events.some( + (event) => event.type === "run.updated" && event.runId === ids.runId, + ) + ) { + yield* Ref.update(observed, (current) => [...current, "root-finalized"]); + } + return []; + }), + writeIfRunCurrent: () => Effect.succeed({ committed: true, storedEvents: [] }), + writeIfProviderThreadOwner: () => + Effect.succeed({ committed: false, storedEvents: [] }), + }), + idAllocatorLayer, + Layer.mock(ProviderEventIngestorV2)({ + ingestNormalized: (input) => + Effect.gen(function* () { + const event = input.event; + if (event.type === "turn.terminal") { + yield* Ref.update(observed, (current) => [...current, "terminal"]); + } + if ( + event.type === "provider_thread.updated" && + input.writeIfProviderThreadOwner !== undefined + ) { + yield* Ref.update(observed, (current) => [...current, "stale-owner-rejected"]); + return []; + } + if (event.type === "turn_item.updated") { + yield* Ref.update(observed, (current) => [ + ...current, + `turn_item:${event.turnItem.status}`, + ]); + } + return []; + }), + }), + ServerSettingsService.layerTest(), + ), + ), + ); + + const providerThreadBase = { + id: ids.providerThreadId, + driver, + providerInstanceId, + providerSessionId: ProviderSessionId.make(`session:${key}`), + appThreadId: ids.threadId, + ownerNodeId: null, + nativeThreadRef: null, + nativeConversationHeadRef: null, + status: "idle" as const, + firstRunOrdinal: 1, + lastRunOrdinal: 1, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }; + + yield* Effect.gen(function* () { + const runExecution = yield* RunExecutionServiceV2; + yield* runExecution.startRootRun({ + commandId: CommandId.make(`command:${key}`), + appThread: { id: ids.threadId } as OrchestrationV2AppThread, + providerSessionId: ProviderSessionId.make(`session:${key}`), + session: { + events: Stream.empty, + hasPendingBackgroundWork: Effect.succeed(true), + hasPendingBackgroundWorkForThread: () => Effect.succeed(true), + subscribeEvents: Effect.succeed({ + events: Stream.fromIterable([ + backgroundTurnItemEvent(ids, "command_execution", "running", 1), + rootTerminalEvent(ids, "completed"), + // Ownership reject after a newer run claimed lastRunOrdinal. + { + type: "provider_thread.updated", + driver, + providerThread: { + ...providerThreadBase, + status: "idle" as const, + lastRunOrdinal: 1, + pendingBackgroundTasks: [ + { taskId: "bg-stale", description: "sleep 20", taskType: "local_bash" }, + ], + updatedAt: now, + }, + } as ProviderAdapterV2Event, + // Late completion still writable (turn_item writes are not + // ownership-gated); stream must stay open for it. + backgroundTurnItemEvent(ids, "command_execution", "completed", 2), + ]), + close: Deferred.succeed(ingestionDone, undefined), + }), + startTurn: () => Effect.void, + } as unknown as ProviderAdapterV2SessionRuntime, + run: { + id: ids.runId, + threadId: ids.threadId, + ordinal: 1, + providerInstanceId, + } as OrchestrationV2Run, + rootNode: { id: ids.rootNodeId } as OrchestrationV2ExecutionNode, + checkpointScope: { + id: CheckpointScopeId.make(`checkpoint-scope:${key}`), + } as OrchestrationV2CheckpointScope, + providerThread: providerThreadBase as OrchestrationV2ProviderThread, + attempt: { + id: ids.attemptId, + providerTurnId: ids.rootProviderTurnId, + } as OrchestrationV2RunAttempt, + attemptId: ids.attemptId, + providerTurnOrdinal: 1, + message: { + messageId: MessageId.make(`message:${key}:user`), + text: "Background item after ownership loss.", + attachments: [], + createdBy: "user", + creationSource: "web", + }, + modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" }, + runtimePolicy: { + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + approvalPolicy: "never", + sandboxPolicy: { + type: "readOnly", + access: { type: "fullAccess" }, + networkAccess: false, + }, + }, + }); + }).pipe(Effect.provide(testLayer)); + + const closed = yield* Deferred.await(ingestionDone).pipe(Effect.timeoutOption("2 seconds")); + assert.isTrue( + Option.isSome(closed), + "subscription must release after background item completes past ownership loss", + ); + assert.deepEqual(yield* Ref.get(observed), [ + "turn_item:running", + "terminal", + "root-finalized", + "stale-owner-rejected", + "turn_item:completed", + ]); + }), ); -it.effect("does not pin ingestion on background items when the root turn is interrupted", () => - Effect.gen(function* () { - const observed = yield* runBackgroundItemScenario("bg-interrupted", (ids) => [ - backgroundTurnItemEvent(ids, "command_execution", "running", 1), - rootTerminalEvent(ids, "interrupted"), - backgroundTurnItemEvent(ids, "command_execution", "completed", 2), - ]); - assert.deepEqual(observed, ["turn_item:running", "root-finalized"]); - }), +it.effect( + "does not pin ingestion on a sibling session-wide pending state when this thread has no roster", + () => + Effect.gen(function* () { + const key = "bg-roster-sibling-not-pin"; + const ids = backgroundScenarioIds(key); + const providerInstanceId = ProviderInstanceId.make("codex"); + const now = yield* DateTime.now; + const observed = yield* Ref.make>([]); + const ingestionDone = yield* Deferred.make(); + const scopedProbeArgs = yield* Ref.make>([]); + const testLayer = runExecutionServiceLayer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(CheckpointServiceV2)({ captureBaseline: () => Effect.void }), + Layer.mock(EventSinkV2)({ + write: () => Effect.succeed([]), + writeWithEffects: (input) => + Effect.gen(function* () { + if ( + input.events.some( + (event) => event.type === "run.updated" && event.runId === ids.runId, + ) + ) { + yield* Ref.update(observed, (current) => [...current, "root-finalized"]); + } + return []; + }), + writeIfRunCurrent: () => Effect.succeed({ committed: true, storedEvents: [] }), + }), + idAllocatorLayer, + Layer.mock(ProviderEventIngestorV2)({ + ingestNormalized: (input) => + Effect.gen(function* () { + if (input.event.type === "turn.terminal") { + yield* Ref.update(observed, (current) => [...current, "terminal"]); + } + return []; + }), + }), + ServerSettingsService.layerTest(), + ), + ), + ); + + const providerThreadBase = { + id: ids.providerThreadId, + driver, + providerInstanceId, + providerSessionId: ProviderSessionId.make(`session:${key}`), + appThreadId: ids.threadId, + ownerNodeId: null, + nativeThreadRef: { + driver, + nativeId: "native-self", + strength: "strong" as const, + }, + nativeConversationHeadRef: null, + status: "idle" as const, + firstRunOrdinal: 1, + lastRunOrdinal: 1, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }; + const siblingProviderThreadId = ProviderThreadId.make(`provider-thread:${key}:sibling`); + + yield* Effect.gen(function* () { + const runExecution = yield* RunExecutionServiceV2; + yield* runExecution.startRootRun({ + commandId: CommandId.make(`command:${key}`), + appThread: { id: ids.threadId } as OrchestrationV2AppThread, + providerSessionId: ProviderSessionId.make(`session:${key}`), + session: { + events: Stream.empty, + // Session-wide stays true (sibling has work). Stop must use only + // the scoped probe for this root's provider thread. + hasPendingBackgroundWork: Effect.succeed(true), + hasPendingBackgroundWorkForThread: (providerThread: OrchestrationV2ProviderThread) => + Effect.gen(function* () { + yield* Ref.update(scopedProbeArgs, (current) => [...current, providerThread.id]); + // Own thread has no roster; sibling would report true if probed. + return providerThread.id !== ids.providerThreadId; + }), + subscribeEvents: Effect.succeed({ + events: Stream.fromIterable([ + rootTerminalEvent(ids, "completed"), + // Sibling thread update after terminal. Own-thread scoped + // pending is false, so this root must release without waiting + // for sibling-driven session-wide pending work. + { + type: "provider_thread.updated", + driver, + providerThread: { + ...providerThreadBase, + id: siblingProviderThreadId, + appThreadId: ThreadId.make(`thread:${key}:sibling`), + nativeThreadRef: { + driver, + nativeId: "native-sibling", + strength: "strong" as const, + }, + pendingBackgroundTasks: [{ taskId: "sibling-bg", description: "other thread" }], + updatedAt: now, + }, + } as ProviderAdapterV2Event, + ]), + close: Deferred.succeed(ingestionDone, undefined), + }), + startTurn: () => Effect.void, + } as unknown as ProviderAdapterV2SessionRuntime, + run: { + id: ids.runId, + threadId: ids.threadId, + ordinal: 1, + providerInstanceId, + } as OrchestrationV2Run, + rootNode: { id: ids.rootNodeId } as OrchestrationV2ExecutionNode, + checkpointScope: { + id: CheckpointScopeId.make(`checkpoint-scope:${key}`), + } as OrchestrationV2CheckpointScope, + providerThread: providerThreadBase as OrchestrationV2ProviderThread, + attempt: { + id: ids.attemptId, + providerTurnId: ids.rootProviderTurnId, + } as OrchestrationV2RunAttempt, + attemptId: ids.attemptId, + providerTurnOrdinal: 1, + message: { + messageId: MessageId.make(`message:${key}:user`), + text: "Settle without local pending work.", + attachments: [], + createdBy: "user", + creationSource: "web", + }, + modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" }, + runtimePolicy: { + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + approvalPolicy: "never", + sandboxPolicy: { + type: "readOnly", + access: { type: "fullAccess" }, + networkAccess: false, + }, + }, + }); + }).pipe(Effect.provide(testLayer)); + + const closed = yield* Deferred.await(ingestionDone).pipe(Effect.timeoutOption("2 seconds")); + assert.isTrue(Option.isSome(closed), "event subscription did not release"); + // Critical: release while session-wide hasPendingBackgroundWork stays true + // and the scoped probe reports false for this root's provider thread. + const observedEvents = [...(yield* Ref.get(observed))]; + assert.includeMembers(observedEvents, ["terminal", "root-finalized"]); + const probedIds = yield* Ref.get(scopedProbeArgs); + assert.isTrue(probedIds.includes(ids.providerThreadId)); + assert.isFalse(probedIds.includes(siblingProviderThreadId)); + }), ); it.effect( @@ -1750,6 +2839,20 @@ function backgroundTurnItemEvent( } as ProviderAdapterV2Event; } +function backgroundTurnItemEventForRun( + ids: BackgroundScenarioIds, + runId: RunId, + type: "command_execution" | "dynamic_tool" | "subagent", + status: "running" | "completed", + ordinal: number, +): ProviderAdapterV2Event { + const event = backgroundTurnItemEvent(ids, type, status, ordinal); + if (event.type !== "turn_item.updated") { + return event; + } + return { ...event, turnItem: { ...event.turnItem, runId } }; +} + function subagentEvent( ids: BackgroundScenarioIds, status: "running" | "completed", @@ -1956,6 +3059,13 @@ function rootTerminalEvent( function runBackgroundItemScenario( key: string, makeEvents: (ids: BackgroundScenarioIds) => ReadonlyArray, + options?: { + readonly keepEventStreamOpen?: boolean; + readonly loadInheritedBackgroundTurnItems?: () => Effect.Effect< + ReadonlyArray<{ readonly id: TurnItemId; readonly runId: RunId }> + >; + readonly onSubscribe?: Effect.Effect; + }, ) { return Effect.gen(function* () { const ids = backgroundScenarioIds(key); @@ -2014,9 +3124,16 @@ function runBackgroundItemScenario( providerSessionId: ProviderSessionId.make(`session:${key}`), session: { events: Stream.empty, - subscribeEvents: Effect.succeed({ - events: Stream.fromIterable(makeEvents(ids)), - close: Deferred.succeed(ingestionDone, undefined), + subscribeEvents: Effect.gen(function* () { + yield* options?.onSubscribe ?? Effect.void; + const events = Stream.fromIterable(makeEvents(ids)); + return { + events: + options?.keepEventStreamOpen === true + ? events.pipe(Stream.concat(Stream.never)) + : events, + close: Deferred.succeed(ingestionDone, undefined), + }; }), startTurn: () => Effect.void, } as unknown as ProviderAdapterV2SessionRuntime, @@ -2039,6 +3156,11 @@ function runBackgroundItemScenario( providerTurnId: ids.rootProviderTurnId, } as OrchestrationV2RunAttempt, attemptId: ids.attemptId, + ...(options?.loadInheritedBackgroundTurnItems === undefined + ? {} + : { + loadInheritedBackgroundTurnItems: options.loadInheritedBackgroundTurnItems, + }), providerTurnOrdinal: 1, message: { messageId: MessageId.make(`message:${key}:user`), diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index a3d72b065b7..211cbf670ec 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -52,6 +52,7 @@ export interface ProviderEventRoutingState { readonly ownedThreadIds: ReadonlySet; readonly ownedProviderThreadIds: ReadonlySet; readonly ownedProviderTurnIds: ReadonlySet; + readonly inheritedBackgroundTurnItems: ReadonlyMap; readonly rootProviderTurnId: ProviderTurnId | null; } @@ -62,6 +63,11 @@ export interface ProviderEventRouteIdentity { readonly providerThreadId: ProviderThreadId; } +export interface InheritedBackgroundTurnItemRoute { + readonly id: TurnItemId; + readonly runId: OrchestrationV2Run["id"]; +} + type ProviderTerminalEvent = Extract; function isTerminalProviderTurnStatus(status: OrchestrationV2ProviderTurn["status"]): boolean { @@ -101,6 +107,46 @@ function isTerminalTurnItemStatus(status: OrchestrationV2TurnItem["status"]): bo ); } +function isSettledRunEligibleForInheritedBackground(status: OrchestrationV2Run["status"]): boolean { + return status === "interrupted" || status === "failed" || status === "cancelled"; +} + +/** + * Transfer delivery permission for exact live background items whose original + * run no longer has a subscriber. Provider sessions are runtime containers and + * can host multiple provider threads, so the durable provider-thread lineage is + * the discriminator. Completed, rolled-back, and already-terminal items remain + * excluded. + */ +export function selectInheritedBackgroundTurnItems(input: { + readonly threadId: ThreadId; + readonly currentProviderThreadId: ProviderThreadId; + readonly currentRunOrdinal: number; + readonly runs: ReadonlyArray; + readonly turnItems: ReadonlyArray; +}): ReadonlyArray { + const settledPriorRunIds = new Set( + input.runs + .filter( + (run) => + run.threadId === input.threadId && + run.ordinal < input.currentRunOrdinal && + isSettledRunEligibleForInheritedBackground(run.status), + ) + .map((run) => run.id), + ); + return input.turnItems.flatMap((turnItem) => + turnItem.threadId === input.threadId && + turnItem.providerThreadId === input.currentProviderThreadId && + turnItem.runId !== null && + settledPriorRunIds.has(turnItem.runId) && + backgroundCapableTurnItemTypes.has(turnItem.type) && + !isTerminalTurnItemStatus(turnItem.status) + ? [{ id: turnItem.id, runId: turnItem.runId }] + : [], + ); +} + type SubagentTurnItem = Extract; type OpenRunOwnedSubagentProjection = { @@ -273,6 +319,7 @@ export function finalProviderThreadStatus( export function makeProviderEventRoutingState(input: { readonly identity: ProviderEventRouteIdentity; + readonly inheritedBackgroundTurnItems?: ReadonlyArray; readonly providerTurnId: ProviderTurnId | null; readonly relatedThreadIds?: ReadonlyArray; readonly relatedProviderThreadIds?: ReadonlyArray; @@ -285,6 +332,9 @@ export function makeProviderEventRoutingState(input: { ]), ownedProviderTurnIds: input.providerTurnId === null ? new Set() : new Set([input.providerTurnId]), + inheritedBackgroundTurnItems: new Map( + (input.inheritedBackgroundTurnItems ?? []).map((item) => [item.id, item.runId]), + ), rootProviderTurnId: input.providerTurnId, }; } @@ -362,8 +412,28 @@ export function routeProviderEvent( return [ownsRun(event.subagent.runId) || ownsChildThread(event.subagent.threadId), state]; case "message.updated": return [ownsRun(event.message.runId) || ownsChildThread(event.message.threadId), state]; - case "turn_item.updated": - return [ownsRun(event.turnItem.runId) || ownsChildThread(event.turnItem.threadId), state]; + case "turn_item.updated": { + if (ownsRun(event.turnItem.runId) || ownsChildThread(event.turnItem.threadId)) { + return [true, state]; + } + const inheritedRunId = state.inheritedBackgroundTurnItems.get(event.turnItem.id); + // Preserve the item's original ownership while allowing the one live run + // to deliver an exact carryover identity selected from the projection. + const isInheritedBackgroundItem = + event.turnItem.threadId === input.threadId && + event.turnItem.runId !== null && + event.turnItem.runId === inheritedRunId && + backgroundCapableTurnItemTypes.has(event.turnItem.type); + if (!isInheritedBackgroundItem) { + return [false, state]; + } + if (!isTerminalTurnItemStatus(event.turnItem.status)) { + return [true, state]; + } + const inheritedBackgroundTurnItems = new Map(state.inheritedBackgroundTurnItems); + inheritedBackgroundTurnItems.delete(event.turnItem.id); + return [true, { ...state, inheritedBackgroundTurnItems }]; + } case "plan.updated": return [ownsRun(event.plan.runId) || ownsChildThread(event.plan.threadId), state]; case "runtime_request.updated": @@ -427,6 +497,10 @@ export interface RunExecutionServiceV2StartRootRunInput { readonly attempt: OrchestrationV2RunAttempt; readonly attemptId: RunAttemptId; readonly providerTurnOrdinal: number; + readonly loadInheritedBackgroundTurnItems?: () => Effect.Effect< + ReadonlyArray, + unknown + >; readonly relatedThreadIds?: ReadonlyArray; readonly relatedProviderThreadIds?: ReadonlyArray; readonly shouldStartProviderTurn?: () => Effect.Effect; @@ -759,9 +833,30 @@ export const layer: Layer.Layer< attemptId: input.attempt.id, providerThreadId: input.providerThread.id, }; + const eventSubscription = + input.session.subscribeEvents === undefined + ? { events: input.session.events, close: Effect.void } + : yield* input.session.subscribeEvents; + const inheritedBackgroundTurnItems = yield* ( + input.loadInheritedBackgroundTurnItems?.() ?? Effect.succeed([]) + ).pipe( + Effect.onError(() => eventSubscription.close), + Effect.mapError( + (cause) => + new RunExecutionStartError({ + commandId: input.commandId, + runId: input.run.id, + cause, + }), + ), + ); + const inheritedBackgroundTurnItemsById = new Map( + inheritedBackgroundTurnItems.map((item) => [item.id, item.runId]), + ); const eventRouting = yield* Ref.make( makeProviderEventRoutingState({ identity: routeIdentity, + inheritedBackgroundTurnItems, providerTurnId: input.attempt.providerTurnId, ...(input.relatedThreadIds === undefined ? {} @@ -773,11 +868,12 @@ export const layer: Layer.Layer< ); const rootTerminalSeen = yield* Ref.make(false); const rootRunFinalized = yield* Ref.make(false); + const providerThreadOwnerLost = yield* Ref.make(false); const activeChildProviderTurns = yield* Ref.make>(new Set()); const activeChildSubagents = yield* Ref.make>(new Set()); const activeBackgroundTurnItems = yield* Ref.make< ReadonlySet - >(new Set()); + >(new Set(inheritedBackgroundTurnItemsById.keys())); const openRunOwnedSubagents = yield* Ref.make(emptyOpenRunOwnedSubagentProjection()); const finalizeRootRun = (terminal: ProviderTerminalEvent) => Effect.gen(function* () { @@ -889,9 +985,15 @@ export const layer: Layer.Layer< const belongsToOwnedChildThread = event.turnItem.threadId !== input.run.threadId && routing.ownedThreadIds.has(event.turnItem.threadId); + const belongsToInheritedBackgroundItem = + event.turnItem.threadId === input.run.threadId && + event.turnItem.runId !== null && + inheritedBackgroundTurnItemsById.get(event.turnItem.id) === event.turnItem.runId; if ( backgroundCapableTurnItemTypes.has(event.turnItem.type) && - (belongsToRootRun || belongsToOwnedChildThread) + (belongsToRootRun || + belongsToOwnedChildThread || + belongsToInheritedBackgroundItem) ) { yield* Ref.update(activeBackgroundTurnItems, (current) => { const next = new Set(current); @@ -934,6 +1036,7 @@ export const layer: Layer.Layer< return false; } const terminal = yield* Ref.get(terminalEvent); + // Non-completed terminals drop background tracking immediately. if (terminal !== null && terminal.status !== "completed") { return true; } @@ -950,16 +1053,40 @@ export const layer: Layer.Layer< // non-terminal, so their late completion events reach the // projection (stuck-spinner fix). Only for completed runs: // interrupted/failed turns intentionally drop background tracking - // rather than pinning the stream open. Assumes adapters emit an - // item's non-terminal event before the root terminal; an item - // first seen after the terminal is not pinned. + // rather than pinning the stream open. Newly owned items depend on + // adapters emitting a non-terminal event before the root terminal. + // Exact inherited items are seeded from their selected durable rows. + // + // Owner loss (a newer run claimed lastRunOrdinal) must not close + // this stream while these sets are non-empty: turn_item.updated + // writes are not ownership-gated, so late completions still land. const backgroundItems = yield* Ref.get(activeBackgroundTurnItems); - return backgroundItems.size === 0; + if (backgroundItems.size > 0) { + return false; + } + // Owner loss means do not hold the stream open solely for the + // roster probe; once background sets are empty, release. + if (yield* Ref.get(providerThreadOwnerLost)) { + return true; + } + // Claude background Bash has no turn-item projection. Keep the + // stream open while this root's provider thread still reports + // pending roster work so late empty updates can clear Waiting. + // Use only the thread-scoped probe: session-wide pending work + // (siblings, wake buffers, session subagents) must not pin this + // root subscription. Session idle release still uses + // hasPendingBackgroundWork via ProviderSessionManager. + const latestProviderThreadSnapshot = yield* Ref.get(latestProviderThread); + if (input.session.hasPendingBackgroundWorkForThread !== undefined) { + const hasPendingWork = yield* input.session + .hasPendingBackgroundWorkForThread(latestProviderThreadSnapshot) + .pipe(Effect.catchCause(() => Effect.succeed(false))); + if (hasPendingWork) { + return false; + } + } + return true; }); - const eventSubscription = - input.session.subscribeEvents === undefined - ? { events: input.session.events, close: Effect.void } - : yield* input.session.subscribeEvents; const providerEventFiber = yield* eventSubscription.events.pipe( Stream.filterEffect((event) => Ref.modify(eventRouting, (state) => routeProviderEvent(event, routeIdentity, state)), @@ -969,6 +1096,15 @@ export const layer: Layer.Layer< let storedEventCount = 0; const shouldDeliver = shouldDeliverProviderEvent(event, assistantStreamingEnabled); if (shouldDeliver) { + // Root provider_thread.updated always uses an ownership gate: + // pre-terminal writeIfRunCurrent (attempt still running), or + // post-terminal writeIfProviderThreadOwner so late roster + // clears still land while this attempt owns the run and this + // run owns lastRunOrdinal. + const rootTerminalAlreadySeen = yield* Ref.get(rootTerminalSeen); + const isRootProviderThreadUpdate = + event.type === "provider_thread.updated" && + event.providerThread.id === input.providerThread.id; const storedEvents = yield* providerEventIngestor.ingestNormalized({ providerSessionId: input.providerSessionId, providerInstanceId: input.run.providerInstanceId, @@ -976,18 +1112,35 @@ export const layer: Layer.Layer< runId: input.run.id, nodeId: input.rootNode.id, event, - ...(event.type === "provider_thread.updated" && - event.providerThread.id === input.providerThread.id - ? { - writeIfRunCurrent: { - runId: input.run.id, - activeAttemptId: input.attempt.id, - expectedStatus: "running" as const, - }, - } + ...(isRootProviderThreadUpdate + ? rootTerminalAlreadySeen + ? { + writeIfProviderThreadOwner: { + providerThreadId: input.providerThread.id, + runId: input.run.id, + activeAttemptId: input.attempt.id, + expectedLastRunOrdinal: input.run.ordinal, + }, + } + : { + writeIfRunCurrent: { + runId: input.run.id, + activeAttemptId: input.attempt.id, + expectedStatus: "running" as const, + }, + } : {}), }); storedEventCount = storedEvents.length; + if ( + isRootProviderThreadUpdate && + rootTerminalAlreadySeen && + storedEventCount === 0 + ) { + // Ownership lost (or thread row missing). Stop pinning the + // stream on this run's background probe. + yield* Ref.set(providerThreadOwnerLost, true); + } } if (event.type === "provider_thread.updated") { if (event.providerThread.id === input.providerThread.id && storedEventCount > 0) { diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 6a06e654f46..2e6eb6f18c7 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -44,6 +44,7 @@ import { resolvePromptInjectedEffort, } from "@t3tools/shared/model"; import { CHAT_LIST_ANCHOR_OFFSET } from "@t3tools/shared/chatList"; +import { derivePendingBackgroundWork } from "@t3tools/shared/orchestrationV2PendingBackgroundWork"; import { projectScriptCwd, projectScriptRuntimeEnv } from "@t3tools/shared/projectScripts"; import { truncate } from "@t3tools/shared/String"; import { nextTerminalId, resolveTerminalSessionLabel } from "@t3tools/shared/terminalLabels"; @@ -2125,6 +2126,26 @@ function ChatViewContent(props: ChatViewProps) { threadError, }); const isWorking = phase === "running" || isSendBusy || isConnecting || isRevertingCheckpoint; + const pendingBackgroundTasks = useMemo(() => { + if (serverProjection === null || serverProjection === undefined) { + return []; + } + const latestRun = + serverProjection.runs.length === 0 + ? null + : serverProjection.runs.reduce((latest, candidate) => + candidate.ordinal > latest.ordinal ? candidate : latest, + ); + return [ + ...derivePendingBackgroundWork({ + latestRun, + providerThreads: serverProjection.providerThreads, + turnItems: serverProjection.turnItems, + activeProviderThreadId: serverProjection.thread.activeProviderThreadId, + runs: serverProjection.runs, + }), + ]; + }, [serverProjection]); const activeWorkStartedAt = deriveActiveWorkStartedAt( activeLatestRun, activeRuntime, @@ -5978,6 +5999,7 @@ function ChatViewContent(props: ChatViewProps) { isWorking={isWorking} activeTurnInProgress={isWorking || !latestRunSettled} activeTurnStartedAt={activeWorkStartedAt} + pendingBackgroundTasks={pendingBackgroundTasks} listRef={legendListRef} timelineEntries={timelineEntries} latestRun={activeLatestRun} diff --git a/apps/web/src/components/Sidebar.logic.test.ts b/apps/web/src/components/Sidebar.logic.test.ts index 8e92a5bc7ab..94744d88563 100644 --- a/apps/web/src/components/Sidebar.logic.test.ts +++ b/apps/web/src/components/Sidebar.logic.test.ts @@ -896,6 +896,54 @@ describe("resolveThreadStatusPill", () => { ).toMatchObject({ label: "Working", pulse: true }); }); + it("shows waiting for an idle thread with pending background tasks", () => { + expect( + resolveThreadStatusPill({ + thread: { + ...baseThread, + pendingBackgroundTasks: [{ taskId: "bg-1", description: "sleep 20" }], + runtime: { + ...baseThread.runtime, + status: "idle", + activeRunId: null, + }, + }, + }), + ).toMatchObject({ + label: "Waiting", + colorClass: "text-sidebar-muted-foreground", + dotClass: "bg-sidebar-muted-foreground", + pulse: false, + }); + }); + + it("keeps an active turn working when background tasks are also present", () => { + expect( + resolveThreadStatusPill({ + thread: { + ...baseThread, + pendingBackgroundTasks: [{ taskId: "bg-1", description: "sleep 20" }], + }, + }), + ).toMatchObject({ label: "Working", pulse: true }); + }); + + it("does not show waiting after the background task roster clears", () => { + expect( + resolveThreadStatusPill({ + thread: { + ...baseThread, + pendingBackgroundTasks: [], + runtime: { + ...baseThread.runtime, + status: "idle", + activeRunId: null, + }, + }, + }), + ).toBeNull(); + }); + it("shows plan ready when a settled plan turn has a proposed plan ready for follow-up", () => { expect( resolveThreadStatusPill({ @@ -1018,6 +1066,38 @@ describe("resolveProjectStatusIndicator", () => { ]), ).toMatchObject({ label: "Plan Ready", dotClass: "bg-violet-500" }); }); + + it("ranks waiting below active work and above plan-ready", () => { + const waiting = { + label: "Waiting" as const, + colorClass: "text-sidebar-muted-foreground", + dotClass: "bg-sidebar-muted-foreground", + pulse: false, + }; + + expect( + resolveProjectStatusIndicator([ + waiting, + { + label: "Working", + colorClass: "text-sky-600", + dotClass: "bg-sky-500", + pulse: true, + }, + ]), + ).toMatchObject({ label: "Working" }); + expect( + resolveProjectStatusIndicator([ + { + label: "Plan Ready", + colorClass: "text-violet-600", + dotClass: "bg-violet-500", + pulse: false, + }, + waiting, + ]), + ).toMatchObject({ label: "Waiting" }); + }); }); describe("getVisibleThreadsForProject", () => { diff --git a/apps/web/src/components/Sidebar.logic.ts b/apps/web/src/components/Sidebar.logic.ts index a917631d1a4..fd391dd6715 100644 --- a/apps/web/src/components/Sidebar.logic.ts +++ b/apps/web/src/components/Sidebar.logic.ts @@ -114,6 +114,7 @@ export interface ThreadStatusPill { | "Completed" | "Pending Approval" | "Awaiting Input" + | "Waiting" | "Plan Ready"; colorClass: string; dotClass: string; @@ -125,6 +126,7 @@ const THREAD_STATUS_PRIORITY: Record = { "Awaiting Input": 4, Working: 3, Connecting: 3, + Waiting: 2.5, "Plan Ready": 2, Completed: 1, }; @@ -139,6 +141,7 @@ type ThreadStatusInput = Pick< | "runtime" > & { lastVisitedAt?: string | undefined; + pendingBackgroundTasks?: SidebarThreadSummary["pendingBackgroundTasks"] | undefined; }; export interface ThreadJumpHintVisibilityController { @@ -599,6 +602,15 @@ export function resolveThreadStatusPill(input: { }; } + if ((thread.pendingBackgroundTasks?.length ?? 0) > 0) { + return { + label: "Waiting", + colorClass: "text-sidebar-muted-foreground", + dotClass: "bg-sidebar-muted-foreground", + pulse: false, + }; + } + const hasPlanReadyPrompt = !thread.hasPendingUserInput && thread.interactionMode === "plan" && diff --git a/apps/web/src/components/chat/MessagesTimeline.logic.test.ts b/apps/web/src/components/chat/MessagesTimeline.logic.test.ts index 47a157f9c5f..8ca6205eff3 100644 --- a/apps/web/src/components/chat/MessagesTimeline.logic.test.ts +++ b/apps/web/src/components/chat/MessagesTimeline.logic.test.ts @@ -1436,3 +1436,66 @@ describe("computeStableMessagesTimelineRows", () => { expect(reordered.result).toEqual([initial.result[1], initial.result[0]]); }); }); + +describe("deriveMessagesTimelineRows waiting-background", () => { + it("renders the waiting-background row instead of Working for a parked waiting runtime", () => { + const rows = deriveMessagesTimelineRows({ + timelineEntries: [], + isWorking: false, + activeTurnStartedAt: null, + pendingBackgroundTasks: [{ taskId: "bg-1", description: "Run Codex review" }], + turnDiffSummaryByAssistantMessageId: new Map(), + revertTurnCountByUserMessageId: new Map(), + }); + + expect(rows).toEqual([ + { + kind: "waiting-background", + id: "waiting-background-row", + createdAt: null, + description: "Run Codex review", + taskCount: 1, + label: "Waiting on background task: Run Codex review", + }, + ]); + }); + + it("suppresses a stale waiting roster while an active running turn is Working", () => { + const rows = deriveMessagesTimelineRows({ + timelineEntries: [], + isWorking: true, + activeTurnStartedAt: "2026-01-01T00:00:00Z", + pendingBackgroundTasks: [{ taskId: "bg-1", description: "Run Codex review" }], + turnDiffSummaryByAssistantMessageId: new Map(), + revertTurnCountByUserMessageId: new Map(), + }); + + expect(rows.some((row) => row.kind === "waiting-background")).toBe(false); + expect(rows.some((row) => row.kind === "working")).toBe(true); + }); + + it("includes task count for multiple background tasks", () => { + const rows = deriveMessagesTimelineRows({ + timelineEntries: [], + isWorking: false, + activeTurnStartedAt: null, + pendingBackgroundTasks: [ + { taskId: "bg-1", description: "first" }, + { taskId: "bg-2", description: "second" }, + ], + turnDiffSummaryByAssistantMessageId: new Map(), + revertTurnCountByUserMessageId: new Map(), + }); + + expect(rows).toEqual([ + { + kind: "waiting-background", + id: "waiting-background-row", + createdAt: null, + description: "first", + taskCount: 2, + label: "Waiting on 2 background tasks: first, …", + }, + ]); + }); +}); diff --git a/apps/web/src/components/chat/MessagesTimeline.logic.ts b/apps/web/src/components/chat/MessagesTimeline.logic.ts index a57c402671a..f41f3036c0e 100644 --- a/apps/web/src/components/chat/MessagesTimeline.logic.ts +++ b/apps/web/src/components/chat/MessagesTimeline.logic.ts @@ -13,6 +13,7 @@ import { type RunId, } from "@t3tools/contracts"; import type { ThreadRunSummary } from "@t3tools/client-runtime/state/shell"; +import { formatPendingBackgroundWorkLabel } from "@t3tools/shared/orchestrationV2PendingBackgroundWork"; import { resolveT3McpToolPresentation, type T3McpToolPresentation, @@ -191,7 +192,15 @@ export type MessagesTimelineRow = createdAt: string; proposedPlan: ProposedPlan; } - | { kind: "working"; id: string; createdAt: string | null }; + | { kind: "working"; id: string; createdAt: string | null } + | { + kind: "waiting-background"; + id: string; + createdAt: string | null; + description: string | null; + taskCount: number; + label: string; + }; export interface StableMessagesTimelineRowsState { byId: Map; @@ -489,6 +498,10 @@ export function deriveMessagesTimelineRows(input: { expandedAttemptIds?: ReadonlySet; isWorking: boolean; activeTurnStartedAt: string | null; + pendingBackgroundTasks?: ReadonlyArray<{ + readonly taskId: string; + readonly description?: string | undefined; + }> | null; turnDiffSummaryByAssistantMessageId: ReadonlyMap; revertTurnCountByUserMessageId: ReadonlyMap; }): MessagesTimelineRow[] { @@ -655,6 +668,20 @@ export function deriveMessagesTimelineRows(input: { id: "working-indicator-row", createdAt: input.activeTurnStartedAt, }); + } else if (input.pendingBackgroundTasks && input.pendingBackgroundTasks.length > 0) { + // Root run settled but finite provider background work remains. Show Waiting + // instead of looking fully idle until the roster drains. + const firstTask = input.pendingBackgroundTasks[0]; + nextRows.push({ + kind: "waiting-background", + id: "waiting-background-row", + createdAt: input.activeTurnStartedAt, + description: firstTask?.description ?? null, + taskCount: input.pendingBackgroundTasks.length, + label: + formatPendingBackgroundWorkLabel(input.pendingBackgroundTasks) ?? + "Waiting on a background task", + }); } return nextRows; @@ -688,6 +715,16 @@ function isRowUnchanged(a: MessagesTimelineRow, b: MessagesTimelineRow): boolean case "working": return a.createdAt === (b as typeof a).createdAt; + case "waiting-background": { + const bw = b as typeof a; + return ( + a.createdAt === bw.createdAt && + a.description === bw.description && + a.taskCount === bw.taskCount && + a.label === bw.label + ); + } + case "turn-fold": { const bf = b as typeof a; return a.createdAt === bf.createdAt && a.label === bf.label && a.expanded === bf.expanded; diff --git a/apps/web/src/components/chat/MessagesTimeline.tsx b/apps/web/src/components/chat/MessagesTimeline.tsx index c3b1e14eeaf..2bc50e41852 100644 --- a/apps/web/src/components/chat/MessagesTimeline.tsx +++ b/apps/web/src/components/chat/MessagesTimeline.tsx @@ -180,6 +180,10 @@ interface MessagesTimelineProps { isWorking: boolean; activeTurnInProgress: boolean; activeTurnStartedAt: string | null; + pendingBackgroundTasks?: ReadonlyArray<{ + readonly taskId: string; + readonly description?: string | undefined; + }> | null; listRef: React.RefObject; timelineEntries: ReadonlyArray; latestRun: TimelineLatestRun | null; @@ -227,6 +231,7 @@ export const MessagesTimeline = memo(function MessagesTimeline({ isWorking, activeTurnInProgress, activeTurnStartedAt, + pendingBackgroundTasks = null, listRef, timelineEntries, latestRun, @@ -323,6 +328,7 @@ export const MessagesTimeline = memo(function MessagesTimeline({ expandedAttemptIds, isWorking, activeTurnStartedAt, + pendingBackgroundTasks, turnDiffSummaryByAssistantMessageId, revertTurnCountByUserMessageId, }), @@ -333,6 +339,7 @@ export const MessagesTimeline = memo(function MessagesTimeline({ expandedAttemptIds, isWorking, activeTurnStartedAt, + pendingBackgroundTasks, turnDiffSummaryByAssistantMessageId, revertTurnCountByUserMessageId, ], @@ -910,6 +917,7 @@ const TimelineRowContent = memo(function TimelineRowContent({ row }: { row: Time {row.kind === "proposed-plan" ? : null} {row.kind === "event" ? : null} {row.kind === "working" ? : null} + {row.kind === "waiting-background" ? : null} ); }); @@ -1478,6 +1486,25 @@ function WorkingTimelineRow({ row }: { row: Extract; +}) { + return ( +
+
+ + + + + + {row.label} +
+
+ ); +} + // --------------------------------------------------------------------------- // Self-ticking labels — update their own text nodes so elapsed-time display // does not create a React commit every second while a response is streaming. diff --git a/apps/web/src/hooks/useHandleNewThread.ts b/apps/web/src/hooks/useHandleNewThread.ts index c9a874c0067..28bb22ce8ff 100644 --- a/apps/web/src/hooks/useHandleNewThread.ts +++ b/apps/web/src/hooks/useHandleNewThread.ts @@ -191,7 +191,7 @@ export function useNewThreadHandler() { reusableStoredDraftThread.draftId, { threadId: reusableStoredDraftThread.threadId, - ...(workspaceContext ?? {}), + ...workspaceContext, ...(carryRuntimeMode ? { runtimeMode: carryRuntimeMode } : {}), ...(carryInteractionMode ? { interactionMode: carryInteractionMode } : {}), }, diff --git a/packages/client-runtime/src/state/entities.test.ts b/packages/client-runtime/src/state/entities.test.ts index f187665acc7..729b9aa6478 100644 --- a/packages/client-runtime/src/state/entities.test.ts +++ b/packages/client-runtime/src/state/entities.test.ts @@ -68,6 +68,99 @@ describe("V2 client presentation", () => { expect(shell.source).toBe(v2ThreadShell); }); + it("parks presented runtime at idle when a settled shell has pending background tasks", () => { + const runId = RunId.make("run-completed"); + const shell = presentThreadShell(environmentId, { + ...v2ThreadShell, + latestRunId: runId, + activeRunId: null, + status: "completed", + pendingBackgroundTasks: [{ taskId: "bg-1", description: "sleep 20" }], + }); + + expect(shell.latestRun).toMatchObject({ runId, status: "completed" }); + expect(shell.runtime).toMatchObject({ + status: "idle", + activeRunId: null, + }); + expect(shell.pendingBackgroundTasks).toEqual([{ taskId: "bg-1", description: "sleep 20" }]); + }); + + it("keeps terminal runtime completed when there are no pending background tasks", () => { + const runId = RunId.make("run-completed"); + const shell = presentThreadShell(environmentId, { + ...v2ThreadShell, + latestRunId: runId, + activeRunId: null, + status: "completed", + pendingBackgroundTasks: [], + }); + + expect(shell.latestRun).toMatchObject({ runId, status: "completed" }); + expect(shell.runtime).toMatchObject({ + status: "completed", + activeRunId: null, + }); + }); + + it("keeps runtime running when there is no background roster", () => { + const runId = RunId.make("run-running"); + const shell = presentThreadShell(environmentId, { + ...v2ThreadShell, + latestRunId: runId, + activeRunId: runId, + status: "running", + pendingBackgroundTasks: [], + }); + + expect(shell.latestRun).toMatchObject({ runId, status: "running" }); + expect(shell.runtime).toMatchObject({ + status: "running", + activeRunId: runId, + }); + }); + + it("parks runtime idle over stale shell running when the roster is nonempty", () => { + const runId = RunId.make("run-stale-running"); + const shell = presentThreadShell(environmentId, { + ...v2ThreadShell, + latestRunId: runId, + activeRunId: runId, + // Stale: server already projected a post-settlement roster, but shell + // status still says running (packaged orchestrator-v2 bug). + status: "running", + pendingBackgroundTasks: [{ taskId: "bg-1", description: "sleep 20" }], + }); + + expect(shell.latestRun).toMatchObject({ runId, status: "running" }); + expect(shell.runtime).toMatchObject({ + status: "idle", + activeRunId: runId, + }); + expect(shell.pendingBackgroundTasks).toEqual([{ taskId: "bg-1", description: "sleep 20" }]); + }); + + it("parks runtime idle over stale checkpoint waiting when the roster is nonempty", () => { + const runId = RunId.make("run-stale-waiting"); + const shell = presentThreadShell(environmentId, { + ...v2ThreadShell, + latestRunId: runId, + activeRunId: runId, + // Stale: checkpoint-oriented waiting masks post-settlement background work. + status: "waiting", + pendingBackgroundTasks: [{ taskId: "bg-2", description: "background bash" }], + }); + + expect(shell.latestRun).toMatchObject({ runId, status: "waiting" }); + expect(shell.runtime).toMatchObject({ + status: "idle", + activeRunId: runId, + }); + expect(shell.pendingBackgroundTasks).toEqual([ + { taskId: "bg-2", description: "background bash" }, + ]); + }); + it("derives execution summaries without wrapping or copying the projection", () => { const runId = RunId.make("run-1"); const now = DateTime.makeUnsafe("2026-06-20T01:00:00.000Z"); @@ -124,6 +217,67 @@ describe("V2 client presentation", () => { }); }); + it("parks waiting runtime for a post-settlement roster without hiding active running work", () => { + const runId = RunId.make("run-background-presentation"); + const now = DateTime.makeUnsafe("2026-06-20T01:00:00.000Z"); + const run = { + id: runId, + threadId: v2Projection.thread.id, + ordinal: 1, + providerInstanceId: v2Projection.thread.providerInstanceId, + modelSelection: v2Projection.thread.modelSelection, + providerThreadId: null, + userMessageId: MessageId.make("message-background-presentation"), + rootNodeId: null, + activeAttemptId: null, + status: "waiting" as const, + requestedAt: now, + startedAt: now, + completedAt: null, + checkpointId: null, + contextHandoffId: null, + }; + const backgroundItem = { + id: TurnItemId.make("item-background-command"), + threadId: v2Projection.thread.id, + runId, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 0, + status: "running" as const, + title: "Background command", + startedAt: now, + completedAt: null, + updatedAt: now, + type: "command_execution" as const, + input: "sleep 20", + }; + + expect( + deriveThreadRuntime({ + ...v2Projection, + runs: [run], + turnItems: [backgroundItem], + }), + ).toMatchObject({ + status: "idle", + activeRunId: runId, + }); + expect( + deriveThreadRuntime({ + ...v2Projection, + runs: [{ ...run, status: "running" as const }], + turnItems: [backgroundItem], + }), + ).toMatchObject({ + status: "running", + activeRunId: runId, + }); + }); + it("joins pending request entities to their native turn-item display data", () => { const now = DateTime.makeUnsafe("2026-06-20T01:00:00.000Z"); const requestId = RuntimeRequestId.make("request-approval"); diff --git a/packages/client-runtime/src/state/models.ts b/packages/client-runtime/src/state/models.ts index d467e310bba..390c4027fa8 100644 --- a/packages/client-runtime/src/state/models.ts +++ b/packages/client-runtime/src/state/models.ts @@ -86,6 +86,9 @@ export interface EnvironmentThreadShell { readonly hasPendingApprovals: boolean; readonly hasPendingUserInput: boolean; readonly hasActionableProposedPlan: boolean; + readonly pendingBackgroundTasks: ReadonlyArray< + NonNullable[number] + >; readonly itemCount: number; readonly visibleItemCount: number; readonly createdAt: string; @@ -117,10 +120,17 @@ function terminalRunStatus(status: OrchestrationV2RunStatus): boolean { ); } +// Park runtime at idle when the post-settlement background roster is nonempty +// so #4415 Waiting (session.idle) can consume CTM runtime. The server only +// derives a nonempty roster when the latest root run is terminal, so a roster +// is the stronger presentation signal even if cached shell status is still +// running or checkpoint-oriented waiting. latestRun keeps the shell status. function shellRuntime(thread: OrchestrationV2ThreadShell): ThreadRuntimeSummary | null { if (thread.latestRunId === null && thread.activeProviderThreadId === null) return null; + const hasPendingBackgroundTasks = (thread.pendingBackgroundTasks?.length ?? 0) > 0; + const status = hasPendingBackgroundTasks ? "idle" : thread.status; return { - status: thread.status, + status, activeRunId: thread.activeRunId, providerInstanceId: thread.providerInstanceId, providerName: null, @@ -176,6 +186,7 @@ export function presentThreadShell( thread.pendingRuntimeRequest.kind !== "auth_refresh", hasPendingUserInput: thread.pendingRuntimeRequest?.kind === "user_input", hasActionableProposedPlan: thread.hasActionableProposedPlan, + pendingBackgroundTasks: thread.pendingBackgroundTasks ?? [], itemCount: thread.itemCount, visibleItemCount: thread.visibleItemCount, createdAt: iso(thread.createdAt), diff --git a/packages/client-runtime/src/state/threadExecution.ts b/packages/client-runtime/src/state/threadExecution.ts index d302a90623b..c706125d528 100644 --- a/packages/client-runtime/src/state/threadExecution.ts +++ b/packages/client-runtime/src/state/threadExecution.ts @@ -1,18 +1,23 @@ import type { OrchestrationV2ThreadProjection } from "@t3tools/contracts"; +import { derivePendingBackgroundWork } from "@t3tools/shared/orchestrationV2PendingBackgroundWork"; import * as DateTime from "effect/DateTime"; import type { ThreadRunSummary, ThreadRuntimeSummary } from "./models.ts"; const ACTIVE_RUN_STATUSES = new Set(["preparing", "queued", "starting", "running", "waiting"]); -export function deriveLatestThreadRun( - projection: OrchestrationV2ThreadProjection, -): ThreadRunSummary | null { - const run = projection.runs.reduce<(typeof projection.runs)[number] | null>( +function latestProjectionRun(projection: OrchestrationV2ThreadProjection) { + return projection.runs.reduce<(typeof projection.runs)[number] | null>( (latest, candidate) => latest === null || candidate.ordinal > latest.ordinal ? candidate : latest, null, ); +} + +export function deriveLatestThreadRun( + projection: OrchestrationV2ThreadProjection, +): ThreadRunSummary | null { + const run = latestProjectionRun(projection); if (run === null) return null; return { runId: run.id, @@ -32,14 +37,23 @@ export function deriveThreadRuntime( projection: OrchestrationV2ThreadProjection, ): ThreadRuntimeSummary | null { const latestRun = deriveLatestThreadRun(projection); + const latestRunProjection = latestProjectionRun(projection); const providerSession = projection.providerSessions.findLast( (session) => session.providerInstanceId === projection.thread.providerInstanceId, ); if (latestRun === null && projection.thread.activeProviderThreadId === null) return null; const activeRunId = projection.runs.findLast((run) => ACTIVE_RUN_STATUSES.has(run.status))?.id ?? null; + const hasPendingBackgroundTasks = + derivePendingBackgroundWork({ + latestRun: latestRunProjection, + providerThreads: projection.providerThreads, + turnItems: projection.turnItems, + activeProviderThreadId: projection.thread.activeProviderThreadId, + runs: projection.runs, + }).length > 0; return { - status: latestRun?.status ?? "idle", + status: hasPendingBackgroundTasks ? "idle" : (latestRun?.status ?? "idle"), activeRunId, providerInstanceId: projection.thread.providerInstanceId, providerName: providerSession?.driver ?? null, diff --git a/packages/contracts/src/orchestrationV2.test.ts b/packages/contracts/src/orchestrationV2.test.ts index 8c4ed1d6495..9e08c8e4fa8 100644 --- a/packages/contracts/src/orchestrationV2.test.ts +++ b/packages/contracts/src/orchestrationV2.test.ts @@ -24,9 +24,12 @@ import { OrchestrationV2CheckpointScope, OrchestrationV2Command, OrchestrationV2DomainEvent, + OrchestrationV2ProviderThread, + OrchestrationV2ProviderThreadJson, OrchestrationV2ShellSnapshot, OrchestrationV2Subagent, OrchestrationV2ThreadProjection, + OrchestrationV2ThreadShell, OrchestrationV2TurnItem, } from "./orchestrationV2.ts"; @@ -41,6 +44,11 @@ const LegacyShellStreamItem = Schema.Union([ const decodeLegacyShellStreamItem = Schema.decodeUnknownSync(LegacyShellStreamItem); const decodeOrchestrationV2Command = Schema.decodeUnknownSync(OrchestrationV2Command); const decodeOrchestrationV2TurnItem = Schema.decodeUnknownSync(OrchestrationV2TurnItem); +const decodeOrchestrationV2ProviderThreadJson = Schema.decodeUnknownSync( + OrchestrationV2ProviderThreadJson, +); +const decodeOrchestrationV2ProviderThread = Schema.decodeUnknownSync(OrchestrationV2ProviderThread); +const decodeOrchestrationV2ThreadShell = Schema.decodeUnknownSync(OrchestrationV2ThreadShell); describe("orchestration V2 contracts", () => { it("lets legacy snapshot decoders ignore enrichment metadata", () => { @@ -593,4 +601,92 @@ describe("orchestration V2 contracts", () => { expect(CheckpointRef.make("git-ref-1")).toBe("git-ref-1"); expect(ContextTransferId.make("context-transfer-1")).toBe("context-transfer-1"); }); + + it("decodes historical provider-thread JSON without pendingBackgroundTasks as empty roster", () => { + const providerThread = decodeOrchestrationV2ProviderThreadJson({ + id: "provider-thread-1", + driver: "claude", + providerInstanceId: "claudeAgent", + providerSessionId: "provider-session-1", + appThreadId: "thread-1", + ownerNodeId: null, + nativeThreadRef: { + driver: "claude", + nativeId: "native-session-1", + strength: "strong", + }, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: 1, + lastRunOrdinal: 1, + handoffIds: [], + forkedFrom: null, + createdAt: "2026-04-20T00:00:00.000Z", + updatedAt: "2026-04-20T00:00:00.000Z", + }); + + expect(providerThread.pendingBackgroundTasks).toEqual([]); + + const runtimeThread = decodeOrchestrationV2ProviderThread({ + id: "provider-thread-2", + driver: "claude", + providerInstanceId: "claudeAgent", + providerSessionId: null, + appThreadId: "thread-2", + ownerNodeId: null, + nativeThreadRef: null, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: null, + lastRunOrdinal: null, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }); + expect(runtimeThread.pendingBackgroundTasks).toEqual([]); + }); + + it("decodes historical thread shell JSON without pendingBackgroundTasks as empty roster", () => { + const shell = decodeOrchestrationV2ThreadShell({ + createdBy: "user", + creationSource: "web", + id: "thread-1", + projectId: "project-1", + title: "Thread", + providerInstanceId: "claudeAgent", + modelSelection: { + instanceId: ProviderInstanceId.make("claudeAgent"), + model: "claude-sonnet", + }, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + lineage: { + parentThreadId: null, + relationshipToParent: null, + rootThreadId: "thread-1", + }, + forkedFrom: null, + activeProviderThreadId: "provider-thread-1", + latestRunId: "run-1", + activeRunId: null, + status: "completed", + pendingRuntimeRequest: null, + latestVisibleMessage: null, + latestUserMessageAt: null, + hasActionableProposedPlan: false, + itemCount: 0, + visibleItemCount: 0, + createdAt: now, + updatedAt: now, + archivedAt: null, + settledOverride: null, + settledAt: null, + deletedAt: null, + }); + + expect(shell.pendingBackgroundTasks).toEqual([]); + }); }); diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 186e0b36fb0..6311c4a775f 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -496,6 +496,18 @@ export const OrchestrationV2ProviderSessionDetached = Schema.Struct({ export type OrchestrationV2ProviderSessionDetached = typeof OrchestrationV2ProviderSessionDetached.Type; +/** + * Provider-owned background work that can outlive the root turn (for example a + * Claude background Bash task). Associated with the provider thread so shared + * runtimes cannot make an unrelated app thread look busy. + */ +export const OrchestrationV2PendingBackgroundTask = Schema.Struct({ + taskId: TrimmedNonEmptyString, + description: Schema.optional(TrimmedNonEmptyString), + taskType: Schema.optional(TrimmedNonEmptyString), +}); +export type OrchestrationV2PendingBackgroundTask = typeof OrchestrationV2PendingBackgroundTask.Type; + export const OrchestrationV2ProviderThread = Schema.Struct({ id: ProviderThreadId, driver: ProviderDriverKind, @@ -516,6 +528,10 @@ export const OrchestrationV2ProviderThread = Schema.Struct({ checkpointId: Schema.optional(CheckpointId), }), ), + // Optional Type so adapters can omit empty rosters; historical JSON decodes to []. + pendingBackgroundTasks: Schema.optional(Schema.Array(OrchestrationV2PendingBackgroundTask)).pipe( + Schema.withDecodingDefault(Effect.succeed([])), + ), createdAt: Schema.DateTimeUtc, updatedAt: Schema.DateTimeUtc, }); @@ -1171,6 +1187,11 @@ export const OrchestrationV2ThreadShell = Schema.Struct({ latestVisibleMessage: Schema.NullOr(OrchestrationV2LatestVisibleMessageSummary), latestUserMessageAt: Schema.NullOr(Schema.DateTimeUtc), hasActionableProposedPlan: Schema.Boolean, + // Normalized post-settlement background work for sidebar Waiting pills. + // Empty when the latest root run is still active or no pending work remains. + pendingBackgroundTasks: Schema.optional(Schema.Array(OrchestrationV2PendingBackgroundTask)).pipe( + Schema.withDecodingDefault(Effect.succeed([])), + ), itemCount: NonNegativeInt, visibleItemCount: NonNegativeInt, createdAt: Schema.DateTimeUtc, diff --git a/packages/shared/package.json b/packages/shared/package.json index 06c9681fcf8..9408fc626ee 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -107,6 +107,10 @@ "types": "./src/orchestrationV2Timeline.ts", "import": "./src/orchestrationV2Timeline.ts" }, + "./orchestrationV2PendingBackgroundWork": { + "types": "./src/orchestrationV2PendingBackgroundWork.ts", + "import": "./src/orchestrationV2PendingBackgroundWork.ts" + }, "./remote": { "types": "./src/remote.ts", "import": "./src/remote.ts" diff --git a/packages/shared/src/orchestrationV2PendingBackgroundWork.test.ts b/packages/shared/src/orchestrationV2PendingBackgroundWork.test.ts new file mode 100644 index 00000000000..89d2564d02b --- /dev/null +++ b/packages/shared/src/orchestrationV2PendingBackgroundWork.test.ts @@ -0,0 +1,349 @@ +import { describe, expect, it } from "vite-plus/test"; +import { + derivePendingBackgroundWork, + formatPendingBackgroundWorkLabel, +} from "./orchestrationV2PendingBackgroundWork.ts"; + +describe("derivePendingBackgroundWork", () => { + it("returns empty while the latest run is not settled", () => { + const tasks = derivePendingBackgroundWork({ + latestRun: { id: "run-1" as never, ordinal: 1, status: "running" }, + providerThreads: [ + { + id: "pt-1" as never, + pendingBackgroundTasks: [{ taskId: "bg-1", description: "sleep 20" }], + }, + ], + turnItems: [ + { + id: "item-1" as never, + type: "command_execution", + status: "running", + title: "npm test", + nativeItemRef: null, + input: "npm test", + }, + ], + }); + expect(tasks).toEqual([]); + }); + + it("returns pending work when the latest run is waiting (post-success, pre-checkpoint)", () => { + const tasks = derivePendingBackgroundWork({ + latestRun: { id: "run-1" as never, ordinal: 1, status: "waiting" }, + providerThreads: [{ id: "pt-1" as never }], + turnItems: [ + { + id: "item-1" as never, + type: "command_execution", + status: "running", + title: "npm test", + nativeItemRef: { nativeId: "cmd-1" }, + input: "npm test", + }, + ], + }); + expect(tasks).toEqual([ + { taskId: "cmd-1", description: "npm test", taskType: "command_execution" }, + ]); + }); + + it("still returns empty when the latest run is running even with background items", () => { + const tasks = derivePendingBackgroundWork({ + latestRun: { id: "run-1" as never, ordinal: 1, status: "running" }, + providerThreads: [ + { + id: "pt-1" as never, + pendingBackgroundTasks: [{ taskId: "bg-1", description: "sleep 20" }], + }, + ], + turnItems: [ + { + id: "item-1" as never, + type: "subagent", + status: "running", + title: "review", + nativeItemRef: { nativeId: "sub-1" }, + prompt: "review", + }, + ], + }); + expect(tasks).toEqual([]); + }); + + it("excludes rolled_back items when the latest run is waiting", () => { + const tasks = derivePendingBackgroundWork({ + latestRun: { id: "run-2" as never, ordinal: 2, status: "waiting" }, + providerThreads: [{ id: "pt-1" as never }], + runs: [ + { id: "run-1" as never, ordinal: 1, status: "rolled_back" }, + { id: "run-2" as never, ordinal: 2, status: "waiting" }, + ], + turnItems: [ + { + id: "item-old" as never, + type: "command_execution", + status: "running", + title: "from rolled-back run", + runId: "run-1", + nativeItemRef: { nativeId: "cmd-old" }, + input: "sleep 99", + }, + { + id: "item-new" as never, + type: "command_execution", + status: "running", + title: "still pending", + runId: "run-2", + nativeItemRef: { nativeId: "cmd-new" }, + input: "npm test", + }, + ], + }); + expect(tasks).toEqual([ + { taskId: "cmd-new", description: "still pending", taskType: "command_execution" }, + ]); + }); + + it("returns the provider-thread roster after settlement", () => { + const tasks = derivePendingBackgroundWork({ + latestRun: { id: "run-1" as never, ordinal: 1, status: "completed" }, + providerThreads: [ + { + id: "pt-1" as never, + pendingBackgroundTasks: [ + { taskId: "bg-1", description: "Run Codex review", taskType: "local_bash" }, + ], + }, + ], + turnItems: [], + activeProviderThreadId: "pt-1", + }); + expect(tasks).toEqual([ + { taskId: "bg-1", description: "Run Codex review", taskType: "local_bash" }, + ]); + }); + + it("includes nonterminal turn items and excludes completed ones", () => { + const tasks = derivePendingBackgroundWork({ + latestRun: { id: "run-1" as never, ordinal: 1, status: "completed" }, + providerThreads: [{ id: "pt-1" as never }], + turnItems: [ + { + id: "item-1" as never, + type: "command_execution", + status: "running", + title: "npm test", + nativeItemRef: { nativeId: "cmd-1" }, + input: "npm test", + }, + { + id: "item-2" as never, + type: "command_execution", + status: "completed", + title: "done", + nativeItemRef: { nativeId: "cmd-2" }, + input: "echo done", + }, + ], + }); + expect(tasks).toEqual([ + { taskId: "cmd-1", description: "npm test", taskType: "command_execution" }, + ]); + }); + + it("dedupes roster entries against turn items by native task id", () => { + const tasks = derivePendingBackgroundWork({ + latestRun: { id: "run-1" as never, ordinal: 1, status: "completed" }, + providerThreads: [ + { + id: "pt-1" as never, + pendingBackgroundTasks: [{ taskId: "task-9", description: "Agent review" }], + }, + ], + turnItems: [ + { + id: "item-sub" as never, + type: "subagent", + status: "running", + title: "Agent review", + nativeItemRef: { nativeId: "task-9" }, + prompt: "review the plan", + }, + ], + }); + expect(tasks).toEqual([{ taskId: "task-9", description: "Agent review" }]); + }); + + it("excludes Grok persistent monitors", () => { + const tasks = derivePendingBackgroundWork({ + latestRun: { id: "run-1" as never, ordinal: 1, status: "completed" }, + providerThreads: [{ id: "pt-1" as never }], + turnItems: [ + { + id: "item-1" as never, + type: "dynamic_tool", + status: "running", + title: "monitor logs", + nativeItemRef: { nativeId: "mon-1" }, + input: { persistent: true, command: "tail -f" }, + }, + { + id: "item-2" as never, + type: "dynamic_tool", + status: "running", + title: "finite monitor", + nativeItemRef: { nativeId: "mon-2" }, + input: { persistent: false, command: "sleep 5" }, + }, + ], + }); + expect(tasks).toEqual([ + { taskId: "mon-2", description: "finite monitor", taskType: "dynamic_tool" }, + ]); + }); + + it("returns multiple tasks with stable ordering from insertion", () => { + const tasks = derivePendingBackgroundWork({ + latestRun: { id: "run-1" as never, ordinal: 1, status: "completed" }, + providerThreads: [ + { + id: "pt-1" as never, + pendingBackgroundTasks: [ + { taskId: "bg-1", description: "first" }, + { taskId: "bg-2", description: "second" }, + ], + }, + ], + turnItems: [ + { + id: "item-1" as never, + type: "command_execution", + status: "running", + title: "third", + nativeItemRef: { nativeId: "cmd-3" }, + input: "third", + }, + ], + }); + expect(tasks.map((task) => task.taskId)).toEqual(["bg-1", "bg-2", "cmd-3"]); + }); + + it("excludes turn items owned by a rolled_back run", () => { + const tasks = derivePendingBackgroundWork({ + latestRun: { id: "run-1" as never, ordinal: 1, status: "rolled_back" }, + providerThreads: [{ id: "pt-1" as never }], + runs: [{ id: "run-1" as never, ordinal: 1, status: "rolled_back" }], + turnItems: [ + { + id: "item-1" as never, + type: "command_execution", + status: "running", + title: "abandoned", + runId: "run-1", + nativeItemRef: { nativeId: "cmd-1" }, + input: "sleep 99", + }, + ], + }); + expect(tasks).toEqual([]); + }); + + it("returns empty when the latest run is rolled_back even with a nonempty roster", () => { + const tasks = derivePendingBackgroundWork({ + latestRun: { id: "run-1" as never, ordinal: 1, status: "rolled_back" }, + providerThreads: [ + { + id: "pt-1" as never, + pendingBackgroundTasks: [{ taskId: "bg-1", description: "sleep 20" }], + }, + ], + turnItems: [ + { + id: "item-1" as never, + type: "command_execution", + status: "running", + title: "abandoned", + runId: "run-1", + nativeItemRef: { nativeId: "cmd-1" }, + input: "sleep 99", + }, + ], + }); + expect(tasks).toEqual([]); + }); + + it("excludes an older rolled_back nonterminal item when the latest run is completed", () => { + const tasks = derivePendingBackgroundWork({ + latestRun: { id: "run-2" as never, ordinal: 2, status: "completed" }, + providerThreads: [{ id: "pt-1" as never }], + runs: [ + { id: "run-1" as never, ordinal: 1, status: "rolled_back" }, + { id: "run-2" as never, ordinal: 2, status: "completed" }, + ], + turnItems: [ + { + id: "item-old" as never, + type: "command_execution", + status: "running", + title: "from rolled-back run", + runId: "run-1", + nativeItemRef: { nativeId: "cmd-old" }, + input: "sleep 99", + }, + { + id: "item-new" as never, + type: "command_execution", + status: "running", + title: "still pending", + runId: "run-2", + nativeItemRef: { nativeId: "cmd-new" }, + input: "npm test", + }, + ], + }); + expect(tasks).toEqual([ + { taskId: "cmd-new", description: "still pending", taskType: "command_execution" }, + ]); + }); + + it("includes items with a null run id even when runs list has rolled_back rows", () => { + const tasks = derivePendingBackgroundWork({ + latestRun: { id: "run-1" as never, ordinal: 1, status: "completed" }, + providerThreads: [{ id: "pt-1" as never }], + runs: [{ id: "run-1" as never, ordinal: 1, status: "rolled_back" }], + turnItems: [ + { + id: "item-null-run" as never, + type: "command_execution", + status: "running", + title: "orphan item", + runId: null, + nativeItemRef: { nativeId: "cmd-null" }, + input: "echo orphan", + }, + ], + }); + expect(tasks).toEqual([ + { taskId: "cmd-null", description: "orphan item", taskType: "command_execution" }, + ]); + }); +}); + +describe("formatPendingBackgroundWorkLabel", () => { + it("formats single and multi-task labels", () => { + expect(formatPendingBackgroundWorkLabel([])).toBeNull(); + expect(formatPendingBackgroundWorkLabel([{ taskId: "a" }])).toBe( + "Waiting on a background task", + ); + expect( + formatPendingBackgroundWorkLabel([{ taskId: "a", description: "Run Codex review" }]), + ).toBe("Waiting on background task: Run Codex review"); + expect( + formatPendingBackgroundWorkLabel([ + { taskId: "a", description: "first" }, + { taskId: "b", description: "second" }, + ]), + ).toBe("Waiting on 2 background tasks: first, …"); + }); +}); diff --git a/packages/shared/src/orchestrationV2PendingBackgroundWork.ts b/packages/shared/src/orchestrationV2PendingBackgroundWork.ts new file mode 100644 index 00000000000..60a7a292535 --- /dev/null +++ b/packages/shared/src/orchestrationV2PendingBackgroundWork.ts @@ -0,0 +1,226 @@ +import type { + OrchestrationV2PendingBackgroundTask, + OrchestrationV2ProviderThread, + OrchestrationV2Run, + OrchestrationV2TurnItem, +} from "@t3tools/contracts"; + +const BACKGROUND_TURN_ITEM_TYPES = new Set([ + "command_execution", + "dynamic_tool", + "subagent", +]); + +const TERMINAL_TURN_ITEM_STATUSES = new Set([ + "completed", + "interrupted", + "failed", + "cancelled", +]); + +/** + * True terminal run statuses (includes `rolled_back`). Retained as the + * canonical terminal set for this module; do not widen or reuse it as the + * background-wait gate — that gate intentionally excludes `rolled_back`. + */ +const TERMINAL_RUN_STATUSES = new Set([ + "cancelled", + "completed", + "failed", + "interrupted", + "rolled_back", +]); + +/** + * Run statuses that allow a pending-background roster to surface. + * Includes `waiting`: a successful turn persists as waiting until checkpoint + * capture flips it to completed, and waiting is only set from completed. + * Excludes `rolled_back`: the provider-thread roster is not run-tied, so a + * rolled-back latest run must fail the gate entirely (not only item filter). + * Built explicitly rather than spreading or deleting from TERMINAL_RUN_STATUSES. + */ +const SETTLED_FOR_BACKGROUND_WAIT_RUN_STATUSES = new Set([ + "cancelled", + "completed", + "failed", + "interrupted", + "waiting", +]); + +// Keep TERMINAL_RUN_STATUSES referenced so the true-terminal set stays defined +// next to the background-wait subset (rolled_back is terminal, not wait-settled). +void TERMINAL_RUN_STATUSES; + +export type PendingBackgroundWorkTask = OrchestrationV2PendingBackgroundTask; + +type PendingBackgroundWorkRun = Pick; + +type PendingBackgroundWorkProviderThread = Pick< + OrchestrationV2ProviderThread, + "id" | "pendingBackgroundTasks" +>; + +type PendingBackgroundWorkTurnItem = { + readonly id: OrchestrationV2TurnItem["id"] | string; + readonly type: OrchestrationV2TurnItem["type"]; + readonly status: OrchestrationV2TurnItem["status"]; + readonly title: string | null; + /** When present and the run is rolled_back, the item is abandoned, not pending. */ + readonly runId?: OrchestrationV2Run["id"] | string | null; + readonly nativeItemRef?: { + readonly nativeId: string | null; + } | null; + readonly input?: unknown; + readonly prompt?: string | undefined; +}; + +function isTerminalTurnItemStatus(status: OrchestrationV2TurnItem["status"]): boolean { + return TERMINAL_TURN_ITEM_STATUSES.has(status); +} + +function isLatestRunSettledForBackgroundWait( + latestRun: PendingBackgroundWorkRun | null | undefined, +): boolean { + if (latestRun === undefined || latestRun === null) { + return false; + } + return SETTLED_FOR_BACKGROUND_WAIT_RUN_STATUSES.has(latestRun.status); +} + +function isPersistentDynamicToolInput(input: unknown): boolean { + if (input === null || typeof input !== "object" || Array.isArray(input)) { + return false; + } + return Reflect.get(input, "persistent") === true; +} + +function descriptionFromTurnItem(item: PendingBackgroundWorkTurnItem): string | undefined { + if (typeof item.title === "string" && item.title.trim().length > 0) { + return item.title; + } + if (item.type === "command_execution" && typeof item.input === "string") { + const command = item.input.trim(); + return command.length > 0 ? command : undefined; + } + if (item.type === "dynamic_tool") { + const toolName = Reflect.get(item, "toolName"); + if (typeof toolName === "string" && toolName.trim().length > 0) { + return toolName; + } + } + if (item.type === "subagent" && typeof item.prompt === "string") { + const prompt = item.prompt.trim(); + return prompt.length > 0 ? prompt : undefined; + } + return undefined; +} + +function nativeTaskIdFromTurnItem(item: PendingBackgroundWorkTurnItem): string { + const nativeId = item.nativeItemRef?.nativeId; + if (typeof nativeId === "string" && nativeId.length > 0) { + return nativeId; + } + return String(item.id); +} + +/** + * Derive one normalized pending-background-work list for post-settlement UI. + * + * Sources: + * - Provider-thread roster (Claude SDK background tasks) + * - Nonterminal command_execution / dynamic_tool / subagent turn items + * + * Gated on latest root run settlement. Dedupes by native task ID. Excludes + * Grok persistent monitors (`dynamic_tool` input with `persistent: true`). + * Excludes turn items whose run resolves to `rolled_back` (abandoned work); + * items with a null or absent run id stay eligible (matches SQL shell path). + * Does not consult subagent entities (those double-count turn items). + */ +export function derivePendingBackgroundWork(input: { + readonly latestRun: PendingBackgroundWorkRun | null | undefined; + readonly providerThreads: ReadonlyArray; + readonly turnItems: ReadonlyArray; + readonly activeProviderThreadId?: string | null; + /** + * Run rows used to exclude items owned by rolled_back runs. Optional for + * callers that already filtered (SQL shell path); in-memory callers should + * pass projection runs so policy cannot drift. + */ + readonly runs?: ReadonlyArray; +}): ReadonlyArray { + if (!isLatestRunSettledForBackgroundWait(input.latestRun)) { + return []; + } + + const byTaskId = new Map(); + const rolledBackRunIds = new Set( + (input.runs ?? []).filter((run) => run.status === "rolled_back").map((run) => String(run.id)), + ); + + const providerThreads = + input.activeProviderThreadId === undefined || input.activeProviderThreadId === null + ? input.providerThreads + : input.providerThreads.filter((thread) => thread.id === input.activeProviderThreadId); + + for (const providerThread of providerThreads) { + for (const task of providerThread.pendingBackgroundTasks ?? []) { + if (task.taskId.length === 0 || byTaskId.has(task.taskId)) { + continue; + } + byTaskId.set(task.taskId, { + taskId: task.taskId, + ...(task.description === undefined ? {} : { description: task.description }), + ...(task.taskType === undefined ? {} : { taskType: task.taskType }), + }); + } + } + + for (const item of input.turnItems) { + if (!BACKGROUND_TURN_ITEM_TYPES.has(item.type)) { + continue; + } + if (isTerminalTurnItemStatus(item.status)) { + continue; + } + if (item.type === "dynamic_tool" && isPersistentDynamicToolInput(item.input)) { + continue; + } + // Null/absent run id stays eligible; only known rolled_back runs drop. + const itemRunId = item.runId; + if (itemRunId !== undefined && itemRunId !== null && rolledBackRunIds.has(String(itemRunId))) { + continue; + } + + const taskId = nativeTaskIdFromTurnItem(item); + if (byTaskId.has(taskId)) { + continue; + } + + const description = descriptionFromTurnItem(item); + byTaskId.set(taskId, { + taskId, + ...(description === undefined ? {} : { description }), + taskType: item.type, + }); + } + + return Array.from(byTaskId.values()); +} + +export function formatPendingBackgroundWorkLabel( + tasks: ReadonlyArray, +): string | null { + if (tasks.length === 0) { + return null; + } + const firstDescription = tasks[0]?.description?.trim(); + if (tasks.length === 1) { + return firstDescription && firstDescription.length > 0 + ? `Waiting on background task: ${firstDescription}` + : "Waiting on a background task"; + } + if (firstDescription && firstDescription.length > 0) { + return `Waiting on ${tasks.length} background tasks: ${firstDescription}, …`; + } + return `Waiting on ${tasks.length} background tasks`; +} From 58b6f09bf474ae1f1ab2e7053476f3acd006c231 Mon Sep 17 00:00:00 2001 From: Mike Olson Date: Sat, 25 Jul 2026 16:17:34 -0400 Subject: [PATCH 2/2] fix(orchestrator): Keep agent output attached after a mid-turn steer Steering a Claude turn while a tool was running settled the run while the agent was still working. The reply to the steer then had no live turn to attach to, so it was buffered and, with nothing asking for a continuation, silently dropped: the thread went quiet and returned to Ready with the steer unanswered. Two independent defects produced that. The steer guard keyed on one exact string. `isClaudeActiveSteeringAbortResult` matched only `terminal_reason: "aborted_streaming"`, which the CLI reports when a steer interrupts a streaming assistant message. A steer that lands while a tool is running is queued instead and delivered when the tool result arrives, and the CLI ends that native turn with `aborted_tools`, answering the steer in the next one. That result did not match, so the turn finalized while the CLI kept working. A newly recorded fixture captures the shape from a real session: `subtype: success`, `terminal_reason: "aborted_tools"`, empty text, followed by a fresh native turn carrying the answer. `isClaudeSteeringHandoffResult` replaces it. While a steer is outstanding, a result hands off when the CLI cut the turn short (`aborted_streaming` or `aborted_tools`, including on the error result that is how the first of those arrives) or when the turn ended cleanly with nothing to say. A result carrying text has answered something and still terminalizes, so a steer the CLI absorbs into the running turn does not hold the run open; `max_turns`, `background_requested`, `tool_deferred` and the hook reasons are real terminals even as an empty success. Each accepted steer swallows exactly one result, so the turn still ends on the result that answers it, and the handoff is decided before anything else reads the result so it cannot seed the turn's fallback assistant text. Because a handoff makes the turn depend on a native turn that has not opened yet, it is bounded: if no frame reaches the turn within 60 seconds it settles as a failure rather than leaving the run running for the rest of the session. A failure, not a quiet completion, because an accepted steer that was never answered is exactly the silence this change exists to stop. Buffered assistant text could also be stranded. When a run terminalizes early while the query is still live, later frames go to the wake buffer, which only requested a continuation for a tracked task notification, a running subagent, or a result. Ordinary post-settle assistant text asked for nothing, so it sat in the buffer until the session recycled. Assistant text now requests a continuation on its own. Tool_use and tool_result frames stay gated: they are the model working rather than speaking, and the notification that follows them carries the wake detail. Also record replay `stream_event` frames, which every query receives since `includePartialMessages` landed, and allow a fixture to hold its steer until the target run reports a given turn item so a mid-tool steer can be replayed deterministically. --- .../record-claude-agent-sdk-replay-fixture.ts | 11 +- .../Adapters/ClaudeAdapterV2.test.ts | 477 ++++++++++++++++++ .../Adapters/ClaudeAdapterV2.testkit.ts | 22 +- .../Adapters/ClaudeAdapterV2.ts | 193 ++++++- .../testkit/fixtures/index.ts | 17 + .../claude_output.ts | 43 ++ .../claude_transcript.ndjson | 50 ++ .../message_steering_mid_tool/input.ts | 19 + .../testkit/fixtures/shared.ts | 16 + 9 files changed, 831 insertions(+), 17 deletions(-) create mode 100644 apps/server/src/orchestration-v2/testkit/fixtures/message_steering_mid_tool/claude_output.ts create mode 100644 apps/server/src/orchestration-v2/testkit/fixtures/message_steering_mid_tool/claude_transcript.ndjson create mode 100644 apps/server/src/orchestration-v2/testkit/fixtures/message_steering_mid_tool/input.ts diff --git a/apps/server/scripts/record-claude-agent-sdk-replay-fixture.ts b/apps/server/scripts/record-claude-agent-sdk-replay-fixture.ts index 77f9e88d2a4..0ee3ec452c6 100644 --- a/apps/server/scripts/record-claude-agent-sdk-replay-fixture.ts +++ b/apps/server/scripts/record-claude-agent-sdk-replay-fixture.ts @@ -18,6 +18,7 @@ import { makeCheckpointWorkspace } from "../src/orchestration-v2/testkit/ReplayF import { CLAUDE_MODEL_SELECTION } from "../src/orchestration-v2/testkit/fixtures/shared.ts"; import { MESSAGE_STEERING_INITIAL_PROMPT, + MESSAGE_STEERING_MID_TOOL_PROMPT, MULTI_TURN_FIRST_PROMPT, MESSAGE_STEERING_STEER_PROMPT, READ_ONLY_NEVER_POLICY, @@ -91,6 +92,12 @@ const CLAUDE_RECORDINGS = { queryMode: "active_steering", enableTools: true, }, + message_steering_mid_tool: { + prompts: [MESSAGE_STEERING_MID_TOOL_PROMPT, MESSAGE_STEERING_STEER_PROMPT], + defaultTranscriptFile: "fixtures/message_steering_mid_tool/claude_transcript.ndjson", + queryMode: "active_steering_mid_tool", + enableTools: true, + }, turn_interrupt_mid_tool: { prompts: [TURN_INTERRUPT_MID_TOOL_PROMPT], defaultTranscriptFile: "fixtures/turn_interrupt_mid_tool/claude_transcript.ndjson", @@ -240,6 +247,7 @@ type ClaudeRecordingQueryMode = | "fork_session_merge_back" | "fork_session_merge_back_siblings" | "active_steering" + | "active_steering_mid_tool" | "interrupt" | "interrupt_restart"; @@ -259,13 +267,14 @@ function selectedQueryMode(defaultMode: ClaudeRecordingQueryMode): ClaudeRecordi raw === "fork_session_merge_back" || raw === "fork_session_merge_back_siblings" || raw === "active_steering" || + raw === "active_steering_mid_tool" || raw === "interrupt" || raw === "interrupt_restart" ) { return raw; } throw new Error( - `Unsupported Claude replay query mode '${raw}'. Use 'streaming', 'restart', 'resume_at_cursor', 'fork_session', 'fork_session_prior_turn', 'fork_session_continue', 'fork_session_siblings', 'fork_session_merge_back', 'fork_session_merge_back_siblings', 'active_steering', 'interrupt', or 'interrupt_restart'.`, + `Unsupported Claude replay query mode '${raw}'. Use 'streaming', 'restart', 'resume_at_cursor', 'fork_session', 'fork_session_prior_turn', 'fork_session_continue', 'fork_session_siblings', 'fork_session_merge_back', 'fork_session_merge_back_siblings', 'active_steering', 'active_steering_mid_tool', 'interrupt', or 'interrupt_restart'.`, ); } diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 58e84b251c3..be535755646 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -34,6 +34,7 @@ import * as Queue from "effect/Queue"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; +import { TestClock } from "effect/testing"; import { Tool } from "effect/unstable/ai"; import { attachmentRelativePath } from "../../attachmentStore.ts"; @@ -1145,15 +1146,39 @@ describe("ClaudeAdapterV2 background wake turns", () => { uuid: "00000000-0000-4000-8000-000000000101", session_id: WAKE_NATIVE_SESSION, }); + const makeAssistantTextFrame = (input: { readonly uuid: string; readonly text: string }) => + claudeSdkFrame({ + type: "assistant", + message: { + model: "claude-sonnet-4-6", + id: `msg_${input.uuid}`, + type: "message", + role: "assistant", + content: [{ type: "text", text: input.text }], + stop_reason: null, + stop_sequence: null, + usage: { + input_tokens: 1, + output_tokens: 1, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }, + }, + parent_tool_use_id: null, + uuid: input.uuid, + session_id: WAKE_NATIVE_SESSION, + }); const makeResultFrame = (input: { readonly uuid: string; readonly result: string; readonly numTurns?: number; readonly origin?: { readonly kind: "task-notification" }; + readonly terminalReason?: string; }) => claudeSdkFrame({ type: "result", subtype: "success", + ...(input.terminalReason === undefined ? {} : { terminal_reason: input.terminalReason }), duration_ms: 10, duration_api_ms: 10, is_error: false, @@ -2076,6 +2101,458 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + it.effect("keeps a steered turn alive across the CLI's handoff result", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const idAllocator = yield* IdAllocatorV2; + const now = yield* DateTime.now; + const attemptId = RunAttemptId.make("attempt-claude-steer-handoff"); + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId, + text: "Run the long command, then report.", + attachments: [], + }), + ); + yield* harness.runtime.steerTurn({ + threadId: harness.threadId, + runId: RunId.make("run-claude-steer-handoff"), + providerThread: harness.providerThread, + providerTurnId: idAllocator.derive.providerTurn({ + driver: CLAUDE_PROVIDER, + nativeTurnId: `turn:${attemptId}`, + }), + message: { + createdBy: "user", + creationSource: "web", + messageId: MessageId.make("message-claude-steer-handoff"), + text: "Actually, just report now.", + attachments: [], + }, + }); + yield* awaitUntil(() => harness.offeredMessages.length === 2, "steer offered"); + + // The CLI takes the steer by ending the native turn: mid-tool that is + // terminal_reason "aborted_tools", not "aborted_streaming". The steered + // work has not been answered yet, so the T3 turn must stay alive. + yield* Queue.offer( + harness.sdkMessages, + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000201", + result: "", + terminalReason: "aborted_tools", + }), + ); + let handoffYields = 0; + yield* awaitUntil(() => handoffYields++ >= 50, "handoff result to be consumed"); + assert.lengthOf(harness.terminalEvents(), 0); + assert.lengthOf(harness.continuationRequests, 0); + + yield* Queue.offer( + harness.sdkMessages, + makeAssistantTextFrame({ + uuid: "00000000-0000-4000-8000-000000000202", + text: "Reporting now, as asked.", + }), + ); + yield* Queue.offer( + harness.sdkMessages, + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000203", + result: "Reporting now, as asked.", + terminalReason: "completed", + }), + ); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "steered turn terminal"); + assert.equal(harness.terminalEvents()[0]?.status, "completed"); + // The answer attaches to the run that carries the steer; no + // continuation run is needed to rescue it. + assert.lengthOf(harness.continuationRequests, 0); + assert.isTrue( + harness.events.some( + (event) => + event.type === "message.updated" && event.message.text === "Reporting now, as asked.", + ), + ); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("terminalizes a steered turn on a completed result that answers it", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const idAllocator = yield* IdAllocatorV2; + const now = yield* DateTime.now; + const attemptId = RunAttemptId.make("attempt-claude-steer-absorbed-completed"); + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId, + text: "Run the long command, then report.", + attachments: [], + }), + ); + yield* harness.runtime.steerTurn({ + threadId: harness.threadId, + runId: RunId.make("run-claude-steer-absorbed-completed"), + providerThread: harness.providerThread, + providerTurnId: idAllocator.derive.providerTurn({ + driver: CLAUDE_PROVIDER, + nativeTurnId: `turn:${attemptId}`, + }), + message: { + createdBy: "user", + creationSource: "web", + messageId: MessageId.make("message-claude-steer-absorbed-completed"), + text: "Actually, just report now.", + attachments: [], + }, + }); + yield* awaitUntil(() => harness.offeredMessages.length === 2, "steer offered"); + + // A clean success carrying text has answered something, so it ends the + // turn even with a steer outstanding. Swallowing it would hold the run + // open until the handoff bound expired on every turn that the CLI + // absorbs a steer into. + yield* Queue.offer( + harness.sdkMessages, + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000232", + result: "Reported, including the change you asked for.", + terminalReason: "completed", + }), + ); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "steered turn terminal"); + assert.equal(harness.terminalEvents()[0]?.status, "completed"); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("terminalizes a steered turn on an aborted success that answers it", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const idAllocator = yield* IdAllocatorV2; + const now = yield* DateTime.now; + const attemptId = RunAttemptId.make("attempt-claude-steer-absorbed"); + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId, + text: "Run the long command, then report.", + attachments: [], + }), + ); + yield* harness.runtime.steerTurn({ + threadId: harness.threadId, + runId: RunId.make("run-claude-steer-absorbed"), + providerThread: harness.providerThread, + providerTurnId: idAllocator.derive.providerTurn({ + driver: CLAUDE_PROVIDER, + nativeTurnId: `turn:${attemptId}`, + }), + message: { + createdBy: "user", + creationSource: "web", + messageId: MessageId.make("message-claude-steer-absorbed"), + text: "Actually, just report now.", + attachments: [], + }, + }); + yield* awaitUntil(() => harness.offeredMessages.length === 2, "steer offered"); + + // A clean aborted_tools success carrying text has answered something, + // so it ends the turn even with a steer outstanding. Swallowing it + // would hold the run open until the handoff bound expires. + yield* Queue.offer( + harness.sdkMessages, + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000231", + result: "Reported, including the change you asked for.", + terminalReason: "aborted_tools", + }), + ); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "steered turn terminal"); + assert.equal(harness.terminalEvents()[0]?.status, "completed"); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("terminalizes a steered turn on a non-handoff terminal reason", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const idAllocator = yield* IdAllocatorV2; + const now = yield* DateTime.now; + const attemptId = RunAttemptId.make("attempt-claude-steer-max-turns"); + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId, + text: "Keep going until it is done.", + attachments: [], + }), + ); + yield* harness.runtime.steerTurn({ + threadId: harness.threadId, + runId: RunId.make("run-claude-steer-max-turns"), + providerThread: harness.providerThread, + providerTurnId: idAllocator.derive.providerTurn({ + driver: CLAUDE_PROVIDER, + nativeTurnId: `turn:${attemptId}`, + }), + message: { + createdBy: "user", + creationSource: "web", + messageId: MessageId.make("message-claude-steer-max-turns"), + text: "Actually, stop after this step.", + attachments: [], + }, + }); + yield* awaitUntil(() => harness.offeredMessages.length === 2, "steer offered"); + + // max_turns ends the turn for real even though it arrives as an empty + // success. Only "completed" (or an older CLI that omits the field) + // means the CLI is about to open another native turn. + yield* Queue.offer( + harness.sdkMessages, + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000241", + result: "", + terminalReason: "max_turns", + }), + ); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "steered turn terminal"); + assert.equal(harness.terminalEvents()[0]?.status, "completed"); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("settles a steered turn when the CLI never opens the handoff turn", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const idAllocator = yield* IdAllocatorV2; + const now = yield* DateTime.now; + const attemptId = RunAttemptId.make("attempt-claude-steer-silent"); + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId, + text: "Run the long command, then report.", + attachments: [], + }), + ); + yield* harness.runtime.steerTurn({ + threadId: harness.threadId, + runId: RunId.make("run-claude-steer-silent"), + providerThread: harness.providerThread, + providerTurnId: idAllocator.derive.providerTurn({ + driver: CLAUDE_PROVIDER, + nativeTurnId: `turn:${attemptId}`, + }), + message: { + createdBy: "user", + creationSource: "web", + messageId: MessageId.make("message-claude-steer-silent"), + text: "Actually, just report now.", + attachments: [], + }, + }); + yield* awaitUntil(() => harness.offeredMessages.length === 2, "steer offered"); + yield* Queue.offer( + harness.sdkMessages, + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000211", + result: "", + terminalReason: "aborted_tools", + }), + ); + let handoffYields = 0; + yield* awaitUntil(() => handoffYields++ >= 50, "handoff result to be consumed"); + assert.lengthOf(harness.terminalEvents(), 0); + + // Nothing follows the handoff, so the bound settles the turn rather + // than leaving the run running for the rest of the session. It settles + // as a failure: the steer was accepted and never answered. + yield* TestClock.adjust("60 seconds"); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "bounded handoff terminal"); + assert.equal(harness.terminalEvents()[0]?.status, "failed"); + + // A frame that arrives after the bound has claimed the turn must not + // produce a second terminal for it. The bound and the message handler + // run on separate fibers, so the claim inside finalizeActiveTurn is + // what keeps this to one. + yield* Queue.offer( + harness.sdkMessages, + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000212", + result: "Late answer that lost the race.", + terminalReason: "completed", + }), + ); + let lateYields = 0; + yield* awaitUntil(() => lateYields++ >= 50, "late result to be handled"); + assert.lengthOf(harness.terminalEvents(), 1); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("requests a continuation for assistant output stranded after a settle", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + const strandedText = "Output produced after the run had already settled."; + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-stranded-1"), + text: "Do the thing.", + attachments: [], + }), + ); + yield* Queue.offer(harness.sdkMessages, turnOneResult); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "first turn terminal"); + assert.lengthOf(harness.continuationRequests, 0); + + // No background task and no notification: just the model still talking + // after an early terminal left it with no turn to talk into. Without a + // continuation these frames sit in the buffer until the session + // recycles, which is how the output disappears. + yield* Queue.offer( + harness.sdkMessages, + makeAssistantTextFrame({ + uuid: "00000000-0000-4000-8000-000000000221", + text: strandedText, + }), + ); + yield* awaitUntil(() => harness.continuationRequests.length === 1, "continuation request"); + assert.equal(harness.continuationRequests[0]?.threadId, harness.threadId); + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-stranded-2"), + text: "Continuing.", + attachments: [], + providerTurnOrdinal: 2, + messageCreatedBy: "agent", + messageCreationSource: "provider", + }), + ); + yield* awaitUntil( + () => + harness.events.some( + (event) => event.type === "message.updated" && event.message.text === strandedText, + ), + "stranded output projected", + ); + assert.lengthOf(harness.offeredMessages, 1); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("does not continue for post-settle frames that carry no output", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-stranded-quiet"), + text: "Do the thing.", + attachments: [], + }), + ); + yield* Queue.offer(harness.sdkMessages, turnOneResult); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "first turn terminal"); + + // Tool chatter after settle is the model working, not speaking. The + // notification or result that follows it owns the continuation and its + // detail, so these frames must buffer without offering. + yield* Queue.offer( + harness.sdkMessages, + claudeSdkFrame({ + type: "assistant", + message: { + role: "assistant", + content: [ + { type: "tool_use", id: "toolu_quiet", name: "Bash", input: { command: "ls" } }, + ], + }, + parent_tool_use_id: null, + uuid: "00000000-0000-4000-8000-000000000251", + session_id: WAKE_NATIVE_SESSION, + }), + ); + yield* Queue.offer( + harness.sdkMessages, + claudeSdkFrame({ + type: "user", + message: { + role: "user", + content: [{ type: "tool_result", tool_use_id: "toolu_quiet", content: [] }], + }, + parent_tool_use_id: null, + uuid: "00000000-0000-4000-8000-000000000252", + session_id: WAKE_NATIVE_SESSION, + }), + ); + let quietYields = 0; + yield* awaitUntil(() => quietYields++ >= 50, "tool frames to buffer"); + assert.lengthOf(harness.continuationRequests, 0); + + // Two text frames in the same stranded burst still share one request. + yield* Queue.offer( + harness.sdkMessages, + makeAssistantTextFrame({ + uuid: "00000000-0000-4000-8000-000000000253", + text: "First stranded line.", + }), + ); + yield* awaitUntil(() => harness.continuationRequests.length === 1, "continuation request"); + yield* Queue.offer( + harness.sdkMessages, + makeAssistantTextFrame({ + uuid: "00000000-0000-4000-8000-000000000254", + text: "Second stranded line.", + }), + ); + let duplicateYields = 0; + yield* awaitUntil(() => duplicateYields++ >= 50, "second text frame to buffer"); + assert.lengthOf(harness.continuationRequests, 1); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + it.effect("ignores a live task-notification origin result during a normal user turn", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.ts index 9a11565806e..56cf2300a79 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.testkit.ts @@ -295,7 +295,11 @@ function isClaudeSdkReplayMessage(frame: unknown): frame is SDKMessage { type === "user" || type === "result" || type === "system" || - type === "rate_limit_event" + type === "rate_limit_event" || + // The adapter opens every query with includePartialMessages, so the CLI + // interleaves stream_event frames with the message frames. Fixtures + // recorded before that option landed have none. + type === "stream_event" ); } @@ -1261,6 +1265,7 @@ async function recordClaudeActiveSteeringQuery(input: { readonly allowDangerouslySkipPermissions?: boolean; readonly enablePermissionCallback?: boolean; readonly permissionDecision?: ProviderApprovalDecision; + readonly steerAfter?: "tool_use"; }): Promise { if (input.prompts.length < 2) { throw new Error("Claude active steering replay recording requires at least two prompts."); @@ -1350,6 +1355,17 @@ async function recordClaudeActiveSteeringQuery(input: { const iterator = queryRuntime[Symbol.asyncIterator](); try { offerPrompt(0); + if (input.steerAfter === "tool_use") { + // Steer while a tool is still running. The CLI queues the steer instead + // of aborting the stream, so its result carries a terminal reason other + // than aborted_streaming. + await recordMessagesUntilFirstToolUse({ + iterator, + entries: input.entries, + scenario: input.scenario, + }); + await Effect.runPromise(Effect.sleep(Duration.seconds(2))); + } offerSteeringPrompts(); const completed = await recordMessagesUntilTurnResults({ iterator, @@ -2175,6 +2191,7 @@ export async function recordClaudeAgentSdkReplayTranscript(input: { | "fork_session_merge_back" | "fork_session_merge_back_siblings" | "active_steering" + | "active_steering_mid_tool" | "interrupt" | "interrupt_restart"; readonly enableTools?: boolean; @@ -2220,9 +2237,10 @@ export async function recordClaudeAgentSdkReplayTranscript(input: { ? {} : { permissionDecision: input.permissionDecision }), }); - } else if (queryMode === "active_steering") { + } else if (queryMode === "active_steering" || queryMode === "active_steering_mid_tool") { await recordClaudeActiveSteeringQuery({ scenario: input.scenario, + ...(queryMode === "active_steering_mid_tool" ? { steerAfter: "tool_use" as const } : {}), prompts: input.prompts, modelSelection: input.modelSelection, cwd: input.cwd, diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index f27b2b8c116..f4857049de8 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -48,6 +48,7 @@ import * as Context from "effect/Context"; import * as Crypto from "effect/Crypto"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; @@ -1476,6 +1477,11 @@ type ClaudeNativeToolOutput = const NO_CLAUDE_NATIVE_TOOL_OUTPUT = { type: "none" } satisfies ClaudeNativeToolOutput; +// How long a turn stays alive on a steer handoff result before settling anyway. +// The CLI opens the steered native turn within a second or two, so this only +// fires when it never opens one at all. +const CLAUDE_STEERING_HANDOFF_TIMEOUT = Duration.seconds(60); + function claudeNativeToolOutputFromToolResult( toolResult: ClaudeToolResultContentBlock, ): ClaudeNativeToolOutput { @@ -1811,8 +1817,34 @@ function terminalStatusFromResult( return "failed"; } -function isClaudeActiveSteeringAbortResult(message: SDKResultMessage): boolean { - return message.terminal_reason === "aborted_streaming"; +// The CLI ends the native turn to take a steer and answers it in the next one, +// so that first result hands off rather than ending the T3 turn. Recorded +// shapes: mid-stream it aborts the stream and reports `aborted_streaming` on an +// error result, mid-tool it aborts the tools and reports `aborted_tools` on a +// clean success whose text is empty. +// +// A turn that ends normally with nothing to say also counts: it has not +// answered the steer either. Both extra conditions are deliberate. A result +// carrying text has answered something, so it terminalizes the turn rather than +// holding the run open until the handoff bound expires. And `completed` (or an +// older CLI that omits the field) is the only non-abort reason that means "this +// turn ended cleanly"; `max_turns`, `background_requested`, `tool_deferred` and +// the hook reasons are real terminals even when they arrive as a success. +// +// The two aborted reasons hand off unless a clean success carries non-empty +// text. Error subtypes have no result field and stay handoffs, which preserves +// the recorded `aborted_streaming` shape. +function isClaudeSteeringHandoffResult(message: SDKResultMessage): boolean { + if ( + message.terminal_reason === "aborted_streaming" || + message.terminal_reason === "aborted_tools" + ) { + return message.subtype !== "success" || message.is_error || message.result.trim().length === 0; + } + if (message.terminal_reason !== undefined && message.terminal_reason !== "completed") { + return false; + } + return message.subtype === "success" && !message.is_error && message.result.trim().length === 0; } function isClaudeProviderContinuationTurn(input: ProviderAdapterV2TurnInput): boolean { @@ -2036,7 +2068,15 @@ export function makeClaudeAdapterV2( const events = yield* Queue.unbounded(); const activeTurn = yield* Ref.make(null); const interruptedTurns = yield* Ref.make(new Set()); - const steeredTurns = yield* Ref.make(new Set()); + // Steers awaiting a handoff result, counted per provider turn: each + // accepted steer ends one native turn, so each one swallows exactly one + // result before the turn can terminalize again. + const steeredTurns = yield* Ref.make(new Map()); + // Frames seen per provider turn, so a swallowed handoff can tell "the + // steered turn started" from "the CLI went silent". + const turnFrameCounts = yield* Ref.make( + new Map(), + ); const queryContext = yield* Ref.make(null); const openedNativeThreads = yield* Ref.make(new Set()); const itemOrdinals = yield* Ref.make(new Map()); @@ -3205,6 +3245,19 @@ export function makeClaudeAdapterV2( readonly failure?: OrchestrationV2ProviderFailure; readonly threadDisposition?: "reusable" | "broken"; }) { + // Claim the turn before emitting anything. The handoff bound runs on + // its own fiber, so without this a frame handler and the bound can + // both finalize the same turn and emit two terminals for it. + const claimed = yield* Ref.modify(activeTurn, (current) => { + if (current?.providerTurnId !== input.context.providerTurnId) { + return [false, current] as const; + } + return [true, null] as const; + }); + if (!claimed) { + return; + } + for (const toolCall of input.context.toolCalls.values()) { const artifacts = buildToolCallArtifacts({ context: input.context, @@ -3357,14 +3410,72 @@ export function makeClaudeAdapterV2( ], { concurrency: 1 }, ); - yield* Ref.update(activeTurn, (current) => - current?.providerTurnId === input.context.providerTurnId ? null : current, - ); yield* Ref.update(interruptedTurns, (current) => { const next = new Set(current); next.delete(input.context.providerTurnId); return next; }); + yield* Ref.update(steeredTurns, (current) => { + const next = new Map(current); + next.delete(input.context.providerTurnId); + return next; + }); + yield* Ref.update(turnFrameCounts, (current) => { + const next = new Map(current); + next.delete(input.context.providerTurnId); + return next; + }); + }); + + // A steer handoff keeps the T3 turn alive so the steered work still + // attaches to it, which leaves the turn depending on a native turn that + // the CLI has not opened yet. Bound that wait: if no further frame + // reaches this turn, settle it rather than leaving the run running. + const boundSteeringHandoff = Effect.fnUntraced(function* ( + context: ActiveClaudeTurnContext, + ) { + const framesAtHandoff = + (yield* Ref.get(turnFrameCounts)).get(context.providerTurnId) ?? 0; + yield* Effect.gen(function* () { + yield* Effect.sleep(CLAUDE_STEERING_HANDOFF_TIMEOUT); + const current = yield* Ref.get(activeTurn); + if (current?.providerTurnId !== context.providerTurnId) { + return; + } + // Claim the settle in one update, so overlapping bounds from + // several steers on the same turn cannot both finalize it. + const claimed = yield* Ref.modify(turnFrameCounts, (counts) => { + const framesNow = counts.get(context.providerTurnId) ?? 0; + if (framesNow !== framesAtHandoff) { + return [false, counts] as const; + } + const next = new Map(counts); + next.delete(context.providerTurnId); + return [true, next] as const; + }); + if (!claimed) { + return; + } + yield* Effect.logWarning("orchestration-v2.claude-steer-handoff-timeout", { + providerSessionId: input.providerSessionId, + providerThreadId: context.input.providerThread.id, + providerTurnId: context.providerTurnId, + }); + // Settle as a failure, not a quiet completion. The steer was + // accepted and never answered, which is the silence this whole + // path exists to stop the user from having to guess at. + const completedAt = yield* DateTime.now; + yield* finalizeActiveTurn({ + context, + status: "failed", + completedAt, + failure: makeProviderFailure({ + message: "Claude accepted the steer but never opened a turn to answer it.", + code: "steer_handoff_timeout", + class: "transport_error", + }), + }); + }).pipe(Effect.forkIn(sessionScope)); }); const emitAssistantTextArtifacts = Effect.fnUntraced(function* (input: { @@ -3551,9 +3662,22 @@ export function makeClaudeAdapterV2( ) ?? false; const isNativeOpaqueWakeFrame = hasBufferedNotification && (message.type === "assistant" || message.type === "user"); + // Assistant text with no active turn is the other exception: an early + // terminal settled the run while the query kept working, so no + // notification is coming and the next result may be minutes away or + // never arrive. Without an offer here those frames sit in the buffer + // until the session recycles and are never projected. + // + // Only text qualifies. A tool_use or tool_result frame is the model + // working rather than speaking, and the notification or result that + // follows it carries the wake detail; offering on those would open + // the continuation early and strip that detail from it. + const isStrandedNativeOutput = + (assistantTextFromSdkMessage(message)?.text.length ?? 0) > 0; if ( !isPendingSubagentNotification && !isNativeOpaqueWakeFrame && + !isStrandedNativeOutput && message.type !== "result" ) { return; @@ -3708,6 +3832,12 @@ export function makeClaudeAdapterV2( return; } + yield* Ref.update(turnFrameCounts, (current) => { + const updated = new Map(current); + updated.set(context.providerTurnId, (updated.get(context.providerTurnId) ?? 0) + 1); + return updated; + }); + if (message.type === "assistant") { context.nativeMessageCursor = message.uuid; } @@ -3891,6 +4021,27 @@ export function makeClaudeAdapterV2( return; } + // Decide the steer handoff before anything reads this result: a + // handoff belongs to the native turn that was cut short, so it must + // not seed this turn's fallback assistant text either. + if (message.type === "result") { + const interrupted = (yield* Ref.get(interruptedTurns)).has(context.providerTurnId); + const pendingSteers = (yield* Ref.get(steeredTurns)).get(context.providerTurnId) ?? 0; + if (!interrupted && pendingSteers > 0 && isClaudeSteeringHandoffResult(message)) { + yield* Ref.update(steeredTurns, (current) => { + const next = new Map(current); + if (pendingSteers <= 1) { + next.delete(context.providerTurnId); + } else { + next.set(context.providerTurnId, pendingSteers - 1); + } + return next; + }); + yield* boundSteeringHandoff(context); + return; + } + } + // An is_error result's text is the error message; it belongs on the // terminal-failure item, not on a synthetic assistant message. const resultText = @@ -3910,12 +4061,8 @@ export function makeClaudeAdapterV2( if (message.type === "result") { const completedAt = yield* DateTime.now; const interrupted = (yield* Ref.get(interruptedTurns)).has(context.providerTurnId); - const wasSteered = (yield* Ref.get(steeredTurns)).has(context.providerTurnId); - if (!interrupted && wasSteered && isClaudeActiveSteeringAbortResult(message)) { - return; - } yield* Ref.update(steeredTurns, (current) => { - const next = new Set(current); + const next = new Map(current); next.delete(context.providerTurnId); return next; }); @@ -4401,12 +4548,30 @@ export function makeClaudeAdapterV2( attachmentsDir, fileSystem, }); + // Count the steer before offering it, so a handoff result that + // races back cannot terminalize the turn, and roll the count back + // if the offer never reached the CLI. yield* Ref.update(steeredTurns, (current) => { - const next = new Set(current); - next.add(turnInput.providerTurnId); + const next = new Map(current); + next.set(turnInput.providerTurnId, (next.get(turnInput.providerTurnId) ?? 0) + 1); return next; }); - yield* existing.query.offer(userMessage); + yield* existing.query.offer(userMessage).pipe( + Effect.onExit((exit) => + Exit.isSuccess(exit) + ? Effect.void + : Ref.update(steeredTurns, (current) => { + const pending = current.get(turnInput.providerTurnId) ?? 0; + const next = new Map(current); + if (pending <= 1) { + next.delete(turnInput.providerTurnId); + } else { + next.set(turnInput.providerTurnId, pending - 1); + } + return next; + }), + ), + ); }, (effect, turnInput) => effect.pipe( diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/index.ts b/apps/server/src/orchestration-v2/testkit/fixtures/index.ts index 510b85c4c46..784380cebe9 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/index.ts +++ b/apps/server/src/orchestration-v2/testkit/fixtures/index.ts @@ -13,6 +13,8 @@ import { assertMessageSteeringOutput } from "./message_steering/codex_output.ts" import { assertCursorMessageSteeringOutput } from "./message_steering/cursor_output.ts"; import { assertGrokMessageSteeringOutput } from "./message_steering/grok_output.ts"; import { messageSteeringInput } from "./message_steering/input.ts"; +import { assertClaudeMessageSteeringMidToolOutput } from "./message_steering_mid_tool/claude_output.ts"; +import { messageSteeringMidToolInput } from "./message_steering_mid_tool/input.ts"; import { assertMultiTurnClaudeOutput } from "./multi_turn/claude_output.ts"; import { assertMultiTurnOutput } from "./multi_turn/codex_output.ts"; import { multiTurnInput } from "./multi_turn/input.ts"; @@ -626,6 +628,21 @@ export const ORCHESTRATOR_REPLAY_FIXTURES = [ }, ], }, + { + name: "message_steering_mid_tool", + buildInput: messageSteeringMidToolInput, + providers: [ + { + driver: ProviderDriverKind.make("claudeAgent"), + transcriptFile: new URL( + "./message_steering_mid_tool/claude_transcript.ndjson", + import.meta.url, + ), + modelSelection: CLAUDE_MODEL_SELECTION, + assertOutput: assertClaudeMessageSteeringMidToolOutput, + }, + ], + }, { name: "turn_interrupt", buildInput: turnInterruptInput, diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/message_steering_mid_tool/claude_output.ts b/apps/server/src/orchestration-v2/testkit/fixtures/message_steering_mid_tool/claude_output.ts new file mode 100644 index 00000000000..cc8eea559c9 --- /dev/null +++ b/apps/server/src/orchestration-v2/testkit/fixtures/message_steering_mid_tool/claude_output.ts @@ -0,0 +1,43 @@ +import { assert } from "@effect/vitest"; +import type { ProviderReplayTranscript } from "@t3tools/contracts"; + +import type { OrchestratorV2ScenarioResult } from "../../OrchestratorScenario.ts"; +import { + assertAssistantTextIncludes, + assertBaseProjection, + assertSemanticProjectionIntegrity, + assertUserMessageInputIntents, + assertUserMessagesInclude, + assertVisibleTurnItemsMirrorLocalTurnItems, + MESSAGE_STEERING_MID_TOOL_PROMPT, + MESSAGE_STEERING_STEER_PROMPT, + projectionFor, +} from "../shared.ts"; + +export function assertClaudeMessageSteeringMidToolOutput( + result: OrchestratorV2ScenarioResult, + transcript: ProviderReplayTranscript, +) { + assert.equal(transcript.provider, "claudeAgent"); + assertBaseProjection({ result, transcript, runCount: 1, runStatuses: ["completed"] }); + + const projection = projectionFor(result, transcript.scenario); + assertSemanticProjectionIntegrity(projection); + assertVisibleTurnItemsMirrorLocalTurnItems(projection); + assertUserMessagesInclude(projection, [ + MESSAGE_STEERING_MID_TOOL_PROMPT, + MESSAGE_STEERING_STEER_PROMPT, + ]); + assertUserMessageInputIntents(projection, ["turn_start", "steer"]); + // The steer lands while the Bash tool is running, so the CLI queues it and + // ends the native turn with terminal_reason "aborted_tools" before answering. + // That result must not settle the run: the answer arrives afterwards and + // belongs to the run that carries the steer. + assertAssistantTextIncludes(projection, "steering fixture observed"); + assert.equal(projection.runs.length, 1, "a mid-tool steer must not open a second run"); + assert.equal( + projection.providerTurns.length, + 1, + "a mid-tool steer must not create a new provider turn", + ); +} diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/message_steering_mid_tool/claude_transcript.ndjson b/apps/server/src/orchestration-v2/testkit/fixtures/message_steering_mid_tool/claude_transcript.ndjson new file mode 100644 index 00000000000..ad5fe57fc90 --- /dev/null +++ b/apps/server/src/orchestration-v2/testkit/fixtures/message_steering_mid_tool/claude_transcript.ndjson @@ -0,0 +1,50 @@ +{"type":"transcript_start","provider":"claudeAgent","protocol":"claude-agent-sdk.query","version":"0.2.111","scenario":"message_steering_mid_tool","metadata":{"prompts":["Run this exact local command: `node -e \"console.log('steering fixture tool started'); setTimeout(() => {}, 20000)\"`. Do not answer until it completes, then respond exactly: steering fixture initial response","Actually, respond with exactly: steering fixture observed"],"model":"claude-sonnet-4-6","nativeSessionId":"590b808f-f261-44e3-b130-88df68cef005","queryMode":"active_steering_mid_tool","tools":"claude_code","permissionMode":"bypassPermissions","generatedBy":"recordClaudeAgentSdkReplayTranscript"}} +{"type":"expect_outbound","label":"query.open","frame":{"type":"query.open","options":{"model":"claude-sonnet-4-6","tools":{"type":"preset","preset":"claude_code"},"permissionMode":"bypassPermissions","allowDangerouslySkipPermissions":true,"sessionId":"590b808f-f261-44e3-b130-88df68cef005"}}} +{"type":"expect_outbound","label":"prompt.offer:1","frame":{"type":"prompt.offer","message":{"type":"user","message":{"role":"user","content":"Run this exact local command: `node -e \"console.log('steering fixture tool started'); setTimeout(() => {}, 20000)\"`. Do not answer until it completes, then respond exactly: steering fixture initial response"},"parent_tool_use_id":null}}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"init","agents":[],"apiKeySource":"none","claude_code_version":"2.1.205","cwd":"/tmp/claude-replay-message_steering_mid_tool","tools":[],"mcp_servers":[],"model":"claude-sonnet-4-6","permissionMode":"bypassPermissions","slash_commands":[],"output_style":"default","skills":[],"plugins":[],"fast_mode_state":"off","uuid":"ccaca191-d37a-4e3e-b2cd-c292764fd073","session_id":"590b808f-f261-44e3-b130-88df68cef005"}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"status","status":"requesting","uuid":"e8bcdda2-e070-44e0-b5e3-40427877b216","session_id":"590b808f-f261-44e3-b130-88df68cef005"}} +{"type":"emit_inbound","label":"rate_limit_event","frame":{"type":"rate_limit_event","rate_limit_info":{"status":"allowed"},"uuid":"a05ae74c-4d08-4319-8313-9b7a1521273b","session_id":"590b808f-f261-44e3-b130-88df68cef005"}} +{"type":"emit_inbound","label":"stream_event","frame":{"type":"stream_event","event":{"type":"message_start","message":{"model":"claude-sonnet-4-6","id":"msg_011CdPNV1N51jRQZKHz7hxq6","type":"message","role":"assistant","content":[],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":3,"cache_creation_input_tokens":21607,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":21607},"output_tokens":8,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":null}},"session_id":"590b808f-f261-44e3-b130-88df68cef005","parent_tool_use_id":null,"uuid":"b5bcee4a-8392-4c2f-a561-d6ce0d3e6aa5","ttft_ms":1334}} +{"type":"emit_inbound","label":"stream_event","frame":{"type":"stream_event","event":{"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}},"session_id":"590b808f-f261-44e3-b130-88df68cef005","parent_tool_use_id":null,"uuid":"ed1fd30f-e295-4efd-bed0-77da8d029902"}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"thinking_tokens","estimated_tokens":1,"estimated_tokens_delta":1,"uuid":"d81305ed-c890-475c-be10-50df22442d20","session_id":"590b808f-f261-44e3-b130-88df68cef005"}} +{"type":"emit_inbound","label":"stream_event","frame":{"type":"stream_event","event":{"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"The","estimated_tokens":null}},"session_id":"590b808f-f261-44e3-b130-88df68cef005","parent_tool_use_id":null,"uuid":"d8ea455d-d717-4e6a-96c0-c5da8547726b"}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"thinking_tokens","estimated_tokens":41,"estimated_tokens_delta":40,"uuid":"900d34f2-6a62-491f-94fb-c5db5f604693","session_id":"590b808f-f261-44e3-b130-88df68cef005"}} +{"type":"emit_inbound","label":"stream_event","frame":{"type":"stream_event","event":{"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":" user wants me to run a specific command and wait for it to complete, then respond with exactly \"steering fixture initial response\".\n\nLet me run the command.","estimated_tokens":null}},"session_id":"590b808f-f261-44e3-b130-88df68cef005","parent_tool_use_id":null,"uuid":"541568f6-090f-4ff4-a8b5-e97c124ac5ca"}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"thinking_tokens","estimated_tokens":102,"estimated_tokens_delta":61,"uuid":"9ddac485-ca82-411e-8d0b-8694cb583c5a","session_id":"590b808f-f261-44e3-b130-88df68cef005"}} +{"type":"emit_inbound","label":"stream_event","frame":{"type":"stream_event","event":{"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"EpADCosBCBAYAipAQT+tT/bkK7Qbp1EpQGhAgl7ZdQ31Qh8jvEWhUYkdN3YtxRpeC0drVFnNiU5Oh2AzQqfYdGW1lqPSWdi960AoeTIRY2xhdWRlLXNvbm5ldC00LTY4AEIIdGhpbmtpbmdaJGYxZWI1ZTQzLWRlMTktNDAzYy04NGMzLWM2MjQxNjY5ZWUxNRIM06zXzffQdXInnm/UGgwXPDqzQ1xo8nduHUgiMAh923ZC/LontR2fs/45hf2Mp5t3ZANrpl5dqDEdIL4zSx0JhhG+tSV3SOXsREY42CqxAbkvc5r2T81skEEdRcJc0+ExDJIhPTl+SjGDSBxQG+vngZ/fDNHM3WqtHpB5LWRXNNEVw9O017MPCy+uAz6Os1YKWBwtzPQ3DPYeTFqXOaQDDKL9MVTqWUleaW3tGiCyHRGwACh3t1loGcJCMYK9scoL8q2YbWuEXn1yNi9B8OICBPXgWOLkL+Wt+9x6P5kyFH03vpHKiBKXO6Kh2rX+o4HVdbFufkg2pz0Q2OiSjmZABBgB"}},"session_id":"590b808f-f261-44e3-b130-88df68cef005","parent_tool_use_id":null,"uuid":"92b8be6a-5b8e-45a4-87bb-af96cb1d17b2"}} +{"type":"emit_inbound","label":"assistant","frame":{"type":"assistant","message":{"model":"claude-sonnet-4-6","id":"msg_011CdPNV1N51jRQZKHz7hxq6","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"The user wants me to run a specific command and wait for it to complete, then respond with exactly \"steering fixture initial response\".\n\nLet me run the command.","signature":"EpADCosBCBAYAipAQT+tT/bkK7Qbp1EpQGhAgl7ZdQ31Qh8jvEWhUYkdN3YtxRpeC0drVFnNiU5Oh2AzQqfYdGW1lqPSWdi960AoeTIRY2xhdWRlLXNvbm5ldC00LTY4AEIIdGhpbmtpbmdaJGYxZWI1ZTQzLWRlMTktNDAzYy04NGMzLWM2MjQxNjY5ZWUxNRIM06zXzffQdXInnm/UGgwXPDqzQ1xo8nduHUgiMAh923ZC/LontR2fs/45hf2Mp5t3ZANrpl5dqDEdIL4zSx0JhhG+tSV3SOXsREY42CqxAbkvc5r2T81skEEdRcJc0+ExDJIhPTl+SjGDSBxQG+vngZ/fDNHM3WqtHpB5LWRXNNEVw9O017MPCy+uAz6Os1YKWBwtzPQ3DPYeTFqXOaQDDKL9MVTqWUleaW3tGiCyHRGwACh3t1loGcJCMYK9scoL8q2YbWuEXn1yNi9B8OICBPXgWOLkL+Wt+9x6P5kyFH03vpHKiBKXO6Kh2rX+o4HVdbFufkg2pz0Q2OiSjmZABBgB"}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":3,"cache_creation_input_tokens":21607,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":21607},"output_tokens":8,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":null,"context_management":null},"parent_tool_use_id":null,"session_id":"590b808f-f261-44e3-b130-88df68cef005","uuid":"0370accc-4735-45a2-9007-f36ceea82f8f","request_id":"req_011CdPNUzA91uQu2JfxGJXeC"}} +{"type":"emit_inbound","label":"stream_event","frame":{"type":"stream_event","event":{"type":"content_block_stop","index":0},"session_id":"590b808f-f261-44e3-b130-88df68cef005","parent_tool_use_id":null,"uuid":"ad889203-72c3-4881-9464-e4a170e30c38"}} +{"type":"emit_inbound","label":"stream_event","frame":{"type":"stream_event","event":{"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_0132dpafPKkywYUW4THHuRsX","name":"Bash","input":{},"caller":{"type":"direct"}}},"session_id":"590b808f-f261-44e3-b130-88df68cef005","parent_tool_use_id":null,"uuid":"8bad0e37-eb29-44dd-8e39-13339abcde33"}} +{"type":"emit_inbound","label":"stream_event","frame":{"type":"stream_event","event":{"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":""}},"session_id":"590b808f-f261-44e3-b130-88df68cef005","parent_tool_use_id":null,"uuid":"c106e96a-0bbc-48b9-917e-cd6f03b2bacb"}} +{"type":"emit_inbound","label":"stream_event","frame":{"type":"stream_event","event":{"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"command\": \"node -e \\\"console.log('steering fixture tool started'); setTimeout(() => {}, 20000)\\\""}},"session_id":"590b808f-f261-44e3-b130-88df68cef005","parent_tool_use_id":null,"uuid":"411e6351-8e71-4af4-8dad-beb5d02648b9"}} +{"type":"emit_inbound","label":"stream_event","frame":{"type":"stream_event","event":{"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"\", \"timeout\": 30000"}},"session_id":"590b808f-f261-44e3-b130-88df68cef005","parent_tool_use_id":null,"uuid":"3aaa48e6-8895-432e-961a-8e512739fcb9"}} +{"type":"emit_inbound","label":"stream_event","frame":{"type":"stream_event","event":{"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"}"}},"session_id":"590b808f-f261-44e3-b130-88df68cef005","parent_tool_use_id":null,"uuid":"4f66d203-5cc1-43e1-99a4-858974029cf3"}} +{"type":"emit_inbound","label":"assistant","frame":{"type":"assistant","message":{"model":"claude-sonnet-4-6","id":"msg_011CdPNV1N51jRQZKHz7hxq6","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_0132dpafPKkywYUW4THHuRsX","name":"Bash","input":{"command":"node -e \"console.log('steering fixture tool started'); setTimeout(() => {}, 20000)\"","timeout":30000},"caller":{"type":"direct"}}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":3,"cache_creation_input_tokens":21607,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":21607},"output_tokens":8,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":null,"context_management":null},"parent_tool_use_id":null,"session_id":"590b808f-f261-44e3-b130-88df68cef005","uuid":"16fefcd2-102e-42f2-aab1-8d76f2584a28","request_id":"req_011CdPNUzA91uQu2JfxGJXeC"}} +{"type":"expect_outbound","label":"prompt.offer:2","frame":{"type":"prompt.offer","message":{"type":"user","message":{"role":"user","content":"Actually, respond with exactly: steering fixture observed"},"parent_tool_use_id":null,"priority":"now"}}} +{"type":"emit_inbound","label":"stream_event","frame":{"type":"stream_event","event":{"type":"content_block_stop","index":1},"session_id":"590b808f-f261-44e3-b130-88df68cef005","parent_tool_use_id":null,"uuid":"da7b6b54-71e4-4da9-88af-b84fe8080b9b"}} +{"type":"emit_inbound","label":"stream_event","frame":{"type":"stream_event","event":{"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null,"stop_details":null},"usage":{"input_tokens":3,"cache_creation_input_tokens":21607,"cache_read_input_tokens":0,"output_tokens":139,"output_tokens_details":{"thinking_tokens":45},"iterations":[{"input_tokens":3,"output_tokens":139,"cache_read_input_tokens":0,"cache_creation_input_tokens":21607,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":21607},"type":"message"}]},"context_management":{"applied_edits":[]}},"session_id":"590b808f-f261-44e3-b130-88df68cef005","parent_tool_use_id":null,"uuid":"3b66623b-b0bd-4a6c-8bad-3d7d0884362b"}} +{"type":"emit_inbound","label":"stream_event","frame":{"type":"stream_event","event":{"type":"message_stop"},"session_id":"590b808f-f261-44e3-b130-88df68cef005","parent_tool_use_id":null,"uuid":"addfb425-ba15-4d5f-89d0-a59ad6eea14b"}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"task_started","task_id":"b12owcb7b","tool_use_id":"toolu_0132dpafPKkywYUW4THHuRsX","description":"node -e \"console.log('steering fixture tool started'); setTimeout(() => {}, 20000)\"","task_type":"local_bash","uuid":"9d5d5dc6-059e-43dc-8111-c7c85a88fb0f","session_id":"590b808f-f261-44e3-b130-88df68cef005"}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"task_notification","task_id":"b12owcb7b","tool_use_id":"toolu_0132dpafPKkywYUW4THHuRsX","status":"completed","output_file":"","summary":"node -e \"console.log('steering fixture tool started'); setTimeout(() => {}, 20000)\"","uuid":"090798fd-a956-47e3-9113-5edd6a0e7479","session_id":"590b808f-f261-44e3-b130-88df68cef005"}} +{"type":"emit_inbound","label":"user","frame":{"type":"user","message":{"role":"user","content":[{"tool_use_id":"toolu_0132dpafPKkywYUW4THHuRsX","type":"tool_result","content":"steering fixture tool started","is_error":false}]},"parent_tool_use_id":null,"session_id":"590b808f-f261-44e3-b130-88df68cef005","uuid":"147952a6-89ac-4a9e-8757-00bf39ffcc58","timestamp":"2026-07-25T20:01:39.946Z","tool_use_result":{"stdout":"steering fixture tool started","stderr":"","interrupted":false,"isImage":false,"noOutputExpected":false}}} +{"type":"emit_inbound","label":"result","frame":{"type":"result","subtype":"success","is_error":false,"api_error_status":null,"duration_ms":22850,"duration_api_ms":3770,"ttft_ms":2300,"ttft_stream_ms":1346,"time_to_request_ms":12,"num_turns":2,"result":"","stop_reason":"tool_use","session_id":"590b808f-f261-44e3-b130-88df68cef005","total_cost_usd":0.132363,"usage":{"input_tokens":3,"cache_creation_input_tokens":21607,"cache_read_input_tokens":0,"output_tokens":139,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":21607,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":3,"output_tokens":139,"cache_read_input_tokens":0,"cache_creation_input_tokens":21607,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":21607},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-haiku-4-5-20251001":{"inputTokens":562,"outputTokens":13,"cacheReadInputTokens":0,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":0.000627,"contextWindow":200000,"maxOutputTokens":32000},"claude-sonnet-4-6":{"inputTokens":3,"outputTokens":139,"cacheReadInputTokens":0,"cacheCreationInputTokens":21607,"webSearchRequests":0,"costUSD":0.13173600000000002,"contextWindow":200000,"maxOutputTokens":32000}},"permission_denials":[],"terminal_reason":"aborted_tools","fast_mode_state":"off","uuid":"30455e02-f9ac-435c-8e4d-5bad130c6642"}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"init","agents":[],"apiKeySource":"none","claude_code_version":"2.1.205","cwd":"/tmp/claude-replay-message_steering_mid_tool","tools":[],"mcp_servers":[],"model":"claude-sonnet-4-6","permissionMode":"bypassPermissions","slash_commands":[],"output_style":"default","skills":[],"plugins":[],"fast_mode_state":"off","uuid":"26d7f232-7e90-40af-9421-319490fd1750","session_id":"590b808f-f261-44e3-b130-88df68cef005"}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"status","status":"requesting","uuid":"60318a81-3f8c-4dac-9928-f97a5fc76f31","session_id":"590b808f-f261-44e3-b130-88df68cef005"}} +{"type":"emit_inbound","label":"stream_event","frame":{"type":"stream_event","event":{"type":"message_start","message":{"model":"claude-sonnet-4-6","id":"msg_011CdPNWggDokqxG1oYRArfv","type":"message","role":"assistant","content":[],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":3,"cache_creation_input_tokens":684,"cache_read_input_tokens":21607,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":684},"output_tokens":8,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":null}},"session_id":"590b808f-f261-44e3-b130-88df68cef005","parent_tool_use_id":null,"uuid":"a948a901-4d43-4a2f-aec7-03cb170291b0","ttft_ms":790}} +{"type":"emit_inbound","label":"stream_event","frame":{"type":"stream_event","event":{"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}},"session_id":"590b808f-f261-44e3-b130-88df68cef005","parent_tool_use_id":null,"uuid":"98f806c9-72e7-438f-8033-e5d6dbb93cda"}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"thinking_tokens","estimated_tokens":1,"estimated_tokens_delta":1,"uuid":"23c5d71d-06a0-4264-87bd-727ebf446e91","session_id":"590b808f-f261-44e3-b130-88df68cef005"}} +{"type":"emit_inbound","label":"stream_event","frame":{"type":"stream_event","event":{"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"The","estimated_tokens":null}},"session_id":"590b808f-f261-44e3-b130-88df68cef005","parent_tool_use_id":null,"uuid":"24f7257b-ad9e-492d-bf35-6e10564f5421"}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"thinking_tokens","estimated_tokens":18,"estimated_tokens_delta":17,"uuid":"0ec0250a-8205-491b-9b8f-b8e02bd9a728","session_id":"590b808f-f261-44e3-b130-88df68cef005"}} +{"type":"emit_inbound","label":"stream_event","frame":{"type":"stream_event","event":{"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":" user wants me to respond with exactly \"steering fixture observed\".","estimated_tokens":null}},"session_id":"590b808f-f261-44e3-b130-88df68cef005","parent_tool_use_id":null,"uuid":"d47b9306-a72c-4d4c-9486-ea2a807f2755"}} +{"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"thinking_tokens","estimated_tokens":79,"estimated_tokens_delta":61,"uuid":"b8baf43e-3eba-4dbe-a0ab-cd9f71daba2a","session_id":"590b808f-f261-44e3-b130-88df68cef005"}} +{"type":"emit_inbound","label":"stream_event","frame":{"type":"stream_event","event":{"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"ErUCCosBCBAYAipAy/kmfak/TKKiskdzN4bYb3Ngwr4Vzd026aojQWjk7SQDfcxXlyn4m+17l0bR5a9NZoiuSfo+rVdX0VhxFlOMHDIRY2xhdWRlLXNvbm5ldC00LTY4AEIIdGhpbmtpbmdaJGYxZWI1ZTQzLWRlMTktNDAzYy04NGMzLWM2MjQxNjY5ZWUxNRIMPsExR49gMclY3P3+GgwqQLRuHagw6SxGbsoiMLNIAeZryiG0vwYjbwkr4XMiNtW25GbsWfbjAZ9l8/XDCyAddoFzUTMmC4ixcnkGtCpXRLNzwHC+oFWqrFHeqM9l+ij+o5aUlo88GWMZgiMTlXBIfZJP6a3G1udukC2WporB0Lp5zbaMAv5A4gykQh4W57sRNACG6JvJyYOokJSTnbRh1deJDWs+GAE="}},"session_id":"590b808f-f261-44e3-b130-88df68cef005","parent_tool_use_id":null,"uuid":"70d43ec1-ac89-42dc-b727-8ff40e660360"}} +{"type":"emit_inbound","label":"assistant","frame":{"type":"assistant","message":{"model":"claude-sonnet-4-6","id":"msg_011CdPNWggDokqxG1oYRArfv","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"The user wants me to respond with exactly \"steering fixture observed\".","signature":"ErUCCosBCBAYAipAy/kmfak/TKKiskdzN4bYb3Ngwr4Vzd026aojQWjk7SQDfcxXlyn4m+17l0bR5a9NZoiuSfo+rVdX0VhxFlOMHDIRY2xhdWRlLXNvbm5ldC00LTY4AEIIdGhpbmtpbmdaJGYxZWI1ZTQzLWRlMTktNDAzYy04NGMzLWM2MjQxNjY5ZWUxNRIMPsExR49gMclY3P3+GgwqQLRuHagw6SxGbsoiMLNIAeZryiG0vwYjbwkr4XMiNtW25GbsWfbjAZ9l8/XDCyAddoFzUTMmC4ixcnkGtCpXRLNzwHC+oFWqrFHeqM9l+ij+o5aUlo88GWMZgiMTlXBIfZJP6a3G1udukC2WporB0Lp5zbaMAv5A4gykQh4W57sRNACG6JvJyYOokJSTnbRh1deJDWs+GAE="}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":3,"cache_creation_input_tokens":684,"cache_read_input_tokens":21607,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":684},"output_tokens":8,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":null,"context_management":null},"parent_tool_use_id":null,"session_id":"590b808f-f261-44e3-b130-88df68cef005","uuid":"c7e5a5f3-33f4-464a-8365-11738b600e3d","request_id":"req_011CdPNWfigByL7yw7fb3Rky"}} +{"type":"emit_inbound","label":"stream_event","frame":{"type":"stream_event","event":{"type":"content_block_stop","index":0},"session_id":"590b808f-f261-44e3-b130-88df68cef005","parent_tool_use_id":null,"uuid":"7a9b0f82-cdbe-46e4-bedc-a9a46a38a33e"}} +{"type":"emit_inbound","label":"stream_event","frame":{"type":"stream_event","event":{"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}},"session_id":"590b808f-f261-44e3-b130-88df68cef005","parent_tool_use_id":null,"uuid":"d1d12ccc-ae45-4121-b82e-2b2703e10a4c"}} +{"type":"emit_inbound","label":"stream_event","frame":{"type":"stream_event","event":{"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"steering fixture observed"}},"session_id":"590b808f-f261-44e3-b130-88df68cef005","parent_tool_use_id":null,"uuid":"3cea60a7-f828-49a0-9fe9-67d89864e110"}} +{"type":"emit_inbound","label":"assistant","frame":{"type":"assistant","message":{"model":"claude-sonnet-4-6","id":"msg_011CdPNWggDokqxG1oYRArfv","type":"message","role":"assistant","content":[{"type":"text","text":"steering fixture observed"}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":3,"cache_creation_input_tokens":684,"cache_read_input_tokens":21607,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":684},"output_tokens":8,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":null,"context_management":null},"parent_tool_use_id":null,"session_id":"590b808f-f261-44e3-b130-88df68cef005","uuid":"4a73b7c5-741d-4cfe-bf0a-351bb3667d49","request_id":"req_011CdPNWfigByL7yw7fb3Rky"}} +{"type":"emit_inbound","label":"stream_event","frame":{"type":"stream_event","event":{"type":"content_block_stop","index":1},"session_id":"590b808f-f261-44e3-b130-88df68cef005","parent_tool_use_id":null,"uuid":"ced94514-93a8-413a-8e61-e8ef54710158"}} +{"type":"emit_inbound","label":"stream_event","frame":{"type":"stream_event","event":{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null,"stop_details":null},"usage":{"input_tokens":3,"cache_creation_input_tokens":684,"cache_read_input_tokens":21607,"output_tokens":34,"output_tokens_details":{"thinking_tokens":26},"iterations":[{"input_tokens":3,"output_tokens":34,"cache_read_input_tokens":21607,"cache_creation_input_tokens":684,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":684},"type":"message"}]},"context_management":{"applied_edits":[]}},"session_id":"590b808f-f261-44e3-b130-88df68cef005","parent_tool_use_id":null,"uuid":"70197525-09be-43d1-ade2-74e44c874ec2"}} +{"type":"emit_inbound","label":"stream_event","frame":{"type":"stream_event","event":{"type":"message_stop"},"session_id":"590b808f-f261-44e3-b130-88df68cef005","parent_tool_use_id":null,"uuid":"c5c7d0ae-7dde-49fc-89f9-c6f5eab030fe"}} +{"type":"emit_inbound","label":"result","frame":{"type":"result","subtype":"success","is_error":false,"api_error_status":null,"duration_ms":2728,"duration_api_ms":5222,"ttft_ms":1399,"ttft_stream_ms":794,"time_to_request_ms":4,"num_turns":1,"result":"steering fixture observed","stop_reason":"end_turn","session_id":"590b808f-f261-44e3-b130-88df68cef005","total_cost_usd":0.14346810000000002,"usage":{"input_tokens":3,"cache_creation_input_tokens":684,"cache_read_input_tokens":21607,"output_tokens":34,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":684,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":3,"output_tokens":34,"cache_read_input_tokens":21607,"cache_creation_input_tokens":684,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":684},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-haiku-4-5-20251001":{"inputTokens":562,"outputTokens":13,"cacheReadInputTokens":0,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":0.000627,"contextWindow":200000,"maxOutputTokens":32000},"claude-sonnet-4-6":{"inputTokens":6,"outputTokens":173,"cacheReadInputTokens":21607,"cacheCreationInputTokens":22291,"webSearchRequests":0,"costUSD":0.14284110000000003,"contextWindow":200000,"maxOutputTokens":32000}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","uuid":"20bcdb90-f320-456e-ab51-eaf18d861dc4"}} +{"type":"runtime_exit","status":"success"} diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/message_steering_mid_tool/input.ts b/apps/server/src/orchestration-v2/testkit/fixtures/message_steering_mid_tool/input.ts new file mode 100644 index 00000000000..0eef77b1836 --- /dev/null +++ b/apps/server/src/orchestration-v2/testkit/fixtures/message_steering_mid_tool/input.ts @@ -0,0 +1,19 @@ +import { + MESSAGE_STEERING_MID_TOOL_PROMPT, + MESSAGE_STEERING_STEER_PROMPT, + type OrchestratorFixtureInput, +} from "../shared.ts"; + +export function messageSteeringMidToolInput(): OrchestratorFixtureInput { + return { + steps: [ + { type: "message", text: MESSAGE_STEERING_MID_TOOL_PROMPT }, + { + type: "steer", + text: MESSAGE_STEERING_STEER_PROMPT, + targetRunIndex: 1, + waitForTurnItemType: "command_execution", + }, + ], + }; +} diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/shared.ts b/apps/server/src/orchestration-v2/testkit/fixtures/shared.ts index 5377ba675a0..d3aefdd367e 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/shared.ts +++ b/apps/server/src/orchestration-v2/testkit/fixtures/shared.ts @@ -63,6 +63,8 @@ export const TURN_INTERRUPT_RECOVERY_PROMPT = "Respond with exactly: interrupt recovery fixture complete"; export const MESSAGE_STEERING_STEER_PROMPT = "Actually, respond with exactly: steering fixture observed"; +export const MESSAGE_STEERING_MID_TOOL_PROMPT = + "Run this exact local command: `node -e \"console.log('steering fixture tool started'); setTimeout(() => {}, 20000)\"`. Do not answer until it completes, then respond exactly: steering fixture initial response"; export const THREAD_ROLLBACK_FIRST_PROMPT = "Respond with exactly: rollback fixture first turn complete"; export const THREAD_ROLLBACK_SECOND_PROMPT = @@ -168,6 +170,12 @@ export type OrchestratorFixtureInputStep = readonly text: string; readonly attachments?: ReadonlyArray; readonly targetRunIndex: number; + /** + * Hold the steer until the target run has produced this turn item, so a + * fixture can steer while a tool is still running rather than while the + * assistant is streaming. + */ + readonly waitForTurnItemType?: OrchestrationV2TurnItem["type"]; } | { readonly type: "restart"; @@ -548,6 +556,14 @@ export function materializeFixtureInput(input: { threadId: ids.threadId, runId: runIdFor(step.targetRunIndex), }); + if (step.waitForTurnItemType !== undefined) { + steps.push({ + type: "await_run_turn_item", + threadId: ids.threadId, + runId: runIdFor(step.targetRunIndex), + itemType: step.waitForTurnItemType, + }); + } pushDispatch( dispatchMessageCommand({ commandId: yield* idAllocator.allocate.command({