From 304936a6759961e17ea8d14d7ce899bcca2a1118 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Mon, 27 Jul 2026 19:19:52 +0530 Subject: [PATCH 1/8] feat(orchestration-v2): attribute reused subagents to the run driving them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A subagent row keeps the run id it was spawned under. Routing admits a row by run or by parent thread, and the parent thread is excluded by construction, so a later run re-activating that subagent could only see its updates while the spawning run's ingestion fiber happened to still be alive. The Claude adapter already re-attributed on reopen for exactly this reason; this brings the same treatment to routing and to Codex. Reuse arrives on item/completed, not item/started: resumeAgent and sendInput only ever complete, and registerSubagentThreads ignores them because they are not spawnAgent. Hooking the rebind there is what makes it fire — subagent_continue now asserts the subagent is attributed to the run that drove it, where before it stayed pinned to the spawning run. Routing gains a third test: subagent identity, seeded from the projection at turn start. The child thread is adopted only while the agent is live, because a settled row is also re-emitted by trailing traffic — a token usage frame or a collab state sweep — and adopting on those would hand a later run an interrupted agent's thread, readmitting the stale events that post-interrupt recovery exists to exclude. A subagent's row and its timeline item are emitted together, so both route by identity; matching only the row advanced the agent while its item stayed frozen on the spawning run. The reactivation predicate is separate from canRouteRelatedSubagent rather than a widening of it, for the same reason: an interrupted agent must not have its child thread pre-owned, but its identity is still resumable, because the user stopped it rather than losing it. Rehydration seeds turn ordinals alongside the registry — restarting them at 1 would re-derive childRootNodeId for a different native turn — and parks non-terminal leftovers at idle, since a new session can never terminalize an activation it did not drive and would otherwise keep the session pinned open. Not proven: rehydration itself. The fixture that would exercise it is recorded but does not replay to completion, for reasons unrelated to these changes, and is left unregistered with the open question written up in fixtures/index.ts. --- .../record-codex-app-server-replay-fixture.ts | 61 +++++ .../Adapters/CodexAdapterV2.ts | 150 +++++++++++- .../src/orchestration-v2/ProviderAdapter.ts | 16 ++ .../ProviderTurnStartService.ts | 50 +++- .../RunExecutionService.test.ts | 226 +++++++++++++++++- .../orchestration-v2/RunExecutionService.ts | 156 +++++++++++- .../testkit/fixtures/index.ts | 8 + .../testkit/fixtures/shared.ts | 8 + .../subagent_continue/codex_output.ts | 12 +- .../subagent_reuse_after_idle/codex_output.ts | 60 +++++ .../codex_transcript.ndjson | 149 ++++++++++++ .../subagent_reuse_after_idle/input.ts | 31 +++ 12 files changed, 903 insertions(+), 24 deletions(-) create mode 100644 apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/codex_output.ts create mode 100644 apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/codex_transcript.ndjson create mode 100644 apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/input.ts diff --git a/apps/server/scripts/record-codex-app-server-replay-fixture.ts b/apps/server/scripts/record-codex-app-server-replay-fixture.ts index 6468e230a83..fce4e224333 100644 --- a/apps/server/scripts/record-codex-app-server-replay-fixture.ts +++ b/apps/server/scripts/record-codex-app-server-replay-fixture.ts @@ -27,6 +27,10 @@ import { SUBAGENT_CONTINUE_PARENT_PROMPT, SUBAGENT_CONTINUE_PROMPT, SUBAGENT_PROMPT, + SUBAGENT_REUSE_AFTER_IDLE_PROMPT, + SUBAGENT_REUSE_AFTER_IDLE_RESUME_PROMPT, + SUBAGENT_REUSE_AFTER_INTERRUPT_PROMPT, + SUBAGENT_REUSE_AFTER_INTERRUPT_RESUME_PROMPT, THREAD_ROLLBACK_AFTER_PROMPT, THREAD_ROLLBACK_FIRST_PROMPT, THREAD_ROLLBACK_SECOND_PROMPT, @@ -73,6 +77,8 @@ const SCENARIO_NAMES = [ "tool_call_restricted_granular", "subagent", "subagent_continue", + "subagent_reuse_after_interrupt", + "subagent_reuse_after_idle", "multi_turn", "provider_thread_resume", "todo_list", @@ -445,6 +451,61 @@ function scenarios(): ReadonlyArray { }, ], }, + { + name: "subagent_reuse_after_interrupt", + fileName: "subagent_reuse_after_interrupt.ndjson", + description: + "A root turn spawns a native Codex subagent and is interrupted while it works, then a later root turn re-activates the same child thread.", + runs: [ + { + name: "reuse-after-interrupt", + description: + "Interrupt the spawning turn mid-subagent, then message the same subagent from a new root turn.", + turnDefaults: { + approvalPolicy: "never", + sandboxPolicy: workspaceWriteSandbox(), + }, + steps: [ + { + type: "interruptedTurn", + label: "spawn-subagent-then-interrupt", + prompt: SUBAGENT_REUSE_AFTER_INTERRUPT_PROMPT, + interruptAfterCommandExecutionStarted: true, + }, + { + type: "turn", + label: "reuse-interrupted-subagent", + prompt: SUBAGENT_REUSE_AFTER_INTERRUPT_RESUME_PROMPT, + }, + ], + }, + ], + }, + { + name: "subagent_reuse_after_idle", + fileName: "subagent_reuse_after_idle.ndjson", + description: + "A root turn spawns a native Codex subagent, then a later root turn re-activates it. Replay advances past the session idle timeout between turns so the adapter registry is empty on reuse.", + runs: [ + { + name: "reuse-after-idle", + description: + "Second root turn messages the same subagent after the session would have been reaped.", + steps: [ + { + type: "turn", + label: "spawn-subagent", + prompt: SUBAGENT_REUSE_AFTER_IDLE_PROMPT, + }, + { + type: "turn", + label: "reuse-subagent-after-idle", + prompt: SUBAGENT_REUSE_AFTER_IDLE_RESUME_PROMPT, + }, + ], + }, + ], + }, { name: "multi_turn", fileName: "multi_turn.ndjson", diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 85de9a9f775..757f26b28e4 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -119,12 +119,6 @@ export const CODEX_DRIVER_KIND = CODEX_PROVIDER; export const CODEX_DEFAULT_INSTANCE_ID = defaultInstanceIdForDriver(CODEX_DRIVER_KIND); const DEFAULT_CODEX_SETTINGS = Schema.decodeSync(CodexSettings)({}); const CODEX_ASSISTANT_DELTA_FLUSH_INTERVAL_MS = 50; -const isSettledCodexSubagent = (subagent: OrchestrationV2Subagent) => - subagent.status === "idle" || - subagent.status === "completed" || - subagent.status === "failed" || - subagent.status === "cancelled" || - subagent.status === "interrupted"; const CodexBackgroundTerminalTerminateResponse = Schema.Struct({ terminated: Schema.Boolean, }); @@ -873,7 +867,9 @@ interface DeferredCodexRootTerminal { } interface CodexSubagentThreadContext { - readonly parentContext: ActiveCodexTurnContext; + // Rebound whenever a later root turn re-activates this subagent, so its rows + // are attributed to the run that is actually driving it. + parentContext: ActiveCodexTurnContext; readonly providerThread: OrchestrationV2ProviderThread; readonly childThread: OrchestrationV2AppThread; readonly subagentNodeId: OrchestrationV2ExecutionNode["id"]; @@ -1555,6 +1551,74 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi const emitProviderEvent = (event: ProviderAdapterV2Event) => Queue.offer(events, event).pipe(Effect.asVoid); + /** + * Rebuild registry entries for subagents that already exist in the + * projection. The registry is process-local, so without this a session + * reap or server restart orphans every prior agent: the returning + * native thread is unrecognised and gets spawned as a new identity. + * Entries created here emit nothing — they only let a later + * re-activation resolve to the original subagent. + */ + const seedExistingSubagents = (context: ActiveCodexTurnContext) => + Effect.gen(function* () { + const existing = context.input.existingSubagents ?? []; + if (existing.length === 0) return; + yield* Ref.update(subagentThreads, (current) => { + const updated = new Map(current); + for (const entry of existing) { + const nativeThreadId = entry.childProviderThread.nativeThreadRef?.nativeId ?? null; + if (nativeThreadId === null || updated.has(nativeThreadId)) continue; + const nativeItemId = entry.subagent.nativeTaskRef?.nativeId ?? null; + if (nativeItemId === null || !nativeItemId.includes(":")) continue; + updated.set(nativeThreadId, { + parentContext: context, + providerThread: entry.childProviderThread, + childThread: entry.childThread, + subagentNodeId: entry.subagent.id, + childRootNodeId: idAllocator.derive.nodeFromProviderItem({ + driver: CODEX_PROVIDER, + nativeItemId: `${nativeItemId}:thread-root`, + }), + childThreadId: entry.childThread.id, + nativeToolCallId: nativeItemId.slice(0, nativeItemId.indexOf(":")), + ordinal: entry.ordinal, + startedAt: entry.subagent.startedAt ?? context.startedAt, + turnItemId: entry.turnItemId, + turnItemOrdinal: entry.turnItemOrdinal, + task: { + ...entry.subagent, + runId: context.projectionRunId, + parentNodeId: context.itemParentNodeId, + // A non-terminal status here is a leftover from the session + // that died; this one never drove that activation and can + // never terminalize it. Left as running it would keep + // hasPendingBackgroundWork true and pin the session open. + status: + entry.subagent.status === "pending" || + entry.subagent.status === "running" || + entry.subagent.status === "waiting" + ? "idle" + : entry.subagent.status, + currentActivationId: null, + }, + }); + } + return updated; + }); + // Turn ordinals live in the same lost registry. Restarting them at + // 1 would re-derive childRootNodeId for a different native turn, so + // continue past the activations already recorded. + yield* Ref.update(nextProviderTurnOrdinals, (current) => { + const updated = new Map(current); + for (const entry of existing) { + const key = String(entry.childProviderThread.id); + if (updated.has(key)) continue; + updated.set(key, Math.max(1, entry.subagent.activationCount)); + } + return updated; + }); + }); + const registerRootTurn = (input: { readonly turnInput: ProviderAdapterV2TurnInput; readonly nativeTurnId: string; @@ -1592,6 +1656,7 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi updated.set(input.nativeTurnId, context); return updated; }); + yield* seedExistingSubagents(context); yield* emitProviderEvent({ type: "provider_turn.updated", driver: CODEX_PROVIDER, @@ -2062,7 +2127,11 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi }) => Effect.gen(function* () { const registeredSubagents = yield* Ref.get(subagentThreads); - if (registeredSubagents.has(input.nativeThreadId)) { + const existing = registeredSubagents.get(input.nativeThreadId); + if (existing !== undefined) { + // Same agent thread referenced again by a later turn: keep the + // identity and adopt it into the run now driving it. + rebindSubagentToContext(existing, input.context); return; } @@ -2307,6 +2376,39 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi } }); + /** + * Adopt an already-registered subagent into the root turn that is + * re-activating it. The task row keeps the run it was spawned under + * otherwise, and every later row would be attributed to a closed run: + * routing drops it, and the current run stops counting the subagent as + * open work and can finalize while it is still going. + */ + const rebindSubagentToContext = ( + subagent: CodexSubagentThreadContext, + context: ActiveCodexTurnContext, + ) => { + if (subagent.parentContext.projectionRunId === context.projectionRunId) return; + subagent.parentContext = context; + subagent.task = { + ...subagent.task, + runId: context.projectionRunId, + parentNodeId: context.itemParentNodeId, + }; + }; + + const rebindSubagentsToCurrentTurn = (input: { + readonly context: ActiveCodexTurnContext; + readonly item: CodexCollabAgentToolCallItem; + }) => + Effect.gen(function* () { + if (input.item.receiverThreadIds.length === 0) return; + const registered = yield* Ref.get(subagentThreads); + for (const nativeThreadId of input.item.receiverThreadIds) { + const subagent = registered.get(nativeThreadId); + if (subagent !== undefined) rebindSubagentToContext(subagent, input.context); + } + }); + const registerSubagentActivity = (input: { readonly context: ActiveCodexTurnContext; readonly item: CodexSubAgentActivityItem; @@ -2340,6 +2442,17 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi return; } + // "interacted" is how Codex reports a root turn messaging an agent + // it did not spawn. It is the only reuse signal on this path — the + // collab tool call that accompanies it carries neither + // receiverThreadIds nor agentsStates — and it arrives before the + // child turn starts, so adopting the identity here is what lets the + // resulting activation be attributed to the run driving it. + if (input.item.kind === "interacted") { + rebindSubagentToContext(subagent, input.context); + return; + } + if (input.item.kind === "interrupted") { yield* emitSubagentTaskUpdate({ subagent, @@ -3411,7 +3524,13 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi yield* client.handleServerNotification("thread/status/changed", (payload) => Effect.gen(function* () { const subagent = (yield* Ref.get(subagentThreads)).get(payload.threadId); - if (subagent === undefined || isSettledCodexSubagent(subagent.task)) return; + // Live status applies only while an activation is in flight. Every + // resting and terminal state clears currentActivationId, so this + // single check drops trailing frames for an idle or finished + // identity while letting a re-activated one resume immediately. + if (subagent === undefined || subagent.task.currentActivationId === null) { + return; + } const status = payload.status.type === "active" ? payload.status.activeFlags.length > 0 @@ -3467,6 +3586,14 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi } } + // A collab tool call that targets an existing agent thread is a + // re-activation; adopt it into this turn before its child turn + // starts so the activation is attributed to the current run. + if (payload.item.type === "collabAgentToolCall") { + yield* rebindSubagentsToCurrentTurn({ context, item: payload.item }); + return; + } + if (payload.item.type === "subAgentActivity") { yield* registerSubagentActivity({ context, @@ -3728,6 +3855,11 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi context, item: payload.item, }); + // Reuse arrives here, not on item/started: resumeAgent and + // sendInput only ever complete. registerSubagentThreads ignores + // them because they are not spawnAgent, so without this the + // returning agent keeps the run it was spawned under. + yield* rebindSubagentsToCurrentTurn({ context, item: payload.item }); yield* updateSubagentStates({ item: payload.item, }); diff --git a/apps/server/src/orchestration-v2/ProviderAdapter.ts b/apps/server/src/orchestration-v2/ProviderAdapter.ts index e05fae8d554..0ac0ba1ceb2 100644 --- a/apps/server/src/orchestration-v2/ProviderAdapter.ts +++ b/apps/server/src/orchestration-v2/ProviderAdapter.ts @@ -392,6 +392,21 @@ export interface ProviderAdapterV2EnsureThreadInput { readonly existingProviderThread?: OrchestrationV2ProviderThread; } +/** + * A reusable subagent that already exists in the projection, supplied so an + * adapter whose in-memory registry was lost (session reap, server restart) can + * still recognise the agent thread and re-activate the same identity instead of + * spawning a duplicate. + */ +export interface ProviderAdapterV2ExistingSubagent { + readonly subagent: OrchestrationV2Subagent; + readonly childThread: OrchestrationV2AppThread; + readonly childProviderThread: OrchestrationV2ProviderThread; + readonly turnItemId: TurnItemId; + readonly turnItemOrdinal: number; + readonly ordinal: number; +} + export interface ProviderAdapterV2TurnInput { readonly appThread: OrchestrationV2AppThread; readonly threadId: ThreadId; @@ -404,6 +419,7 @@ export interface ProviderAdapterV2TurnInput { readonly message: ProviderAdapterV2TurnMessage; readonly modelSelection: ModelSelection; readonly runtimePolicy: ProviderAdapterV2RuntimePolicy; + readonly existingSubagents?: ReadonlyArray; } export interface ProviderAdapterV2SteerInput { diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 3b6f74bf5f7..e2bddd7ec31 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -20,9 +20,14 @@ import { providerMessageWithContextHandoffs, } from "./ContextHandoffService.ts"; import { IdAllocatorV2 } from "./IdAllocator.ts"; +import type { ProviderAdapterV2ExistingSubagent } from "./ProviderAdapter.ts"; import { ProjectionStoreV2 } from "./ProjectionStore.ts"; import { ProviderSessionManagerV2 } from "./ProviderSessionManager.ts"; -import { canRouteRelatedSubagent, RunExecutionServiceV2 } from "./RunExecutionService.ts"; +import { + canReactivateSubagent, + canRouteRelatedSubagent, + RunExecutionServiceV2, +} from "./RunExecutionService.ts"; import { RuntimePolicyV2 } from "./RuntimePolicy.ts"; export class ProviderTurnStartError extends Schema.TaggedErrorClass()( @@ -406,6 +411,47 @@ export const layer: Layer.Layer< const routableSubagents = projection.subagents.filter((subagent) => canRouteRelatedSubagent(subagent.status), ); + // Broader than the child-thread list above: an interrupted agent is still + // resumable, so it stays re-activatable even though its thread must not + // be adopted up front. + const reactivatableSubagents = projection.subagents.filter((subagent) => + canReactivateSubagent(subagent.status), + ); + // Adapters keep their subagent registry in memory, so it is empty after a + // session reap or a server restart. Hand back the reusable identities + // from the projection so a returning agent thread is recognised instead + // of being spawned again as a duplicate. + const existingSubagents = yield* Effect.forEach( + reactivatableSubagents, + (subagent, index): Effect.Effect> => + Effect.gen(function* () { + const childThreadId = subagent.childThreadId; + if (childThreadId === null) return []; + const turnItem = projection.turnItems.find( + (item) => item.type === "subagent" && item.subagentId === subagent.id, + ); + if (turnItem === undefined) return []; + const childProjection = yield* projectionStore + .getThreadProjection(childThreadId) + .pipe(Effect.catch(() => Effect.succeed(null))); + if (childProjection === null) return []; + const childProviderThread = childProjection.providerThreads.find( + (candidate) => candidate.id === subagent.providerThreadId, + ); + if (childProviderThread === undefined) return []; + return [ + { + subagent, + childThread: childProjection.thread, + childProviderThread, + turnItemId: turnItem.id, + turnItemOrdinal: turnItem.ordinal, + ordinal: index + 1, + }, + ]; + }), + { concurrency: "unbounded" }, + ).pipe(Effect.map((entries) => entries.flat())); yield* runExecution.startRootRun({ commandId: CommandId.make(`command:effect:provider-turn.start:${run.id}`), appThread: projection.thread, @@ -423,6 +469,8 @@ export const layer: Layer.Layer< relatedProviderThreadIds: routableSubagents.flatMap((subagent) => subagent.providerThreadId === null ? [] : [subagent.providerThreadId], ), + relatedSubagentIds: reactivatableSubagents.map((subagent) => subagent.id), + existingSubagents, providerTurnOrdinal: Math.max( 0, diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 2a1ce09d1d6..32dca9f8b15 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -13,6 +13,7 @@ import { type OrchestrationV2Run, type OrchestrationV2RunAttempt, type OrchestrationV2Subagent, + type OrchestrationV2SubagentActivation, type OrchestrationV2TurnItem, ProviderDriverKind, ProviderInstanceId, @@ -21,6 +22,7 @@ import { ProviderTurnId, RunAttemptId, RunId, + SubagentActivationId, ThreadId, TurnItemId, } from "@t3tools/contracts"; @@ -40,6 +42,7 @@ import { IdAllocatorV2, layer as idAllocatorLayer } from "./IdAllocator.ts"; import type { ProviderAdapterV2Event, ProviderAdapterV2SessionRuntime } from "./ProviderAdapter.ts"; import { ProviderEventIngestorV2 } from "./ProviderEventIngestor.ts"; import { + canReactivateSubagent, canRouteRelatedSubagent, cascadeTerminalizeRunOwnedSubagents, finalProviderThreadStatus, @@ -154,6 +157,83 @@ it.effect("routes shared-runtime events only to their owning root run", () => }), ); +it.effect("routes a reused subagent's rows to the run that re-activated it", () => + Effect.gen(function* () { + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread:reuse"); + const spawnRunId = RunId.make("run:reuse:1"); + const subagentId = NodeId.make("node:reuse:subagent"); + const childThreadId = ThreadId.make("thread:reuse:child"); + // The turn that re-activates an existing subagent, two runs later. + const laterRun: ProviderEventRouteIdentity = { + threadId, + runId: RunId.make("run:reuse:2"), + attemptId: RunAttemptId.make("attempt:reuse:2"), + providerThreadId: ProviderThreadId.make("provider-thread:reuse"), + }; + + // A reusable subagent keeps the run id it was spawned under, and its row is + // keyed on the parent thread — so neither the run test nor the child-thread + // test can match. Only the identity seeded from the projection can. + const subagentEvent: ProviderAdapterV2Event = { + type: "subagent.updated", + driver, + subagent: { + ...makeRunOwnedSubagentFixture({ + ids: { + threadId, + runId: spawnRunId, + rootNodeId: NodeId.make("node:reuse:root"), + subagentNodeId: subagentId, + } as BackgroundScenarioIds, + providerInstanceId: ProviderInstanceId.make("codex"), + childThreadId, + driver, + status: "running", + }), + activationCount: 2, + updatedAt: now, + }, + }; + const activationEvent: ProviderAdapterV2Event = { + type: "subagent_activation.updated", + driver, + activation: { + id: SubagentActivationId.make(`${subagentId}:activation:2`), + threadId, + subagentId, + runId: spawnRunId, + providerTurnId: null, + ordinal: 2, + status: "running", + usage: null, + startedAt: now, + completedAt: null, + updatedAt: now, + }, + }; + + const withoutSeed = makeProviderEventRoutingState({ + identity: laterRun, + providerTurnId: null, + }); + assert.isFalse(routeProviderEvent(subagentEvent, laterRun, withoutSeed)[0]); + assert.isFalse(routeProviderEvent(activationEvent, laterRun, withoutSeed)[0]); + + const seeded = makeProviderEventRoutingState({ + identity: laterRun, + providerTurnId: null, + relatedSubagentIds: [subagentId], + }); + const [subagentAccepted, afterSubagent] = routeProviderEvent(subagentEvent, laterRun, seeded); + assert.isTrue(subagentAccepted); + assert.isTrue(routeProviderEvent(activationEvent, laterRun, afterSubagent)[0]); + // Accepting the row also adopts its child thread, so the agent's own + // messages and turn items route into this run for the rest of it. + assert.isTrue(afterSubagent.ownedThreadIds.has(childThreadId)); + }), +); + it("does not route a superseded attempt through a reused provider thread", () => { const threadId = ThreadId.make("thread:shared-runtime:restart"); const providerThreadId = ProviderThreadId.make("provider-thread:shared-runtime:restart"); @@ -195,6 +275,13 @@ it("does not carry interrupted child ownership into later attempts", () => { assert.isTrue(canRouteRelatedSubagent("completed")); assert.isTrue(canRouteRelatedSubagent("running")); + // Re-activation is deliberately broader: the user stopped an interrupted + // agent, they did not lose it, and its provider thread is still resumable. + assert.isTrue(canReactivateSubagent("interrupted")); + assert.isTrue(canReactivateSubagent("idle")); + assert.isFalse(canReactivateSubagent("failed")); + assert.isFalse(canReactivateSubagent("cancelled")); + const threadId = ThreadId.make("thread:related-child:next-attempt"); const childThreadId = ThreadId.make("thread:related-child:interrupted"); const identity: ProviderEventRouteIdentity = { @@ -234,6 +321,82 @@ it("does not carry interrupted child ownership into later attempts", () => { assert.isFalse(routeProviderEvent(lateChildNode, identity, state)[0]); }); +it.effect("re-activates an interrupted subagent without adopting its stale child traffic", () => + Effect.gen(function* () { + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread:interrupted-reuse"); + const subagentId = NodeId.make("node:interrupted-reuse:subagent"); + const childThreadId = ThreadId.make("thread:interrupted-reuse:child"); + const laterRun: ProviderEventRouteIdentity = { + threadId, + runId: RunId.make("run:interrupted-reuse:2"), + attemptId: RunAttemptId.make("attempt:interrupted-reuse:2"), + providerThreadId: ProviderThreadId.make("provider-thread:interrupted-reuse"), + }; + + // What ProviderTurnStartService builds for a previously interrupted agent: + // its identity is re-activatable, but its child thread is NOT pre-owned. + const state = makeProviderEventRoutingState({ + identity: laterRun, + providerTurnId: null, + relatedThreadIds: [], + relatedSubagentIds: [subagentId], + }); + + // A late event from the interrupted child thread must still be refused — + // this is the invariant the post-interrupt recovery fix depends on. + const lateChildNode: ProviderAdapterV2Event = { + type: "node.updated", + driver, + node: { + id: NodeId.make("node:interrupted-reuse:late"), + threadId: childThreadId, + runId: null, + parentNodeId: null, + rootNodeId: NodeId.make("node:interrupted-reuse:late"), + kind: "root_turn", + status: "completed", + countsForRun: false, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: null, + completedAt: null, + }, + }; + assert.isFalse(routeProviderEvent(lateChildNode, laterRun, state)[0]); + + // A genuine re-activation of that identity is admitted, and only then is + // the child thread adopted — ownership is earned, not granted up front. + const reactivation: ProviderAdapterV2Event = { + type: "subagent.updated", + driver, + subagent: { + ...makeRunOwnedSubagentFixture({ + ids: { + threadId, + runId: RunId.make("run:interrupted-reuse:1"), + rootNodeId: NodeId.make("node:interrupted-reuse:root"), + subagentNodeId: subagentId, + } as BackgroundScenarioIds, + providerInstanceId: ProviderInstanceId.make("codex"), + childThreadId, + driver, + status: "running", + }), + activationCount: 4, + updatedAt: now, + }, + }; + const [accepted, afterReactivation] = routeProviderEvent(reactivation, laterRun, state); + assert.isTrue(accepted); + assert.isTrue(afterReactivation.ownedThreadIds.has(childThreadId)); + assert.isTrue(routeProviderEvent(lateChildNode, laterRun, afterReactivation)[0]); + }), +); + it.effect("rechecks run ownership immediately before calling the provider", () => Effect.gen(function* () { const runExecution = yield* RunExecutionServiceV2; @@ -1289,6 +1452,12 @@ it.effect("cascade helper is provider-neutral for Claude and Codex-shaped child const threadId = ThreadId.make(`thread:cascade-helper:${driverKind}`); const childThreadId = ThreadId.make(`thread:cascade-helper:${driverKind}:child`); const subagentId = NodeId.make(`node:cascade-helper:${driverKind}:subagent`); + const activationId = SubagentActivationId.make( + `node:cascade-helper:${driverKind}:subagent:activation:2`, + ); + const previousActivationId = SubagentActivationId.make( + `node:cascade-helper:${driverKind}:subagent:activation:1`, + ); const childNodeId = NodeId.make(`node:cascade-helper:${driverKind}:child-root`); const providerInstanceId = ProviderInstanceId.make(String(driverKind)); const terminalStatus = driverKind === "claudeAgent" ? "failed" : "cancelled"; @@ -1313,19 +1482,38 @@ it.effect("cascade helper is provider-neutral for Claude and Codex-shaped child model: null, kind: "subagent", role: { name: "general-purpose", source: "app_default" }, + status: "running", + progress: "partial progress", + result: "partial result", usage: null, - currentActivationId: null, - activationCount: 1, + currentActivationId: activationId, + activationCount: 2, workflow: null, workflowMembership: null, recentActivity: [], + startedAt: now, + completedAt: null, + updatedAt: now, + }; + const activation: OrchestrationV2SubagentActivation = { + id: activationId, + threadId, + subagentId, + runId, + providerTurnId: null, + ordinal: 2, status: "running", - progress: "partial progress", - result: "partial result", + usage: { totalTokens: 120, toolUses: 2 }, startedAt: now, completedAt: null, updatedAt: now, }; + const previousActivation: OrchestrationV2SubagentActivation = { + ...activation, + id: previousActivationId, + ordinal: 1, + usage: { totalTokens: 60, toolUses: 1 }, + }; const turnItem = { id: TurnItemId.make(`turn-item:cascade-helper:${driverKind}`), threadId, @@ -1434,6 +1622,10 @@ it.effect("cascade helper is provider-neutral for Claude and Codex-shaped child } as OrchestrationV2Run, open: { subagents: new Map([[subagentId, subagent]]), + activations: new Map([ + [previousActivation.id, previousActivation], + [activation.id, activation], + ]), turnItems: new Map([[subagentId, turnItem]]), childTurnItems: new Map(), nodes: new Map([[subagentId, node]]), @@ -1444,7 +1636,7 @@ it.effect("cascade helper is provider-neutral for Claude and Codex-shaped child allocateEventId, }); - assert.equal(events.length, 3, `${driverKind}: subagent + node + turn item`); + assert.equal(events.length, 5, `${driverKind}: subagent + activations + node + turn item`); const terminalSubagent = events.find((event) => event.type === "subagent.updated"); assert.isDefined(terminalSubagent); if (terminalSubagent?.type !== "subagent.updated") { @@ -1456,6 +1648,25 @@ it.effect("cascade helper is provider-neutral for Claude and Codex-shaped child assert.equal(terminalSubagent.payload.progress, "partial progress"); assert.equal(terminalSubagent.payload.result, "partial result"); assert.equal(terminalSubagent.payload.driver, driverKind); + assert.isNull(terminalSubagent.payload.currentActivationId); + + const terminalActivations = events.filter( + ( + event, + ): event is Extract => + event.type === "subagent-activation.updated", + ); + assert.deepEqual( + terminalActivations.map((event) => ({ + id: event.payload.id, + status: event.payload.status, + totalTokens: event.payload.usage?.totalTokens, + })), + [ + { id: previousActivationId, status: terminalStatus, totalTokens: 60 }, + { id: activationId, status: terminalStatus, totalTokens: 120 }, + ], + ); const terminalItem = events.find( (event) => event.type === "turn-item.updated" && event.payload.type === "subagent", @@ -1481,6 +1692,7 @@ it.effect("cascade helper is provider-neutral for Claude and Codex-shaped child } as OrchestrationV2Run, open: { subagents: new Map(), + activations: new Map(), turnItems: new Map(), childTurnItems: new Map([[childTurnItem.id, childTurnItem]]), nodes: new Map([[childNodeId, openChildNode]]), @@ -1875,14 +2087,14 @@ function makeRunOwnedSubagentFixture(input: { model: null, kind: "subagent", role: { name: "general-purpose", source: "app_default" }, + status: input.status, + result: null, usage: null, currentActivationId: null, activationCount: 1, workflow: null, workflowMembership: null, recentActivity: [], - status: input.status, - result: null, startedAt: now, completedAt: null, updatedAt: now, diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 4638466e5fb..3ea430455c1 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -13,11 +13,13 @@ import { type OrchestrationV2Run, type OrchestrationV2RunAttempt, type OrchestrationV2Subagent, + type OrchestrationV2SubagentActivation, type OrchestrationV2TurnItem, type ProviderSessionId, type ProviderThreadId, type ProviderTurnId, type RunAttemptId, + type SubagentActivationId, type ThreadId, type TurnItemId, } from "@t3tools/contracts"; @@ -43,6 +45,7 @@ import { import type { ProviderAdapterV2Event, ProviderAdapterV2RuntimePolicy, + ProviderAdapterV2ExistingSubagent, ProviderAdapterV2SessionRuntime, ProviderAdapterV2TurnMessage, } from "./ProviderAdapter.ts"; @@ -53,6 +56,10 @@ export interface ProviderEventRoutingState { readonly ownedThreadIds: ReadonlySet; readonly ownedProviderThreadIds: ReadonlySet; readonly ownedProviderTurnIds: ReadonlySet; + // Subagent identities this run may observe. A reusable subagent keeps the + // run id it was spawned under, so a later run that re-activates it cannot + // recognise its rows by run or by parent thread; it matches on identity. + readonly ownedSubagentIds: ReadonlySet; readonly rootProviderTurnId: ProviderTurnId | null; } @@ -76,6 +83,7 @@ function isTerminalProviderTurnStatus(status: OrchestrationV2ProviderTurn["statu function isTerminalSubagentStatus(status: OrchestrationV2Subagent["status"]): boolean { return ( + status === "idle" || status === "completed" || status === "interrupted" || status === "failed" || @@ -106,6 +114,7 @@ type SubagentTurnItem = Extract; + readonly activations: ReadonlyMap; readonly turnItems: ReadonlyMap; readonly childTurnItems: ReadonlyMap; readonly nodes: ReadonlyMap; @@ -128,13 +137,35 @@ function isRunOwnedSubagentTerminalStatus( return status === "interrupted" || status === "failed" || status === "cancelled"; } +/** + * Whether a later run may pre-emptively own this subagent's child thread. + * + * Interrupted children are excluded on purpose: their threads can still emit + * late events, and adopting them up front lets that stale traffic land in the + * next attempt (see "preserve claude/codex post-interrupt recovery state"). + */ export function canRouteRelatedSubagent(status: OrchestrationV2Subagent["status"]): boolean { return status !== "interrupted" && status !== "failed" && status !== "cancelled"; } +/** + * Whether a later run may re-activate this subagent identity. + * + * Deliberately broader than {@link canRouteRelatedSubagent}: an interrupted + * agent is one the user stopped, not one that died, and its provider thread is + * still resumable. Only rows for this exact subagent id are admitted by + * identity, and those are emitted solely on a genuine re-activation — so the + * child thread is adopted when the agent proves it is live again rather than up + * front, and late stale traffic still has no way in. + */ +export function canReactivateSubagent(status: OrchestrationV2Subagent["status"]): boolean { + return status !== "failed" && status !== "cancelled"; +} + function emptyOpenRunOwnedSubagentProjection(): OpenRunOwnedSubagentProjection { return { subagents: new Map(), + activations: new Map(), turnItems: new Map(), childTurnItems: new Map(), nodes: new Map(), @@ -191,6 +222,7 @@ export function cascadeTerminalizeRunOwnedSubagents(input: { payload: { ...subagent, status: input.status, + currentActivationId: null, completedAt: input.completedAt, updatedAt: input.completedAt, }, @@ -241,6 +273,31 @@ export function cascadeTerminalizeRunOwnedSubagents(input: { }); } } + for (const activation of input.open.activations.values()) { + if ( + activation.status === "completed" || + activation.status === "interrupted" || + activation.status === "failed" || + activation.status === "cancelled" + ) { + continue; + } + events.push({ + id: yield* input.allocateEventId(), + type: "subagent-activation.updated", + threadId: activation.threadId, + runId: activation.runId ?? input.run.id, + nodeId: activation.subagentId, + providerInstanceId: input.run.providerInstanceId, + occurredAt: input.completedAt, + payload: { + ...activation, + status: input.status, + completedAt: input.completedAt, + updatedAt: input.completedAt, + }, + }); + } for (const turnItem of input.open.childTurnItems.values()) { if (!childThreadIds.has(turnItem.threadId) || isTerminalTurnItemStatus(turnItem.status)) { continue; @@ -277,6 +334,7 @@ export function makeProviderEventRoutingState(input: { readonly providerTurnId: ProviderTurnId | null; readonly relatedThreadIds?: ReadonlyArray; readonly relatedProviderThreadIds?: ReadonlyArray; + readonly relatedSubagentIds?: ReadonlyArray; }): ProviderEventRoutingState { return { ownedThreadIds: new Set([input.identity.threadId, ...(input.relatedThreadIds ?? [])]), @@ -286,6 +344,7 @@ export function makeProviderEventRoutingState(input: { ]), ownedProviderTurnIds: input.providerTurnId === null ? new Set() : new Set([input.providerTurnId]), + ownedSubagentIds: new Set(input.relatedSubagentIds ?? []), rootProviderTurnId: input.providerTurnId, }; } @@ -311,6 +370,28 @@ export function routeProviderEvent( ownedProviderTurnIds: new Set([...state.ownedProviderTurnIds, providerTurnId]), rootProviderTurnId: root ? providerTurnId : state.rootProviderTurnId, }); + const ownsSubagent = (subagentId: NodeId): boolean => state.ownedSubagentIds.has(subagentId); + /** + * Track the identity so every later row for the same subagent routes. + * + * The child thread is adopted only while the agent is actually live. A + * settled row is also emitted by trailing traffic — a token-usage frame or a + * collab state sweep re-emits the last known status — and adopting on those + * would hand a later run an interrupted agent's thread, letting exactly the + * stale child-thread events that post-interrupt recovery excludes back in. + */ + const addSubagent = ( + subagentId: NodeId, + childThreadId: ThreadId | null, + live: boolean, + ): ProviderEventRoutingState => ({ + ...state, + ownedSubagentIds: new Set([...state.ownedSubagentIds, subagentId]), + ownedThreadIds: + childThreadId === null || !live + ? state.ownedThreadIds + : new Set([...state.ownedThreadIds, childThreadId]), + }); switch (event.type) { case "provider_session.updated": @@ -359,14 +440,41 @@ export function routeProviderEvent( } return [true, addProviderThread(event.node.providerThreadId)]; } - case "subagent.updated": - return [ownsRun(event.subagent.runId) || ownsChildThread(event.subagent.threadId), state]; + case "subagent.updated": { + const belongs = + ownsRun(event.subagent.runId) || + ownsChildThread(event.subagent.threadId) || + ownsSubagent(event.subagent.id); + return belongs + ? [ + true, + addSubagent( + event.subagent.id, + event.subagent.childThreadId, + !isTerminalSubagentStatus(event.subagent.status), + ), + ] + : [false, state]; + } case "subagent_activation.updated": - return [ownsRun(event.activation.runId) || ownsChildThread(event.activation.threadId), state]; + return [ + ownsRun(event.activation.runId) || + ownsChildThread(event.activation.threadId) || + ownsSubagent(event.activation.subagentId), + state, + ]; case "message.updated": return [ownsRun(event.message.runId) || ownsChildThread(event.message.threadId), state]; case "turn_item.updated": - return [ownsRun(event.turnItem.runId) || ownsChildThread(event.turnItem.threadId), state]; + return [ + ownsRun(event.turnItem.runId) || + ownsChildThread(event.turnItem.threadId) || + // A reused subagent's row and its timeline item are emitted together; + // routing only the row would advance the agent while its item stayed + // frozen on the spawning run. + (event.turnItem.type === "subagent" && ownsSubagent(event.turnItem.subagentId)), + state, + ]; case "plan.updated": return [ownsRun(event.plan.runId) || ownsChildThread(event.plan.threadId), state]; case "runtime_request.updated": @@ -432,6 +540,8 @@ export interface RunExecutionServiceV2StartRootRunInput { readonly providerTurnOrdinal: number; readonly relatedThreadIds?: ReadonlyArray; readonly relatedProviderThreadIds?: ReadonlyArray; + readonly relatedSubagentIds?: ReadonlyArray; + readonly existingSubagents?: ReadonlyArray; readonly shouldStartProviderTurn?: () => Effect.Effect; readonly shouldFinalizeRun?: () => Effect.Effect; readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect; @@ -551,6 +661,7 @@ export const layer: Layer.Layer< const open = input.openRunOwnedSubagents ?? emptyOpenRunOwnedSubagentProjection(); const hasOpenSubagentProjection = open.subagents.size > 0 || + open.activations.size > 0 || open.turnItems.size > 0 || open.childTurnItems.size > 0 || open.nodes.size > 0; @@ -772,6 +883,9 @@ export const layer: Layer.Layer< ...(input.relatedProviderThreadIds === undefined ? {} : { relatedProviderThreadIds: input.relatedProviderThreadIds }), + ...(input.relatedSubagentIds === undefined + ? {} + : { relatedSubagentIds: input.relatedSubagentIds }), }), ); const rootTerminalSeen = yield* Ref.make(false); @@ -836,7 +950,11 @@ export const layer: Layer.Layer< } } if (event.type === "subagent.updated") { - const belongsToRootRun = event.subagent.runId === input.run.id; + // A re-activated subagent is adopted by the current run, so it + // holds this run open the same way a freshly spawned one does. + const belongsToRootRun = + event.subagent.runId === input.run.id || + routing.ownedSubagentIds.has(event.subagent.id); const belongsToOwnedChildThread = event.subagent.threadId !== input.run.threadId && routing.ownedThreadIds.has(event.subagent.threadId); @@ -868,6 +986,31 @@ export const layer: Layer.Layer< }); } } + if (event.type === "subagent_activation.updated") { + const belongsToRootRun = + event.activation.runId === input.run.id || + routing.ownedSubagentIds.has(event.activation.subagentId); + const belongsToOwnedChildThread = + event.activation.threadId !== input.run.threadId && + routing.ownedThreadIds.has(event.activation.threadId); + if (!belongsToRootRun && !belongsToOwnedChildThread) { + return; + } + yield* Ref.update(openRunOwnedSubagents, (current) => { + const activations = new Map(current.activations); + if ( + event.activation.status === "completed" || + event.activation.status === "interrupted" || + event.activation.status === "failed" || + event.activation.status === "cancelled" + ) { + activations.delete(event.activation.id); + } else { + activations.set(event.activation.id, event.activation); + } + return { ...current, activations }; + }); + } if (event.type === "node.updated") { const belongsToRootSubagent = event.node.kind === "subagent" && event.node.runId === input.run.id; @@ -1114,6 +1257,9 @@ export const layer: Layer.Layer< message: input.message, modelSelection: input.modelSelection, runtimePolicy: input.runtimePolicy, + ...(input.existingSubagents === undefined + ? {} + : { existingSubagents: input.existingSubagents }), }) .pipe( Effect.catchCause((cause) => diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/index.ts b/apps/server/src/orchestration-v2/testkit/fixtures/index.ts index 04c157b99cf..afb071a6793 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/index.ts +++ b/apps/server/src/orchestration-v2/testkit/fixtures/index.ts @@ -387,6 +387,14 @@ export const ORCHESTRATOR_REPLAY_FIXTURES = [ }, ], }, + // subagent_reuse_after_idle is scaffolded and recorded but not registered: + // replaying it leaves the second run running forever, so the scenario never + // reaches thread idle. Not caused by the reuse routing — the hang reproduces + // with those changes reverted, and persists now that the rebind demonstrably + // fires (subagent_continue proves that). Root cause still unknown; the + // recorded transcript uses the current Codex item shape (subAgentActivity + // started/interacted, collab `wait` with empty receiverThreadIds), whereas + // the passing subagent_continue transcript uses resumeAgent/sendInput. { name: "subagent_v2", buildInput: subagentV2Input, diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/shared.ts b/apps/server/src/orchestration-v2/testkit/fixtures/shared.ts index 0a50aca4e5d..1f2457c51b8 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/shared.ts +++ b/apps/server/src/orchestration-v2/testkit/fixtures/shared.ts @@ -55,6 +55,14 @@ export const SUBAGENT_CONTINUE_PROMPT = export const SUBAGENT_CONTINUE_PARENT_PROMPT = "@hooke have the same subagent reply exactly: continued subagent response"; export const SUBAGENT_CONTINUE_CHILD_PROMPT = "Reply exactly: continued subagent response"; +export const SUBAGENT_REUSE_AFTER_INTERRUPT_PROMPT = + "Spawn one subagent named reuse_probe. Have it run the local shell command `sleep 30` and then reply exactly: reuse probe first response"; +export const SUBAGENT_REUSE_AFTER_INTERRUPT_RESUME_PROMPT = + "Send another message to the same reuse_probe subagent. Do not spawn a replacement. Have it reply exactly: reuse probe resumed response"; +export const SUBAGENT_REUSE_AFTER_IDLE_PROMPT = + "Spawn one subagent named idle_probe and have it reply exactly: idle probe first response"; +export const SUBAGENT_REUSE_AFTER_IDLE_RESUME_PROMPT = + "Send another message to the same idle_probe subagent. Do not spawn a replacement. Have it reply exactly: idle probe resumed response"; export const TURN_INTERRUPT_PROMPT = "Do not answer immediately. First run the local shell command `sleep 30`, then respond with exactly: interrupt fixture should not finish naturally."; export const TURN_INTERRUPT_MID_TOOL_PROMPT = diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/subagent_continue/codex_output.ts b/apps/server/src/orchestration-v2/testkit/fixtures/subagent_continue/codex_output.ts index d9f3dfcd43a..46503730427 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/subagent_continue/codex_output.ts +++ b/apps/server/src/orchestration-v2/testkit/fixtures/subagent_continue/codex_output.ts @@ -34,10 +34,18 @@ export function assertSubagentContinueOutput( assert.lengthOf(projection.subagents, 1); const subagent = projection.subagents[0]!; - // A reusable identity rests at idle between activations rather than - // finishing outright, and carries the latest activation's result. assert.equal(subagent.status, "idle"); + // The continuation runs under a second app run. The task identity is + // reusable, so its row must carry the latest activation's result and be + // adopted by the run that drove it, not stay frozen on the spawning run. assert.equal(subagent.result, "continued subagent response"); + assert.equal(subagent.activationCount, 2); + // Adoption by the resuming run is what keeps the row routable once the + // spawning run's ingestion fiber closes; pinned to run 1 it is only visible + // for as long as that fiber happens to outlive the turn. + const latestRun = projection.runs.at(-1); + assert.isDefined(latestRun); + assert.equal(subagent.runId, latestRun.id); assert.isNotNull(subagent.childThreadId); if (subagent.childThreadId === null) { throw new Error("Continued subagent is missing its child thread"); diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/codex_output.ts b/apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/codex_output.ts new file mode 100644 index 00000000000..00e6e146260 --- /dev/null +++ b/apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/codex_output.ts @@ -0,0 +1,60 @@ +import { assert } from "@effect/vitest"; +import type { ProviderReplayTranscript } from "@t3tools/contracts"; + +import type { OrchestratorV2ScenarioResult } from "../../OrchestratorScenario.ts"; +import { + assertSemanticProjectionIntegrity, + assertUserMessagesInclude, + projectionFor, + SUBAGENT_REUSE_AFTER_IDLE_PROMPT, + SUBAGENT_REUSE_AFTER_IDLE_RESUME_PROMPT, +} from "../shared.ts"; + +export function assertSubagentReuseAfterIdleOutput( + result: OrchestratorV2ScenarioResult, + transcript: ProviderReplayTranscript, +) { + const projection = projectionFor(result, transcript.scenario); + assertSemanticProjectionIntegrity(projection); + assertUserMessagesInclude(projection, [ + SUBAGENT_REUSE_AFTER_IDLE_PROMPT, + SUBAGENT_REUSE_AFTER_IDLE_RESUME_PROMPT, + ]); + + // A duplicate here means the reaped registry was never rehydrated from the + // projection, so the returning agent thread read as a stranger. + assert.lengthOf( + projection.subagents, + 1, + "the session reap caused the reused subagent to be registered as a new identity", + ); + const subagent = projection.subagents[0]!; + + assert.isAtLeast( + subagent.activationCount, + 2, + "the subagent was not re-activated after the session was reaped", + ); + const activations = projection.subagentActivations.filter( + (activation) => activation.subagentId === subagent.id, + ); + assert.isAtLeast(activations.length, 2, "each activation must be recorded separately"); + + const latestRun = projection.runs.at(-1); + assert.isDefined(latestRun); + assert.equal( + subagent.runId, + latestRun.id, + "the re-activated subagent was not adopted by the run driving it", + ); + + assert.isNotNull(subagent.childThreadId); + if (subagent.childThreadId === null) { + throw new Error("Re-activated subagent is missing its child thread"); + } + const childProjection = result.projections.get(subagent.childThreadId); + assert.isDefined(childProjection); + // One child thread carrying both turns proves the original agent thread was + // resumed rather than a second one created alongside it. + assert.isAtLeast(childProjection.providerTurns.length, 2); +} diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/codex_transcript.ndjson b/apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/codex_transcript.ndjson new file mode 100644 index 00000000000..8daa767bfa6 --- /dev/null +++ b/apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/codex_transcript.ndjson @@ -0,0 +1,149 @@ +{"type":"transcript_start","provider":"codex","protocol":"codex.app-server","version":"0.145.0","scenario":"subagent_reuse_after_idle","metadata":{"source":"record-codex-app-server-replay-fixture","fileName":"subagent_reuse_after_idle.ndjson","description":"A root turn spawns a native Codex subagent, then a later root turn re-activates it. Replay advances past the session idle timeout between turns so the adapter registry is empty on reuse."}} +{"type":"expect_outbound","label":"initialize","frame":{"id":1,"method":"initialize","params":{"capabilities":{"experimentalApi":true},"clientInfo":{"name":"t3code_desktop","title":"T3 Code Desktop","version":"0.1.0"}}}} +{"type":"emit_inbound","label":"initialize","frame":{"id":1,"result":{"userAgent":"t3code_desktop/0.145.0 (Mac OS 26.5.2; arm64) unknown (t3code_desktop; 0.1.0)","codexHome":"/Users/shivam/.codex","platformFamily":"unix","platformOs":"macos"}}} +{"type":"expect_outbound","label":"initialized","frame":{"method":"initialized"}} +{"type":"expect_outbound","label":"thread/start","frame":{"id":2,"method":"thread/start","params":{}}} +{"type":"emit_inbound","label":"remoteControl/status/changed","frame":{"method":"remoteControl/status/changed","params":{"status":"disabled","serverName":"shivam-macpro.local","installationId":"6a5680bf-d908-4d8f-b9f4-ee9e1d7089ae","environmentId":null},"emittedAtMs":1785137018968}} +{"type":"emit_inbound","label":"thread/start","frame":{"id":2,"result":{"thread":{"id":"019fa275-3860-7732-8f0b-6d43c27baf50","extra":null,"sessionId":"019fa275-3860-7732-8f0b-6d43c27baf50","forkedFromId":null,"parentThreadId":null,"preview":"","ephemeral":false,"historyMode":"legacy","modelProvider":"openai","createdAt":1785137019,"updatedAt":1785137019,"recencyAt":1785137019,"status":{"type":"idle"},"path":"/Users/shivam/.codex/sessions/2026/07/27/rollout-2026-07-27T12-53-38-019fa275-3860-7732-8f0b-6d43c27baf50.jsonl","cwd":"/Users/shivam/Developer/t3/ccing/open/t3code-2/apps/server","cliVersion":"0.145.0","source":"vscode","canAcceptDirectInput":true,"threadSource":null,"agentNickname":null,"agentRole":null,"gitInfo":null,"name":null,"turns":[]},"model":"gpt-5.6-sol","modelProvider":"openai","serviceTier":"priority","cwd":"/Users/shivam/Developer/t3/ccing/open/t3code-2/apps/server","runtimeWorkspaceRoots":["/Users/shivam/Developer/t3/ccing/open/t3code-2/apps/server"],"instructionSources":["/Users/shivam/.codex/AGENTS.md","/Users/shivam/Developer/t3/ccing/open/t3code-2/AGENTS.md"],"approvalPolicy":"never","approvalsReviewer":"user","sandbox":{"type":"dangerFullAccess"},"activePermissionProfile":null,"reasoningEffort":"high","multiAgentMode":"explicitRequestOnly"}}} +{"type":"expect_outbound","label":"turn/start","frame":{"id":3,"method":"turn/start","params":{"input":[{"text":"Spawn one subagent named idle_probe and have it reply exactly: idle probe first response","type":"text"}],"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50"}}} +{"type":"emit_inbound","label":"thread/started","frame":{"method":"thread/started","params":{"thread":{"id":"019fa275-3860-7732-8f0b-6d43c27baf50","extra":null,"sessionId":"019fa275-3860-7732-8f0b-6d43c27baf50","forkedFromId":null,"parentThreadId":null,"preview":"","ephemeral":false,"historyMode":"legacy","modelProvider":"openai","createdAt":1785137019,"updatedAt":1785137019,"recencyAt":1785137019,"status":{"type":"idle"},"path":"/Users/shivam/.codex/sessions/2026/07/27/rollout-2026-07-27T12-53-38-019fa275-3860-7732-8f0b-6d43c27baf50.jsonl","cwd":"/Users/shivam/Developer/t3/ccing/open/t3code-2/apps/server","cliVersion":"0.145.0","source":"vscode","canAcceptDirectInput":true,"threadSource":null,"agentNickname":null,"agentRole":null,"gitInfo":null,"name":null,"turns":[]}},"emittedAtMs":1785137019108}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"xcodebuildmcp","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137019109}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"codex_apps","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137019109}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"node_repl","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137019109}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"openaiDeveloperDocs","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137019109}} +{"type":"emit_inbound","label":"turn/start","frame":{"id":3,"result":{"turn":{"id":"019fa275-38e7-7a12-9975-235b0e73bc77","items":[],"itemsView":"notLoaded","status":"inProgress","error":null,"startedAt":null,"completedAt":null,"durationMs":null}}}} +{"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","status":{"type":"active","activeFlags":[]}},"emittedAtMs":1785137019148}} +{"type":"emit_inbound","label":"turn/started","frame":{"method":"turn/started","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turn":{"id":"019fa275-38e7-7a12-9975-235b0e73bc77","items":[],"itemsView":"notLoaded","status":"inProgress","error":null,"startedAt":1785137019,"completedAt":null,"durationMs":null}},"emittedAtMs":1785137019148}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"node_repl","status":"ready","error":null,"failureReason":null},"emittedAtMs":1785137019208}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"openaiDeveloperDocs","status":"ready","error":null,"failureReason":null},"emittedAtMs":1785137020206}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"xcodebuildmcp","status":"ready","error":null,"failureReason":null},"emittedAtMs":1785137020728}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"codex_apps","status":"ready","error":null,"failureReason":null},"emittedAtMs":1785137021595}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"userMessage","id":"019fa275-46aa-7841-8d4d-f6884a8e7f83","clientId":null,"content":[{"type":"text","text":"Spawn one subagent named idle_probe and have it reply exactly: idle probe first response","text_elements":[]}]},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","startedAtMs":1785137022634},"emittedAtMs":1785137022634}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"userMessage","id":"019fa275-46aa-7841-8d4d-f6884a8e7f83","clientId":null,"content":[{"type":"text","text":"Spawn one subagent named idle_probe and have it reply exactly: idle probe first response","text_elements":[]}]},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","completedAtMs":1785137022634},"emittedAtMs":1785137022634}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"reasoning","id":"rs_01463f7aca6c9f0e016a6707800dd4819182530528d17657f6","summary":[],"content":[]},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","startedAtMs":1785137024054},"emittedAtMs":1785137024054}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"reasoning","id":"rs_01463f7aca6c9f0e016a6707800dd4819182530528d17657f6","summary":[],"content":[]},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","completedAtMs":1785137024373},"emittedAtMs":1785137024373}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"agentMessage","id":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","text":"","phase":"commentary","memoryCitation":null},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","startedAtMs":1785137024375},"emittedAtMs":1785137024375}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","delta":"I"},"emittedAtMs":1785137024383}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","delta":"’ll"},"emittedAtMs":1785137024383}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","delta":" start"},"emittedAtMs":1785137024383}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","delta":" the"},"emittedAtMs":1785137024383}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","delta":" requested"},"emittedAtMs":1785137024419}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","delta":" sub"},"emittedAtMs":1785137024421}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","delta":"agent"},"emittedAtMs":1785137024421}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","delta":" and"},"emittedAtMs":1785137024422}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","delta":" relay"},"emittedAtMs":1785137024474}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","delta":" its"},"emittedAtMs":1785137024477}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","delta":" exact"},"emittedAtMs":1785137024482}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","delta":" response"},"emittedAtMs":1785137024485}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","delta":"."},"emittedAtMs":1785137024485}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"agentMessage","id":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","text":"I’ll start the requested subagent and relay its exact response.","phase":"commentary","memoryCitation":null},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","completedAtMs":1785137024609},"emittedAtMs":1785137024609}} +{"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","status":{"type":"idle"}},"emittedAtMs":1785137025213}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"subAgentActivity","id":"call_6KAd2XhIg5t3F25Hs8IUu96D","kind":"started","agentThreadId":"019fa275-5034-73b1-a29e-552df3271b27","agentPath":"/root/idle_probe"},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","completedAtMs":1785137025214},"emittedAtMs":1785137025214}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","name":"xcodebuildmcp","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137025215}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","name":"openaiDeveloperDocs","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137025215}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","name":"codex_apps","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137025215}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","name":"node_repl","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137025215}} +{"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","status":{"type":"active","activeFlags":[]}},"emittedAtMs":1785137025218}} +{"type":"emit_inbound","label":"turn/started","frame":{"method":"turn/started","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turn":{"id":"019fa275-50be-7e01-9ce1-a85fe3c2f250","items":[],"itemsView":"notLoaded","status":"inProgress","error":null,"startedAt":1785137025,"completedAt":null,"durationMs":null}},"emittedAtMs":1785137025218}} +{"type":"emit_inbound","label":"thread/tokenUsage/updated","frame":{"method":"thread/tokenUsage/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","tokenUsage":{"total":{"totalTokens":19105,"inputTokens":19025,"cachedInputTokens":0,"cacheWriteInputTokens":0,"outputTokens":80,"reasoningOutputTokens":15},"last":{"totalTokens":19105,"inputTokens":19025,"cachedInputTokens":0,"cacheWriteInputTokens":0,"outputTokens":80,"reasoningOutputTokens":15},"modelContextWindow":258400}},"emittedAtMs":1785137025221}} +{"type":"emit_inbound","label":"account/rateLimits/updated","frame":{"method":"account/rateLimits/updated","params":{"rateLimits":{"limitId":"codex","limitName":null,"primary":{"usedPercent":8,"windowDurationMins":10080,"resetsAt":1785634197},"secondary":null,"credits":{"hasCredits":false,"unlimited":false,"balance":"0"},"individualLimit":null,"spendControlReached":null,"planType":"prolite","rateLimitReachedType":null}},"emittedAtMs":1785137025221}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","name":"node_repl","status":"ready","error":null,"failureReason":null},"emittedAtMs":1785137025288}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","name":"codex_apps","status":"ready","error":null,"failureReason":null},"emittedAtMs":1785137025826}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","name":"xcodebuildmcp","status":"ready","error":null,"failureReason":null},"emittedAtMs":1785137026337}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","name":"openaiDeveloperDocs","status":"ready","error":null,"failureReason":null},"emittedAtMs":1785137026454}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"reasoning","id":"rs_01463f7aca6c9f0e016a670783507481919c0813670bdd3e22","summary":[],"content":[]},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","startedAtMs":1785137027315},"emittedAtMs":1785137027315}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"reasoning","id":"rs_01463f7aca6c9f0e016a670783507481919c0813670bdd3e22","summary":[],"content":[]},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","completedAtMs":1785137027537},"emittedAtMs":1785137027537}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"collabAgentToolCall","id":"call_E7ieRlVqMfllfBpPGi5Dl1UW","tool":"wait","status":"inProgress","senderThreadId":"019fa275-3860-7732-8f0b-6d43c27baf50","receiverThreadIds":[],"prompt":null,"model":null,"reasoningEffort":null,"agentsStates":{}},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","startedAtMs":1785137027699},"emittedAtMs":1785137027699}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"agentMessage","id":"msg_058f48e256fe5748016a6707868b948191b7e425d852b9f4f2","text":"","phase":"final_answer","memoryCitation":null},"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-50be-7e01-9ce1-a85fe3c2f250","startedAtMs":1785137030566},"emittedAtMs":1785137030566}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-50be-7e01-9ce1-a85fe3c2f250","itemId":"msg_058f48e256fe5748016a6707868b948191b7e425d852b9f4f2","delta":"idle"},"emittedAtMs":1785137030566}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-50be-7e01-9ce1-a85fe3c2f250","itemId":"msg_058f48e256fe5748016a6707868b948191b7e425d852b9f4f2","delta":" probe"},"emittedAtMs":1785137030578}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-50be-7e01-9ce1-a85fe3c2f250","itemId":"msg_058f48e256fe5748016a6707868b948191b7e425d852b9f4f2","delta":" first"},"emittedAtMs":1785137030578}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-50be-7e01-9ce1-a85fe3c2f250","itemId":"msg_058f48e256fe5748016a6707868b948191b7e425d852b9f4f2","delta":" response"},"emittedAtMs":1785137030590}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"agentMessage","id":"msg_058f48e256fe5748016a6707868b948191b7e425d852b9f4f2","text":"idle probe first response","phase":"final_answer","memoryCitation":null},"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-50be-7e01-9ce1-a85fe3c2f250","completedAtMs":1785137030698},"emittedAtMs":1785137030698}} +{"type":"emit_inbound","label":"thread/tokenUsage/updated","frame":{"method":"thread/tokenUsage/updated","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-50be-7e01-9ce1-a85fe3c2f250","tokenUsage":{"total":{"totalTokens":22650,"inputTokens":22642,"cachedInputTokens":16128,"cacheWriteInputTokens":0,"outputTokens":8,"reasoningOutputTokens":0},"last":{"totalTokens":22650,"inputTokens":22642,"cachedInputTokens":16128,"cacheWriteInputTokens":0,"outputTokens":8,"reasoningOutputTokens":0},"modelContextWindow":258400}},"emittedAtMs":1785137030774}} +{"type":"emit_inbound","label":"account/rateLimits/updated","frame":{"method":"account/rateLimits/updated","params":{"rateLimits":{"limitId":"codex","limitName":null,"primary":{"usedPercent":8,"windowDurationMins":10080,"resetsAt":1785634197},"secondary":null,"credits":{"hasCredits":false,"unlimited":false,"balance":"0"},"individualLimit":null,"spendControlReached":null,"planType":"prolite","rateLimitReachedType":null}},"emittedAtMs":1785137030774}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"collabAgentToolCall","id":"call_E7ieRlVqMfllfBpPGi5Dl1UW","tool":"wait","status":"completed","senderThreadId":"019fa275-3860-7732-8f0b-6d43c27baf50","receiverThreadIds":[],"prompt":null,"model":null,"reasoningEffort":null,"agentsStates":{}},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","completedAtMs":1785137030823},"emittedAtMs":1785137030824}} +{"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","status":{"type":"idle"}},"emittedAtMs":1785137030824}} +{"type":"emit_inbound","label":"turn/completed","frame":{"method":"turn/completed","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turn":{"id":"019fa275-50be-7e01-9ce1-a85fe3c2f250","items":[],"itemsView":"notLoaded","status":"completed","error":null,"startedAt":1785137025,"completedAt":1785137030,"durationMs":5607}},"emittedAtMs":1785137030824}} +{"type":"emit_inbound","label":"thread/tokenUsage/updated","frame":{"method":"thread/tokenUsage/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","tokenUsage":{"total":{"totalTokens":38262,"inputTokens":38152,"cachedInputTokens":18176,"cacheWriteInputTokens":0,"outputTokens":110,"reasoningOutputTokens":22},"last":{"totalTokens":19157,"inputTokens":19127,"cachedInputTokens":18176,"cacheWriteInputTokens":0,"outputTokens":30,"reasoningOutputTokens":7},"modelContextWindow":258400}},"emittedAtMs":1785137030825}} +{"type":"emit_inbound","label":"account/rateLimits/updated","frame":{"method":"account/rateLimits/updated","params":{"rateLimits":{"limitId":"codex","limitName":null,"primary":{"usedPercent":8,"windowDurationMins":10080,"resetsAt":1785634197},"secondary":null,"credits":{"hasCredits":false,"unlimited":false,"balance":"0"},"individualLimit":null,"spendControlReached":null,"planType":"prolite","rateLimitReachedType":null}},"emittedAtMs":1785137030825}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"agentMessage","id":"msg_01463f7aca6c9f0e016a670788c3a48191b0cf4218c3aa4940","text":"","phase":"final_answer","memoryCitation":null},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","startedAtMs":1785137032765},"emittedAtMs":1785137032765}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a670788c3a48191b0cf4218c3aa4940","delta":"idle"},"emittedAtMs":1785137032767}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a670788c3a48191b0cf4218c3aa4940","delta":" probe"},"emittedAtMs":1785137032768}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a670788c3a48191b0cf4218c3aa4940","delta":" first"},"emittedAtMs":1785137032776}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a670788c3a48191b0cf4218c3aa4940","delta":" response"},"emittedAtMs":1785137032790}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"agentMessage","id":"msg_01463f7aca6c9f0e016a670788c3a48191b0cf4218c3aa4940","text":"idle probe first response","phase":"final_answer","memoryCitation":null},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","completedAtMs":1785137032874},"emittedAtMs":1785137032874}} +{"type":"emit_inbound","label":"thread/tokenUsage/updated","frame":{"method":"thread/tokenUsage/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","tokenUsage":{"total":{"totalTokens":57492,"inputTokens":57374,"cachedInputTokens":36352,"cacheWriteInputTokens":0,"outputTokens":118,"reasoningOutputTokens":22},"last":{"totalTokens":19230,"inputTokens":19222,"cachedInputTokens":18176,"cacheWriteInputTokens":0,"outputTokens":8,"reasoningOutputTokens":0},"modelContextWindow":258400}},"emittedAtMs":1785137032924}} +{"type":"emit_inbound","label":"account/rateLimits/updated","frame":{"method":"account/rateLimits/updated","params":{"rateLimits":{"limitId":"codex","limitName":null,"primary":{"usedPercent":8,"windowDurationMins":10080,"resetsAt":1785634197},"secondary":null,"credits":{"hasCredits":false,"unlimited":false,"balance":"0"},"individualLimit":null,"spendControlReached":null,"planType":"prolite","rateLimitReachedType":null}},"emittedAtMs":1785137032924}} +{"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","status":{"type":"idle"}},"emittedAtMs":1785137032925}} +{"type":"emit_inbound","label":"turn/completed","frame":{"method":"turn/completed","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turn":{"id":"019fa275-38e7-7a12-9975-235b0e73bc77","items":[],"itemsView":"notLoaded","status":"completed","error":null,"startedAt":1785137019,"completedAt":1785137032,"durationMs":13811}},"emittedAtMs":1785137032925}} +{"type":"expect_outbound","label":"turn/start","frame":{"id":4,"method":"turn/start","params":{"input":[{"text":"Send another message to the same idle_probe subagent. Do not spawn a replacement. Have it reply exactly: idle probe resumed response","type":"text"}],"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50"}}} +{"type":"emit_inbound","label":"turn/start","frame":{"id":4,"result":{"turn":{"id":"019fa275-6edd-7d23-932f-9a8bcd630dce","items":[],"itemsView":"notLoaded","status":"inProgress","error":null,"startedAt":null,"completedAt":null,"durationMs":null}}}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"codex_apps","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137032927}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"openaiDeveloperDocs","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137032927}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"node_repl","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137032927}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"xcodebuildmcp","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137032927}} +{"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","status":{"type":"active","activeFlags":[]}},"emittedAtMs":1785137032939}} +{"type":"emit_inbound","label":"turn/started","frame":{"method":"turn/started","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turn":{"id":"019fa275-6edd-7d23-932f-9a8bcd630dce","items":[],"itemsView":"notLoaded","status":"inProgress","error":null,"startedAt":1785137032,"completedAt":null,"durationMs":null}},"emittedAtMs":1785137032939}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"node_repl","status":"ready","error":null,"failureReason":null},"emittedAtMs":1785137033024}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"codex_apps","status":"ready","error":null,"failureReason":null},"emittedAtMs":1785137033516}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"openaiDeveloperDocs","status":"ready","error":null,"failureReason":null},"emittedAtMs":1785137034028}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"userMessage","id":"019fa275-7333-71c1-8c45-67070525dbe9","clientId":null,"content":[{"type":"text","text":"Send another message to the same idle_probe subagent. Do not spawn a replacement. Have it reply exactly: idle probe resumed response","text_elements":[]}]},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","startedAtMs":1785137034035},"emittedAtMs":1785137034035}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"userMessage","id":"019fa275-7333-71c1-8c45-67070525dbe9","clientId":null,"content":[{"type":"text","text":"Send another message to the same idle_probe subagent. Do not spawn a replacement. Have it reply exactly: idle probe resumed response","text_elements":[]}]},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","completedAtMs":1785137034035},"emittedAtMs":1785137034035}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"xcodebuildmcp","status":"ready","error":null,"failureReason":null},"emittedAtMs":1785137034079}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"reasoning","id":"rs_0c64722b069256f7016a67078c265c8191a2be05492bb9f2f8","summary":[],"content":[]},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","startedAtMs":1785137036152},"emittedAtMs":1785137036152}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"reasoning","id":"rs_0c64722b069256f7016a67078c265c8191a2be05492bb9f2f8","summary":[],"content":[]},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","completedAtMs":1785137036530},"emittedAtMs":1785137036530}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"agentMessage","id":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","text":"","phase":"commentary","memoryCitation":null},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","startedAtMs":1785137036532},"emittedAtMs":1785137036532}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":"I"},"emittedAtMs":1785137036535}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":"’ll"},"emittedAtMs":1785137036537}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":" resume"},"emittedAtMs":1785137036539}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":" the"},"emittedAtMs":1785137036540}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":" existing"},"emittedAtMs":1785137036543}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":" `"},"emittedAtMs":1785137036544}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":"idle"},"emittedAtMs":1785137036736}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":"_probe"},"emittedAtMs":1785137036738}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":"`"},"emittedAtMs":1785137036739}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":" sub"},"emittedAtMs":1785137036739}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":"agent"},"emittedAtMs":1785137036773}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":"—"},"emittedAtMs":1785137036784}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":"no"},"emittedAtMs":1785137036784}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":" replacement"},"emittedAtMs":1785137036789}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":"—and"},"emittedAtMs":1785137036833}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":" relay"},"emittedAtMs":1785137036839}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":" its"},"emittedAtMs":1785137036839}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":" exact"},"emittedAtMs":1785137036839}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":" response"},"emittedAtMs":1785137036841}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":"."},"emittedAtMs":1785137036841}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"agentMessage","id":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","text":"I’ll resume the existing `idle_probe` subagent—no replacement—and relay its exact response.","phase":"commentary","memoryCitation":null},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","completedAtMs":1785137036854},"emittedAtMs":1785137036854}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"subAgentActivity","id":"call_Eo2mTZMc8KumZl7Gc0tGySg5","kind":"interacted","agentThreadId":"019fa275-5034-73b1-a29e-552df3271b27","agentPath":"/root/idle_probe"},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","completedAtMs":1785137037205},"emittedAtMs":1785137037205}} +{"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","status":{"type":"active","activeFlags":[]}},"emittedAtMs":1785137037209}} +{"type":"emit_inbound","label":"turn/started","frame":{"method":"turn/started","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turn":{"id":"019fa275-7f95-7181-b5c0-e2959a15e3c1","items":[],"itemsView":"notLoaded","status":"inProgress","error":null,"startedAt":1785137037,"completedAt":null,"durationMs":null}},"emittedAtMs":1785137037209}} +{"type":"emit_inbound","label":"thread/tokenUsage/updated","frame":{"method":"thread/tokenUsage/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","tokenUsage":{"total":{"totalTokens":79941,"inputTokens":79741,"cachedInputTokens":36352,"cacheWriteInputTokens":0,"outputTokens":200,"reasoningOutputTokens":35},"last":{"totalTokens":22449,"inputTokens":22367,"cachedInputTokens":0,"cacheWriteInputTokens":0,"outputTokens":82,"reasoningOutputTokens":13},"modelContextWindow":258400}},"emittedAtMs":1785137037257}} +{"type":"emit_inbound","label":"account/rateLimits/updated","frame":{"method":"account/rateLimits/updated","params":{"rateLimits":{"limitId":"codex","limitName":null,"primary":{"usedPercent":8,"windowDurationMins":10080,"resetsAt":1785634197},"secondary":null,"credits":{"hasCredits":false,"unlimited":false,"balance":"0"},"individualLimit":null,"spendControlReached":null,"planType":"prolite","rateLimitReachedType":null}},"emittedAtMs":1785137037257}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"reasoning","id":"rs_058f48e256fe5748016a67078f36a8819195c0b103e3d41f47","summary":[],"content":[]},"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-7f95-7181-b5c0-e2959a15e3c1","startedAtMs":1785137039233},"emittedAtMs":1785137039233}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"reasoning","id":"rs_058f48e256fe5748016a67078f36a8819195c0b103e3d41f47","summary":[],"content":[]},"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-7f95-7181-b5c0-e2959a15e3c1","completedAtMs":1785137039357},"emittedAtMs":1785137039357}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"agentMessage","id":"msg_058f48e256fe5748016a67078f56148191a87fca200a919c40","text":"","phase":"final_answer","memoryCitation":null},"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-7f95-7181-b5c0-e2959a15e3c1","startedAtMs":1785137039360},"emittedAtMs":1785137039360}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-7f95-7181-b5c0-e2959a15e3c1","itemId":"msg_058f48e256fe5748016a67078f56148191a87fca200a919c40","delta":"idle"},"emittedAtMs":1785137039363}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-7f95-7181-b5c0-e2959a15e3c1","itemId":"msg_058f48e256fe5748016a67078f56148191a87fca200a919c40","delta":" probe"},"emittedAtMs":1785137039370}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-7f95-7181-b5c0-e2959a15e3c1","itemId":"msg_058f48e256fe5748016a67078f56148191a87fca200a919c40","delta":" resumed"},"emittedAtMs":1785137039380}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-7f95-7181-b5c0-e2959a15e3c1","itemId":"msg_058f48e256fe5748016a67078f56148191a87fca200a919c40","delta":" response"},"emittedAtMs":1785137039398}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"reasoning","id":"rs_09cd91668d0353dd016a67078f80048191b5cc53d08e8bca89","summary":[],"content":[]},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","startedAtMs":1785137039500},"emittedAtMs":1785137039500}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"agentMessage","id":"msg_058f48e256fe5748016a67078f56148191a87fca200a919c40","text":"idle probe resumed response","phase":"final_answer","memoryCitation":null},"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-7f95-7181-b5c0-e2959a15e3c1","completedAtMs":1785137039533},"emittedAtMs":1785137039533}} +{"type":"emit_inbound","label":"thread/tokenUsage/updated","frame":{"method":"thread/tokenUsage/updated","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-7f95-7181-b5c0-e2959a15e3c1","tokenUsage":{"total":{"totalTokens":45370,"inputTokens":45343,"cachedInputTokens":38400,"cacheWriteInputTokens":0,"outputTokens":27,"reasoningOutputTokens":9},"last":{"totalTokens":22720,"inputTokens":22701,"cachedInputTokens":22272,"cacheWriteInputTokens":0,"outputTokens":19,"reasoningOutputTokens":9},"modelContextWindow":258400}},"emittedAtMs":1785137039563}} +{"type":"emit_inbound","label":"account/rateLimits/updated","frame":{"method":"account/rateLimits/updated","params":{"rateLimits":{"limitId":"codex","limitName":null,"primary":{"usedPercent":8,"windowDurationMins":10080,"resetsAt":1785634197},"secondary":null,"credits":{"hasCredits":false,"unlimited":false,"balance":"0"},"individualLimit":null,"spendControlReached":null,"planType":"prolite","rateLimitReachedType":null}},"emittedAtMs":1785137039563}} +{"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","status":{"type":"idle"}},"emittedAtMs":1785137039566}} +{"type":"emit_inbound","label":"turn/completed","frame":{"method":"turn/completed","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turn":{"id":"019fa275-7f95-7181-b5c0-e2959a15e3c1","items":[],"itemsView":"notLoaded","status":"completed","error":null,"startedAt":1785137037,"completedAt":1785137039,"durationMs":2358}},"emittedAtMs":1785137039566}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"reasoning","id":"rs_09cd91668d0353dd016a67078f80048191b5cc53d08e8bca89","summary":[],"content":[]},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","completedAtMs":1785137039817},"emittedAtMs":1785137039817}} +{"type":"emit_inbound","label":"thread/tokenUsage/updated","frame":{"method":"thread/tokenUsage/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","tokenUsage":{"total":{"totalTokens":79941,"inputTokens":79741,"cachedInputTokens":36352,"cacheWriteInputTokens":0,"outputTokens":200,"reasoningOutputTokens":35},"last":{"totalTokens":22449,"inputTokens":22367,"cachedInputTokens":0,"cacheWriteInputTokens":0,"outputTokens":82,"reasoningOutputTokens":13},"modelContextWindow":258400}},"emittedAtMs":1785137039820}} +{"type":"emit_inbound","label":"account/rateLimits/updated","frame":{"method":"account/rateLimits/updated","params":{"rateLimits":{"limitId":"codex","limitName":null,"primary":{"usedPercent":8,"windowDurationMins":10080,"resetsAt":1785634198},"secondary":null,"credits":{"hasCredits":false,"unlimited":false,"balance":"0"},"individualLimit":null,"spendControlReached":null,"planType":"prolite","rateLimitReachedType":null}},"emittedAtMs":1785137039820}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"agentMessage","id":"msg_001550b2df635fbe016a670791c46881918e3167f585df353b","text":"","phase":"final_answer","memoryCitation":null},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","startedAtMs":1785137041773},"emittedAtMs":1785137041773}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_001550b2df635fbe016a670791c46881918e3167f585df353b","delta":"idle"},"emittedAtMs":1785137041773}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_001550b2df635fbe016a670791c46881918e3167f585df353b","delta":" probe"},"emittedAtMs":1785137041773}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_001550b2df635fbe016a670791c46881918e3167f585df353b","delta":" resumed"},"emittedAtMs":1785137041797}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_001550b2df635fbe016a670791c46881918e3167f585df353b","delta":" response"},"emittedAtMs":1785137041805}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"agentMessage","id":"msg_001550b2df635fbe016a670791c46881918e3167f585df353b","text":"idle probe resumed response","phase":"final_answer","memoryCitation":null},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","completedAtMs":1785137041975},"emittedAtMs":1785137041975}} +{"type":"emit_inbound","label":"thread/tokenUsage/updated","frame":{"method":"thread/tokenUsage/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","tokenUsage":{"total":{"totalTokens":102942,"inputTokens":102734,"cachedInputTokens":58624,"cacheWriteInputTokens":0,"outputTokens":208,"reasoningOutputTokens":35},"last":{"totalTokens":23001,"inputTokens":22993,"cachedInputTokens":22272,"cacheWriteInputTokens":0,"outputTokens":8,"reasoningOutputTokens":0},"modelContextWindow":258400}},"emittedAtMs":1785137041989}} +{"type":"emit_inbound","label":"account/rateLimits/updated","frame":{"method":"account/rateLimits/updated","params":{"rateLimits":{"limitId":"codex","limitName":null,"primary":{"usedPercent":8,"windowDurationMins":10080,"resetsAt":1785634197},"secondary":null,"credits":{"hasCredits":false,"unlimited":false,"balance":"0"},"individualLimit":null,"spendControlReached":null,"planType":"prolite","rateLimitReachedType":null}},"emittedAtMs":1785137041989}} +{"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","status":{"type":"idle"}},"emittedAtMs":1785137041991}} +{"type":"emit_inbound","label":"turn/completed","frame":{"method":"turn/completed","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turn":{"id":"019fa275-6edd-7d23-932f-9a8bcd630dce","items":[],"itemsView":"notLoaded","status":"completed","error":null,"startedAt":1785137032,"completedAt":1785137041,"durationMs":9053}},"emittedAtMs":1785137041991}} +{"type":"runtime_exit","status":"success"} diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/input.ts b/apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/input.ts new file mode 100644 index 00000000000..4b23f65b0df --- /dev/null +++ b/apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/input.ts @@ -0,0 +1,31 @@ +import { + SUBAGENT_REUSE_AFTER_IDLE_PROMPT, + SUBAGENT_REUSE_AFTER_IDLE_RESUME_PROMPT, + type OrchestratorFixtureInput, +} from "../shared.ts"; + +/** + * Regression for subagent reuse across a session reap. + * + * Adapters track spawned subagents in a process-local map, so the reaper + * releasing the session runtime wipes every identity it knows about. Without + * rehydrating them from the projection, the returning agent thread is + * unrecognised on the next turn and gets registered as a brand new subagent — + * the user sees a duplicate appear and the original frozen forever. + * + * Not registered yet. The recorded transcript does contain a real reuse, but + * replaying it leaves the second run running forever so the scenario never + * reaches thread idle, and that is unrelated to the reuse routing here — it + * reproduces with those changes reverted. An advance_clock step to cross the + * 30-minute idle timeout (which is what would exercise rehydration rather than + * plain reuse) fails the same way, so it is left out until the hang is + * understood. + */ +export function subagentReuseAfterIdleInput(): OrchestratorFixtureInput { + return { + steps: [ + { type: "message", text: SUBAGENT_REUSE_AFTER_IDLE_PROMPT }, + { type: "message", text: SUBAGENT_REUSE_AFTER_IDLE_RESUME_PROMPT }, + ], + }; +} From 5fcaf68008a585aadf8575777288e3784bb38861 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Wed, 29 Jul 2026 01:03:04 +0530 Subject: [PATCH 2/8] test(orchestration-v2): cover subagent registry rehydration after a restart A fresh adapter is the post-restart state: the process-local registry is empty, so a returning native child thread can only be recognised through the identities the projection hands back in existingSubagents. Drives a transcript with no spawn frame, seeds one idle subagent, and asserts the returning thread reuses that identity, continues the activation count and ordinal rather than restarting them, and keeps the usage recorded before the restart. Paired with a negative control that replays the same transcript unseeded. Co-Authored-By: Claude Opus 5 (1M context) --- .../Adapters/CodexAdapterV2.test.ts | 293 ++++++++++++++++++ 1 file changed, 293 insertions(+) diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index ca6c0a44b55..8b33cd03fef 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -17,6 +17,7 @@ import { RunAttemptId, RunId, ThreadId, + TurnItemId, } from "@t3tools/contracts"; import { assert, describe, it } from "@effect/vitest"; import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess"; @@ -41,6 +42,7 @@ import { ProviderAdapterOpenSessionError, ProviderAdapterV2RuntimePolicy, type ProviderAdapterV2Event, + type ProviderAdapterV2ExistingSubagent, type ProviderAdapterV2TurnInput, } from "../ProviderAdapter.ts"; import type { ProviderContinuationRequest } from "../ProviderContinuationRequests.ts"; @@ -3635,4 +3637,295 @@ describe("CodexAdapterV2 post-settle continuation", () => { }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), ), ); + + const RESTART_SCENARIO = "codex-restart-rehydrate"; + const RESTART_NATIVE_THREAD = "native-codex-restart-thread"; + const RESTART_NATIVE_TURN = "native-codex-restart-root-turn"; + const RESTART_CHILD_THREAD = "native-codex-restart-child-thread"; + const RESTART_CHILD_TURN = "native-codex-restart-child-turn-2"; + const RESTART_TOOL_CALL = "call-codex-restart-spawn"; + const RESTART_PROMPT = "Nudge the agent from before the restart."; + const RESTART_SUBAGENT_ID = NodeId.make("node-codex-restart-subagent"); + + const restartChildFrame = (input: { + readonly label: string; + readonly method: string; + readonly params: Record; + }): CodexReplay.CodexAppServerReplayEntry => ({ + type: "emit_inbound", + label: input.label, + frame: { method: input.method, params: input.params }, + }); + + const restartTranscript = makeCodexReplayTranscript({ + scenario: RESTART_SCENARIO, + entries: [ + ...codexReplayPreamble({ + nativeThreadId: RESTART_NATIVE_THREAD, + nativeTurnId: RESTART_NATIVE_TURN, + prompt: RESTART_PROMPT, + }), + // No spawn frame: this process never saw the agent start. It only sees + // the pre-existing native thread come back to life. + restartChildFrame({ + label: "turn/started/child", + method: "turn/started", + params: { + threadId: RESTART_CHILD_THREAD, + turn: makeCodexReplayTurn({ id: RESTART_CHILD_TURN, status: "inProgress" }), + }, + }), + restartChildFrame({ + label: "item/completed/child-answer", + method: "item/completed", + params: { + item: { + type: "agentMessage", + id: "child-restart-answer", + text: "CODEX_REHYDRATED_DONE", + phase: "final_answer", + memoryCitation: null, + }, + threadId: RESTART_CHILD_THREAD, + turnId: RESTART_CHILD_TURN, + completedAtMs: 1782622482000, + }, + }), + restartChildFrame({ + label: "turn/completed/child", + method: "turn/completed", + params: { + threadId: RESTART_CHILD_THREAD, + turn: makeCodexReplayTurn({ id: RESTART_CHILD_TURN, status: "completed" }), + }, + }), + restartChildFrame({ + label: "thread/tokenUsage/updated/child", + method: "thread/tokenUsage/updated", + params: { + threadId: RESTART_CHILD_THREAD, + turnId: RESTART_CHILD_TURN, + tokenUsage: { + total: { + totalTokens: 250, + inputTokens: 200, + cachedInputTokens: 0, + outputTokens: 50, + reasoningOutputTokens: 0, + }, + // Codex reports `total` per child thread, not per activation, so + // this snapshot already contains the 100 recorded before the + // restart; `last` is what this activation added. + last: { + totalTokens: 150, + inputTokens: 120, + cachedInputTokens: 0, + outputTokens: 30, + reasoningOutputTokens: 0, + }, + }, + }, + }), + restartChildFrame({ + label: "item/completed/root-answer", + method: "item/completed", + params: { + item: { + type: "agentMessage", + id: "root-answer-restart", + text: "NUDGED", + phase: "final_answer", + memoryCitation: null, + }, + threadId: RESTART_NATIVE_THREAD, + turnId: RESTART_NATIVE_TURN, + completedAtMs: 1782622483000, + }, + }), + restartChildFrame({ + label: "turn/completed/root", + method: "turn/completed", + params: { + threadId: RESTART_NATIVE_THREAD, + turn: makeCodexReplayTurn({ id: RESTART_NATIVE_TURN, status: "completed" }), + }, + }), + ], + }); + + /** + * The subagent as the projection holds it after a restart: settled at idle, + * still bound to its child thread and native item, one activation recorded. + */ + const makeRestartExistingSubagent = (input: { + readonly parentThreadId: ThreadId; + readonly parentProviderThread: OrchestrationV2ProviderThread; + readonly now: DateTime.Utc; + }): ProviderAdapterV2ExistingSubagent => { + const childThreadId = ThreadId.make("thread-codex-restart-child"); + const childProviderThread: OrchestrationV2ProviderThread = { + id: ProviderThreadId.make("provider-thread-codex-restart-child"), + driver: CODEX_DRIVER_KIND, + providerInstanceId: CODEX_DEFAULT_INSTANCE_ID, + providerSessionId: input.parentProviderThread.providerSessionId, + appThreadId: childThreadId, + ownerNodeId: null, + nativeThreadRef: { + driver: CODEX_DRIVER_KIND, + nativeId: RESTART_CHILD_THREAD, + strength: "strong", + }, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: null, + lastRunOrdinal: null, + handoffIds: [], + forkedFrom: null, + createdAt: input.now, + updatedAt: input.now, + }; + return { + subagent: { + id: RESTART_SUBAGENT_ID, + threadId: input.parentThreadId, + runId: RunId.make("run-before-restart"), + parentNodeId: NodeId.make("node-before-restart-root"), + origin: "provider_native", + createdBy: "agent", + driver: CODEX_DRIVER_KIND, + providerInstanceId: CODEX_DEFAULT_INSTANCE_ID, + providerThreadId: childProviderThread.id, + childThreadId, + nativeTaskRef: { + driver: CODEX_DRIVER_KIND, + nativeId: `${RESTART_TOOL_CALL}:${RESTART_CHILD_THREAD}`, + strength: "strong", + }, + prompt: "Audit the parser.", + title: "Parser audit", + model: "gpt-5.4", + kind: "subagent", + role: { name: "general-purpose", source: "app_default" }, + status: "idle", + result: "CODEX_BEFORE_RESTART_DONE", + usage: { totalTokens: 100 }, + currentActivationId: null, + activationCount: 1, + workflow: null, + workflowMembership: null, + recentActivity: [], + startedAt: input.now, + completedAt: input.now, + updatedAt: input.now, + }, + childThread: { + ...makeCodexTestAppThread({ + threadId: childThreadId, + providerThread: childProviderThread, + now: input.now, + }), + lineage: { + parentThreadId: input.parentThreadId, + relationshipToParent: "subagent" as const, + rootThreadId: input.parentThreadId, + }, + }, + childProviderThread, + turnItemId: TurnItemId.make("turn-item-codex-restart-subagent"), + turnItemOrdinal: 2, + ordinal: 1, + }; + }; + + it.effect("reuses a projection-known subagent after the registry is lost", () => + Effect.scoped( + Effect.gen(function* () { + // A fresh adapter is exactly the post-restart state: the process-local + // registry is empty, so the only way to recognise the returning native + // thread is the projection handing it back through existingSubagents. + const harness = yield* makeCodexReplayHarness(restartTranscript); + const now = yield* DateTime.now; + const existing = makeRestartExistingSubagent({ + parentThreadId: harness.threadId, + parentProviderThread: harness.providerThread, + now, + }); + + yield* harness.runtime.startTurn({ + ...makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-codex-restart"), + text: RESTART_PROMPT, + }), + existingSubagents: [existing], + }); + + yield* awaitUntil( + () => + harness + .subagentUpdates() + .some((event) => event.subagent.result === "CODEX_REHYDRATED_DONE"), + "rehydrated subagent result", + ); + + const updates = harness.subagentUpdates(); + // One identity, and it is the one from before the restart. + assert.deepStrictEqual( + [...new Set(updates.map((event) => event.subagent.id))], + [RESTART_SUBAGENT_ID], + "the returning agent must not be spawned as a second identity", + ); + const latest = updates.at(-1)?.subagent; + assert.equal(latest?.result, "CODEX_REHYDRATED_DONE"); + assert.equal(latest?.childThreadId, existing.childThread.id); + // Continues the prior activation rather than restarting the count. + assert.equal(latest?.activationCount, 2); + // Usage carries across the restart instead of resetting: the seeded + // 100 is not lost, and the thread-cumulative 250 is not double-counted + // on top of it. + assert.equal(latest?.usage?.totalTokens, 250); + // The new activation takes the next ordinal, not a colliding ordinal 1. + const activations = harness.subagentActivationUpdates(); + assert.deepStrictEqual( + [...new Set(activations.map((event) => event.activation.ordinal))], + [2], + ); + assert.deepStrictEqual( + [...new Set(activations.map((event) => event.activation.subagentId))], + [RESTART_SUBAGENT_ID], + ); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("does not adopt an unknown native thread when nothing is rehydrated", () => + Effect.scoped( + Effect.gen(function* () { + // Negative control for the test above: the identical transcript with no + // existingSubagents. Without the seed the returning thread is not the + // prior agent, so the assertion above is proving the seed, not the + // transcript. + const harness = yield* makeCodexReplayHarness(restartTranscript); + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-codex-restart-unseeded"), + text: RESTART_PROMPT, + }), + ); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "root turn terminal"); + + assert.notInclude( + harness.subagentUpdates().map((event) => String(event.subagent.id)), + String(RESTART_SUBAGENT_ID), + ); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); }); From 30af905dfc4ebd16e06aecfe7bf2cb1fbebedfdd Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Wed, 29 Jul 2026 06:45:11 +0530 Subject: [PATCH 3/8] fix(orchestration-v2): harden reused subagent recovery Co-Authored-By: Claude --- .../ProviderTurnStartService.test.ts | 296 ++++++++++++++++++ .../ProviderTurnStartService.ts | 94 +++--- .../RunExecutionService.test.ts | 105 +++++++ .../orchestration-v2/RunExecutionService.ts | 23 +- 4 files changed, 466 insertions(+), 52 deletions(-) create mode 100644 apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts new file mode 100644 index 00000000000..a481f23ad6b --- /dev/null +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts @@ -0,0 +1,296 @@ +import { assert, it } from "@effect/vitest"; +import { + CheckpointScopeId, + MessageId, + NodeId, + type OrchestrationV2ThreadProjection, + ProviderDriverKind, + ProviderInstanceId, + ProviderSessionId, + ProviderThreadId, + RunAttemptId, + RunId, + ThreadId, + TurnItemId, +} from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Ref from "effect/Ref"; + +import { ContextHandoffServiceV2 } from "./ContextHandoffService.ts"; +import { EventSinkV2 } from "./EventSink.ts"; +import { layer as idAllocatorLayer } from "./IdAllocator.ts"; +import type { ProviderAdapterV2SessionRuntime } from "./ProviderAdapter.ts"; +import { + ProjectionStoreReadError, + ProjectionStoreThreadNotFoundError, + ProjectionStoreV2, + type ProjectionStoreV2Error, +} from "./ProjectionStore.ts"; +import { ProviderSessionManagerV2 } from "./ProviderSessionManager.ts"; +import { + layer as providerTurnStartLayer, + ProviderTurnStartServiceV2, +} from "./ProviderTurnStartService.ts"; +import { RunExecutionServiceV2 } from "./RunExecutionService.ts"; +import { RuntimePolicyV2 } from "./RuntimePolicy.ts"; + +const driver = ProviderDriverKind.make("codex"); +const threadId = ThreadId.make("thread:provider-turn-start"); +const childThreadId = ThreadId.make("thread:provider-turn-start:child"); +const runId = RunId.make("run:provider-turn-start"); +const rootNodeId = NodeId.make("node:provider-turn-start:root"); +const subagentId = NodeId.make("node:provider-turn-start:subagent"); +const attemptId = RunAttemptId.make("attempt:provider-turn-start"); +const providerThreadId = ProviderThreadId.make("provider-thread:provider-turn-start"); +const childProviderThreadId = ProviderThreadId.make("provider-thread:provider-turn-start:child"); +const providerSessionId = ProviderSessionId.make("provider-session:provider-turn-start"); +const providerInstanceId = ProviderInstanceId.make("codex"); +const messageId = MessageId.make("message:provider-turn-start"); +const checkpointScopeId = CheckpointScopeId.make("checkpoint-scope:provider-turn-start"); + +function makeRootProjection(now: DateTime.Utc): OrchestrationV2ThreadProjection { + return { + thread: { + id: threadId, + runtimeMode: "full-access", + interactionMode: "default", + worktreePath: "/tmp/provider-turn-start", + }, + runs: [ + { + id: runId, + threadId, + ordinal: 1, + status: "starting", + rootNodeId, + activeAttemptId: attemptId, + providerThreadId, + providerInstanceId, + userMessageId: messageId, + modelSelection: { instanceId: providerInstanceId }, + }, + ], + nodes: [ + { + id: rootNodeId, + checkpointScopeId, + }, + ], + attempts: [ + { + id: attemptId, + providerTurnId: null, + }, + ], + providerThreads: [ + { + id: providerThreadId, + providerSessionId, + nativeThreadRef: { + driver, + nativeId: "native-root-thread", + strength: "strong", + }, + ownerNodeId: rootNodeId, + firstRunOrdinal: 1, + lastRunOrdinal: 1, + handoffIds: [], + forkedFrom: null, + createdAt: now, + }, + ], + messages: [ + { + id: messageId, + text: "continue the agent", + attachments: [], + createdBy: "user", + creationSource: "user", + }, + ], + checkpointScopes: [{ id: checkpointScopeId }], + contextHandoffs: [], + contextTransfers: [], + providerSessions: [], + subagents: [ + { + id: subagentId, + status: "idle", + childThreadId, + providerThreadId: childProviderThreadId, + }, + ], + turnItems: [ + { + id: TurnItemId.make("turn-item:provider-turn-start:subagent"), + type: "subagent", + subagentId, + ordinal: 1, + }, + ], + providerTurns: [], + } as unknown as OrchestrationV2ThreadProjection; +} + +function makeChildProjection(now: DateTime.Utc): OrchestrationV2ThreadProjection { + return { + thread: { id: childThreadId }, + providerThreads: [ + { + id: childProviderThreadId, + nativeThreadRef: { + driver, + nativeId: "native-child-thread", + strength: "strong", + }, + createdAt: now, + }, + ], + } as unknown as OrchestrationV2ThreadProjection; +} + +function makeTestLayer(input: { + readonly rootProjection: OrchestrationV2ThreadProjection; + readonly childProjection: Effect.Effect; + readonly existingSubagentCounts: Ref.Ref>; + readonly providerSessionOpens: Ref.Ref; + readonly runningWrites: Ref.Ref; +}) { + const session = { + driver, + providerSession: { id: providerSessionId }, + resumeThread: ({ providerThread }: { readonly providerThread: unknown }) => + Effect.succeed(providerThread), + } as unknown as ProviderAdapterV2SessionRuntime; + + return providerTurnStartLayer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(ContextHandoffServiceV2)({}), + Layer.mock(EventSinkV2)({ + writeIfRunCurrent: () => + Ref.update(input.runningWrites, (count) => count + 1).pipe( + Effect.as({ committed: true, storedEvents: [] }), + ), + }), + idAllocatorLayer, + Layer.mock(ProjectionStoreV2)({ + getThreadProjection: (requestedThreadId) => + requestedThreadId === threadId + ? Effect.succeed(input.rootProjection) + : input.childProjection, + }), + Layer.mock(ProviderSessionManagerV2)({ + open: () => + Ref.update(input.providerSessionOpens, (count) => count + 1).pipe(Effect.as(session)), + }), + Layer.mock(RunExecutionServiceV2)({ + startRootRun: (startInput) => + Ref.update(input.existingSubagentCounts, (counts) => [ + ...counts, + startInput.existingSubagents?.length ?? 0, + ]), + }), + Layer.mock(RuntimePolicyV2)({ + resolve: () => + Effect.succeed({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: "/tmp/provider-turn-start", + }), + }), + ), + ), + ); +} + +it.effect("fails before provider turn start when child projection rehydration cannot be read", () => + Effect.gen(function* () { + const now = yield* DateTime.now; + const existingSubagentCounts = yield* Ref.make>([]); + const providerSessionOpens = yield* Ref.make(0); + const runningWrites = yield* Ref.make(0); + const readError = new ProjectionStoreReadError({ + threadId: childThreadId, + cause: "temporary database failure", + }); + const error = yield* Effect.gen(function* () { + const service = yield* ProviderTurnStartServiceV2; + return yield* service.start({ threadId, runId }).pipe(Effect.flip); + }).pipe( + Effect.provide( + makeTestLayer({ + rootProjection: makeRootProjection(now), + childProjection: Effect.fail(readError), + existingSubagentCounts, + providerSessionOpens, + runningWrites, + }), + ), + ); + + assert.equal(error._tag, "ProviderTurnStartError"); + assert.strictEqual(error.cause, readError); + assert.deepEqual(yield* Ref.get(existingSubagentCounts), []); + assert.equal(yield* Ref.get(providerSessionOpens), 0); + assert.equal(yield* Ref.get(runningWrites), 0); + }), +); + +it.effect("skips a missing child projection and still starts the provider turn", () => + Effect.gen(function* () { + const now = yield* DateTime.now; + const existingSubagentCounts = yield* Ref.make>([]); + const providerSessionOpens = yield* Ref.make(0); + const runningWrites = yield* Ref.make(0); + yield* Effect.gen(function* () { + const service = yield* ProviderTurnStartServiceV2; + yield* service.start({ threadId, runId }); + }).pipe( + Effect.provide( + makeTestLayer({ + rootProjection: makeRootProjection(now), + childProjection: Effect.fail( + new ProjectionStoreThreadNotFoundError({ threadId: childThreadId }), + ), + existingSubagentCounts, + providerSessionOpens, + runningWrites, + }), + ), + ); + + assert.deepEqual(yield* Ref.get(existingSubagentCounts), [0]); + assert.equal(yield* Ref.get(providerSessionOpens), 1); + assert.equal(yield* Ref.get(runningWrites), 1); + }), +); + +it.effect("rehydrates an existing child projection before starting the provider turn", () => + Effect.gen(function* () { + const now = yield* DateTime.now; + const existingSubagentCounts = yield* Ref.make>([]); + const providerSessionOpens = yield* Ref.make(0); + const runningWrites = yield* Ref.make(0); + yield* Effect.gen(function* () { + const service = yield* ProviderTurnStartServiceV2; + yield* service.start({ threadId, runId }); + }).pipe( + Effect.provide( + makeTestLayer({ + rootProjection: makeRootProjection(now), + childProjection: Effect.succeed(makeChildProjection(now)), + existingSubagentCounts, + providerSessionOpens, + runningWrites, + }), + ), + ); + + assert.deepEqual(yield* Ref.get(existingSubagentCounts), [1]); + assert.equal(yield* Ref.get(providerSessionOpens), 1); + assert.equal(yield* Ref.get(runningWrites), 1); + }), +); diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index e2bddd7ec31..dc2fbcda749 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -20,8 +20,7 @@ import { providerMessageWithContextHandoffs, } from "./ContextHandoffService.ts"; import { IdAllocatorV2 } from "./IdAllocator.ts"; -import type { ProviderAdapterV2ExistingSubagent } from "./ProviderAdapter.ts"; -import { ProjectionStoreV2 } from "./ProjectionStore.ts"; +import { ProjectionStoreThreadNotFoundError, ProjectionStoreV2 } from "./ProjectionStore.ts"; import { ProviderSessionManagerV2 } from "./ProviderSessionManager.ts"; import { canReactivateSubagent, @@ -130,6 +129,53 @@ export const layer: Layer.Layer< }); } const providerSessionId = providerThread.providerSessionId; + const routableSubagents = projection.subagents.filter((subagent) => + canRouteRelatedSubagent(subagent.status), + ); + // Broader than the child-thread list above: an interrupted agent is still + // resumable, so it stays re-activatable even though its thread must not + // be adopted up front. + const reactivatableSubagents = projection.subagents.filter((subagent) => + canReactivateSubagent(subagent.status), + ); + // Resolve the persisted registry before opening the provider session or + // advancing the run. Operational read failures must leave the run in + // `starting` so the durable effect can retry safely. + const existingSubagents = yield* Effect.forEach( + reactivatableSubagents, + (subagent, index) => + Effect.gen(function* () { + const childThreadId = subagent.childThreadId; + if (childThreadId === null) return []; + const turnItem = projection.turnItems.find( + (item) => item.type === "subagent" && item.subagentId === subagent.id, + ); + if (turnItem === undefined) return []; + const childProjection = yield* projectionStore + .getThreadProjection(childThreadId) + .pipe( + Effect.catchIf(Schema.is(ProjectionStoreThreadNotFoundError), () => + Effect.succeed(null), + ), + ); + if (childProjection === null) return []; + const childProviderThread = childProjection.providerThreads.find( + (candidate) => candidate.id === subagent.providerThreadId, + ); + if (childProviderThread === undefined) return []; + return [ + { + subagent, + childThread: childProjection.thread, + childProviderThread, + turnItemId: turnItem.id, + turnItemOrdinal: turnItem.ordinal, + ordinal: index + 1, + }, + ]; + }), + { concurrency: "unbounded" }, + ).pipe(Effect.map((entries) => entries.flat())); const isCurrentAttemptInStatus = ( expectedStatus: OrchestrationV2Run["status"], ): Effect.Effect => @@ -408,50 +454,6 @@ export const layer: Layer.Layer< if (!runningWrite.committed) { return; } - const routableSubagents = projection.subagents.filter((subagent) => - canRouteRelatedSubagent(subagent.status), - ); - // Broader than the child-thread list above: an interrupted agent is still - // resumable, so it stays re-activatable even though its thread must not - // be adopted up front. - const reactivatableSubagents = projection.subagents.filter((subagent) => - canReactivateSubagent(subagent.status), - ); - // Adapters keep their subagent registry in memory, so it is empty after a - // session reap or a server restart. Hand back the reusable identities - // from the projection so a returning agent thread is recognised instead - // of being spawned again as a duplicate. - const existingSubagents = yield* Effect.forEach( - reactivatableSubagents, - (subagent, index): Effect.Effect> => - Effect.gen(function* () { - const childThreadId = subagent.childThreadId; - if (childThreadId === null) return []; - const turnItem = projection.turnItems.find( - (item) => item.type === "subagent" && item.subagentId === subagent.id, - ); - if (turnItem === undefined) return []; - const childProjection = yield* projectionStore - .getThreadProjection(childThreadId) - .pipe(Effect.catch(() => Effect.succeed(null))); - if (childProjection === null) return []; - const childProviderThread = childProjection.providerThreads.find( - (candidate) => candidate.id === subagent.providerThreadId, - ); - if (childProviderThread === undefined) return []; - return [ - { - subagent, - childThread: childProjection.thread, - childProviderThread, - turnItemId: turnItem.id, - turnItemOrdinal: turnItem.ordinal, - ordinal: index + 1, - }, - ]; - }), - { concurrency: "unbounded" }, - ).pipe(Effect.map((entries) => entries.flat())); yield* runExecution.startRootRun({ commandId: CommandId.make(`command:effect:provider-turn.start:${run.id}`), appThread: projection.thread, diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 32dca9f8b15..08cce151f80 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -46,6 +46,7 @@ import { canRouteRelatedSubagent, cascadeTerminalizeRunOwnedSubagents, finalProviderThreadStatus, + isRunOwnedSubagentTurnItem, layer as runExecutionServiceLayer, makeProviderEventRoutingState, type ProviderEventRouteIdentity, @@ -212,6 +213,34 @@ it.effect("routes a reused subagent's rows to the run that re-activated it", () updatedAt: now, }, }; + const turnItemEvent = { + type: "turn_item.updated", + driver, + turnItem: { + id: TurnItemId.make("turn-item:reuse:subagent"), + threadId, + runId: spawnRunId, + nodeId: subagentId, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 2, + status: "running", + title: "reused agent", + startedAt: now, + completedAt: null, + updatedAt: now, + type: "subagent", + subagentId, + origin: "provider_native", + driver, + providerInstanceId: ProviderInstanceId.make("codex"), + childThreadId, + prompt: "continue", + result: null, + }, + } satisfies ProviderAdapterV2Event; const withoutSeed = makeProviderEventRoutingState({ identity: laterRun, @@ -219,6 +248,7 @@ it.effect("routes a reused subagent's rows to the run that re-activated it", () }); assert.isFalse(routeProviderEvent(subagentEvent, laterRun, withoutSeed)[0]); assert.isFalse(routeProviderEvent(activationEvent, laterRun, withoutSeed)[0]); + assert.isFalse(routeProviderEvent(turnItemEvent, laterRun, withoutSeed)[0]); const seeded = makeProviderEventRoutingState({ identity: laterRun, @@ -228,6 +258,14 @@ it.effect("routes a reused subagent's rows to the run that re-activated it", () const [subagentAccepted, afterSubagent] = routeProviderEvent(subagentEvent, laterRun, seeded); assert.isTrue(subagentAccepted); assert.isTrue(routeProviderEvent(activationEvent, laterRun, afterSubagent)[0]); + assert.isTrue(routeProviderEvent(turnItemEvent, laterRun, afterSubagent)[0]); + assert.isTrue( + isRunOwnedSubagentTurnItem({ + turnItem: turnItemEvent.turnItem, + runId: laterRun.runId, + ownedSubagentIds: afterSubagent.ownedSubagentIds, + }), + ); // Accepting the row also adopts its child thread, so the agent's own // messages and turn items route into this run for the rest of it. assert.isTrue(afterSubagent.ownedThreadIds.has(childThreadId)); @@ -1737,6 +1775,73 @@ it.effect("cascade helper is provider-neutral for Claude and Codex-shaped child }), ); +it.effect("terminalizes an identity-owned reused subagent turn item from its spawning run", () => + Effect.gen(function* () { + const now = yield* DateTime.now; + const runId = RunId.make("run:reused-turn-item:current"); + const spawnRunId = RunId.make("run:reused-turn-item:spawn"); + const threadId = ThreadId.make("thread:reused-turn-item"); + const subagentId = NodeId.make("node:reused-turn-item:subagent"); + const turnItem = { + id: TurnItemId.make("turn-item:reused-turn-item:subagent"), + threadId, + runId: spawnRunId, + nodeId: subagentId, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 1, + status: "running" as const, + title: "reused agent", + startedAt: now, + completedAt: null, + updatedAt: now, + type: "subagent" as const, + subagentId, + origin: "provider_native" as const, + driver, + providerInstanceId: ProviderInstanceId.make("codex"), + childThreadId: ThreadId.make("thread:reused-turn-item:child"), + prompt: "continue", + progress: "working", + result: null, + } satisfies Extract; + + const events = yield* cascadeTerminalizeRunOwnedSubagents({ + run: { + id: runId, + threadId, + ordinal: 2, + providerInstanceId: ProviderInstanceId.make("codex"), + } as OrchestrationV2Run, + open: { + subagents: new Map(), + activations: new Map(), + turnItems: new Map([[subagentId, turnItem]]), + childTurnItems: new Map(), + nodes: new Map(), + linkedChildThreadIds: new Set(), + }, + status: "interrupted", + completedAt: now, + allocateEventId: () => Effect.succeed(EventId.make("event:reused-turn-item:cascade")), + }); + + assert.lengthOf(events, 1); + const event = events[0]; + assert.isDefined(event); + if (event.type !== "turn-item.updated" || event.payload.type !== "subagent") { + assert.fail("expected reused subagent turn-item.updated event"); + return; + } + assert.equal(event.runId, runId); + assert.equal(event.payload.runId, spawnRunId); + assert.equal(event.payload.status, "interrupted"); + assert.equal(event.payload.id, turnItem.id); + }), +); + it.effect("omits interrupt results and subagent cascade for a superseded attempt", () => Effect.gen(function* () { const written = yield* captureInterruptTerminalTurnItems({ diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 3ea430455c1..40afea9faed 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -162,6 +162,17 @@ export function canReactivateSubagent(status: OrchestrationV2Subagent["status"]) return status !== "failed" && status !== "cancelled"; } +export function isRunOwnedSubagentTurnItem(input: { + readonly turnItem: OrchestrationV2TurnItem; + readonly runId: OrchestrationV2Run["id"]; + readonly ownedSubagentIds: ReadonlySet; +}): boolean { + return ( + input.turnItem.runId === input.runId || + (input.turnItem.type === "subagent" && input.ownedSubagentIds.has(input.turnItem.subagentId)) + ); +} + function emptyOpenRunOwnedSubagentProjection(): OpenRunOwnedSubagentProjection { return { subagents: new Map(), @@ -251,11 +262,7 @@ export function cascadeTerminalizeRunOwnedSubagents(input: { }); } const turnItem = input.open.turnItems.get(key); - if ( - turnItem !== undefined && - turnItem.runId === input.run.id && - !isTerminalTurnItemStatus(turnItem.status) - ) { + if (turnItem !== undefined && !isTerminalTurnItemStatus(turnItem.status)) { events.push({ id: yield* input.allocateEventId(), type: "turn-item.updated", @@ -1031,7 +1038,11 @@ export const layer: Layer.Layer< }); } if (event.type === "turn_item.updated") { - const belongsToRootRun = event.turnItem.runId === input.run.id; + const belongsToRootRun = isRunOwnedSubagentTurnItem({ + turnItem: event.turnItem, + runId: input.run.id, + ownedSubagentIds: routing.ownedSubagentIds, + }); const belongsToOwnedChildThread = event.turnItem.threadId !== input.run.threadId && routing.ownedThreadIds.has(event.turnItem.threadId); From b0270f6ff3eaf97b14afe327078d6c0b855aef65 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Thu, 30 Jul 2026 05:15:32 +0530 Subject: [PATCH 4/8] refactor(orchestration-v2): catch projection errors by tag --- .../src/orchestration-v2/ProviderTurnStartService.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index dc2fbcda749..a0bcb59fa1b 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -151,13 +151,11 @@ export const layer: Layer.Layer< (item) => item.type === "subagent" && item.subagentId === subagent.id, ); if (turnItem === undefined) return []; - const childProjection = yield* projectionStore - .getThreadProjection(childThreadId) - .pipe( - Effect.catchIf(Schema.is(ProjectionStoreThreadNotFoundError), () => - Effect.succeed(null), - ), - ); + const childProjection = yield* projectionStore.getThreadProjection(childThreadId).pipe( + Effect.catchTags({ + ProjectionStoreThreadNotFoundError: () => Effect.succeed(null), + }), + ); if (childProjection === null) return []; const childProviderThread = childProjection.providerThreads.find( (candidate) => candidate.id === subagent.providerThreadId, From b119a0a9c3482967e09e3f0593d363a8cd96d66e Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Thu, 30 Jul 2026 05:27:13 +0530 Subject: [PATCH 5/8] test(orchestration-v2): replay subagent reuse after idle --- .../testkit/fixtures/index.ts | 25 +++++++++++++------ .../codex_transcript.ndjson | 12 ++++++--- .../subagent_reuse_after_idle/input.ts | 8 +----- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/index.ts b/apps/server/src/orchestration-v2/testkit/fixtures/index.ts index afb071a6793..1b72fa064d0 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/index.ts +++ b/apps/server/src/orchestration-v2/testkit/fixtures/index.ts @@ -37,6 +37,8 @@ import { subagentInput } from "./subagent/input.ts"; import { assertCursorSubagentOutput } from "./subagent/cursor_output.ts"; import { assertSubagentContinueOutput } from "./subagent_continue/codex_output.ts"; import { subagentContinueInput } from "./subagent_continue/input.ts"; +import { assertSubagentReuseAfterIdleOutput } from "./subagent_reuse_after_idle/codex_output.ts"; +import { subagentReuseAfterIdleInput } from "./subagent_reuse_after_idle/input.ts"; import { assertSubagentV2Output } from "./subagent_v2/codex_output.ts"; import { subagentV2Input } from "./subagent_v2/input.ts"; import { assertSubagentV2NestedOutput } from "./subagent_v2_nested/codex_output.ts"; @@ -387,14 +389,21 @@ export const ORCHESTRATOR_REPLAY_FIXTURES = [ }, ], }, - // subagent_reuse_after_idle is scaffolded and recorded but not registered: - // replaying it leaves the second run running forever, so the scenario never - // reaches thread idle. Not caused by the reuse routing — the hang reproduces - // with those changes reverted, and persists now that the rebind demonstrably - // fires (subagent_continue proves that). Root cause still unknown; the - // recorded transcript uses the current Codex item shape (subAgentActivity - // started/interacted, collab `wait` with empty receiverThreadIds), whereas - // the passing subagent_continue transcript uses resumeAgent/sendInput. + { + name: "subagent_reuse_after_idle", + buildInput: subagentReuseAfterIdleInput, + providers: [ + { + driver: ProviderDriverKind.make("codex"), + transcriptFile: new URL( + "./subagent_reuse_after_idle/codex_transcript.ndjson", + import.meta.url, + ), + modelSelection: CODEX_MODEL_SELECTION, + assertOutput: assertSubagentReuseAfterIdleOutput, + }, + ], + }, { name: "subagent_v2", buildInput: subagentV2Input, diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/codex_transcript.ndjson b/apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/codex_transcript.ndjson index 8daa767bfa6..89e02074b8d 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/codex_transcript.ndjson +++ b/apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/codex_transcript.ndjson @@ -5,7 +5,7 @@ {"type":"expect_outbound","label":"thread/start","frame":{"id":2,"method":"thread/start","params":{}}} {"type":"emit_inbound","label":"remoteControl/status/changed","frame":{"method":"remoteControl/status/changed","params":{"status":"disabled","serverName":"shivam-macpro.local","installationId":"6a5680bf-d908-4d8f-b9f4-ee9e1d7089ae","environmentId":null},"emittedAtMs":1785137018968}} {"type":"emit_inbound","label":"thread/start","frame":{"id":2,"result":{"thread":{"id":"019fa275-3860-7732-8f0b-6d43c27baf50","extra":null,"sessionId":"019fa275-3860-7732-8f0b-6d43c27baf50","forkedFromId":null,"parentThreadId":null,"preview":"","ephemeral":false,"historyMode":"legacy","modelProvider":"openai","createdAt":1785137019,"updatedAt":1785137019,"recencyAt":1785137019,"status":{"type":"idle"},"path":"/Users/shivam/.codex/sessions/2026/07/27/rollout-2026-07-27T12-53-38-019fa275-3860-7732-8f0b-6d43c27baf50.jsonl","cwd":"/Users/shivam/Developer/t3/ccing/open/t3code-2/apps/server","cliVersion":"0.145.0","source":"vscode","canAcceptDirectInput":true,"threadSource":null,"agentNickname":null,"agentRole":null,"gitInfo":null,"name":null,"turns":[]},"model":"gpt-5.6-sol","modelProvider":"openai","serviceTier":"priority","cwd":"/Users/shivam/Developer/t3/ccing/open/t3code-2/apps/server","runtimeWorkspaceRoots":["/Users/shivam/Developer/t3/ccing/open/t3code-2/apps/server"],"instructionSources":["/Users/shivam/.codex/AGENTS.md","/Users/shivam/Developer/t3/ccing/open/t3code-2/AGENTS.md"],"approvalPolicy":"never","approvalsReviewer":"user","sandbox":{"type":"dangerFullAccess"},"activePermissionProfile":null,"reasoningEffort":"high","multiAgentMode":"explicitRequestOnly"}}} -{"type":"expect_outbound","label":"turn/start","frame":{"id":3,"method":"turn/start","params":{"input":[{"text":"Spawn one subagent named idle_probe and have it reply exactly: idle probe first response","type":"text"}],"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50"}}} +{"type":"expect_outbound","label":"turn/start","frame":{"id":3,"method":"turn/start","params":{"input":[{"text":"Spawn one subagent named idle_probe and have it reply exactly: idle probe first response","type":"text"}],"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","cwd":"","model":"gpt-5.4","approvalsReviewer":"user"}}} {"type":"emit_inbound","label":"thread/started","frame":{"method":"thread/started","params":{"thread":{"id":"019fa275-3860-7732-8f0b-6d43c27baf50","extra":null,"sessionId":"019fa275-3860-7732-8f0b-6d43c27baf50","forkedFromId":null,"parentThreadId":null,"preview":"","ephemeral":false,"historyMode":"legacy","modelProvider":"openai","createdAt":1785137019,"updatedAt":1785137019,"recencyAt":1785137019,"status":{"type":"idle"},"path":"/Users/shivam/.codex/sessions/2026/07/27/rollout-2026-07-27T12-53-38-019fa275-3860-7732-8f0b-6d43c27baf50.jsonl","cwd":"/Users/shivam/Developer/t3/ccing/open/t3code-2/apps/server","cliVersion":"0.145.0","source":"vscode","canAcceptDirectInput":true,"threadSource":null,"agentNickname":null,"agentRole":null,"gitInfo":null,"name":null,"turns":[]}},"emittedAtMs":1785137019108}} {"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"xcodebuildmcp","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137019109}} {"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"codex_apps","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137019109}} @@ -77,8 +77,14 @@ {"type":"emit_inbound","label":"account/rateLimits/updated","frame":{"method":"account/rateLimits/updated","params":{"rateLimits":{"limitId":"codex","limitName":null,"primary":{"usedPercent":8,"windowDurationMins":10080,"resetsAt":1785634197},"secondary":null,"credits":{"hasCredits":false,"unlimited":false,"balance":"0"},"individualLimit":null,"spendControlReached":null,"planType":"prolite","rateLimitReachedType":null}},"emittedAtMs":1785137032924}} {"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","status":{"type":"idle"}},"emittedAtMs":1785137032925}} {"type":"emit_inbound","label":"turn/completed","frame":{"method":"turn/completed","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turn":{"id":"019fa275-38e7-7a12-9975-235b0e73bc77","items":[],"itemsView":"notLoaded","status":"completed","error":null,"startedAt":1785137019,"completedAt":1785137032,"durationMs":13811}},"emittedAtMs":1785137032925}} -{"type":"expect_outbound","label":"turn/start","frame":{"id":4,"method":"turn/start","params":{"input":[{"text":"Send another message to the same idle_probe subagent. Do not spawn a replacement. Have it reply exactly: idle probe resumed response","type":"text"}],"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50"}}} -{"type":"emit_inbound","label":"turn/start","frame":{"id":4,"result":{"turn":{"id":"019fa275-6edd-7d23-932f-9a8bcd630dce","items":[],"itemsView":"notLoaded","status":"inProgress","error":null,"startedAt":null,"completedAt":null,"durationMs":null}}}} +{"type":"runtime_exit","status":"success"} +{"type":"expect_outbound","label":"initialize","frame":{"id":1,"method":"initialize","params":{"capabilities":{"experimentalApi":true},"clientInfo":{"name":"t3code_desktop","title":"T3 Code Desktop","version":"0.1.0"}}}} +{"type":"emit_inbound","label":"initialize","frame":{"id":1,"result":{"userAgent":"t3code_desktop/0.145.0 (Mac OS 26.5.2; arm64) unknown (t3code_desktop; 0.1.0)","codexHome":"/Users/shivam/.codex","platformFamily":"unix","platformOs":"macos"}}} +{"type":"expect_outbound","label":"initialized","frame":{"method":"initialized"}} +{"type":"expect_outbound","label":"thread/resume","frame":{"id":2,"method":"thread/resume","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50"}}} +{"type":"emit_inbound","label":"thread/resume","frame":{"id":2,"result":{"thread":{"id":"019fa275-3860-7732-8f0b-6d43c27baf50","extra":null,"sessionId":"019fa275-3860-7732-8f0b-6d43c27baf50","forkedFromId":null,"parentThreadId":null,"preview":"","ephemeral":false,"historyMode":"legacy","modelProvider":"openai","createdAt":1785137019,"updatedAt":1785137019,"recencyAt":1785137019,"status":{"type":"idle"},"path":"/Users/shivam/.codex/sessions/2026/07/27/rollout-2026-07-27T12-53-38-019fa275-3860-7732-8f0b-6d43c27baf50.jsonl","cwd":"/Users/shivam/Developer/t3/ccing/open/t3code-2/apps/server","cliVersion":"0.145.0","source":"vscode","canAcceptDirectInput":true,"threadSource":null,"agentNickname":null,"agentRole":null,"gitInfo":null,"name":null,"turns":[]},"model":"gpt-5.6-sol","modelProvider":"openai","serviceTier":"priority","cwd":"/Users/shivam/Developer/t3/ccing/open/t3code-2/apps/server","runtimeWorkspaceRoots":["/Users/shivam/Developer/t3/ccing/open/t3code-2/apps/server"],"instructionSources":["/Users/shivam/.codex/AGENTS.md","/Users/shivam/Developer/t3/ccing/open/t3code-2/AGENTS.md"],"approvalPolicy":"never","approvalsReviewer":"user","sandbox":{"type":"dangerFullAccess"},"activePermissionProfile":null,"reasoningEffort":"high","multiAgentMode":"explicitRequestOnly"}}} +{"type":"expect_outbound","label":"turn/start","frame":{"id":3,"method":"turn/start","params":{"input":[{"text":"Send another message to the same idle_probe subagent. Do not spawn a replacement. Have it reply exactly: idle probe resumed response","type":"text"}],"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","cwd":"","model":"gpt-5.4","approvalsReviewer":"user"}}} +{"type":"emit_inbound","label":"turn/start","frame":{"id":3,"result":{"turn":{"id":"019fa275-6edd-7d23-932f-9a8bcd630dce","items":[],"itemsView":"notLoaded","status":"inProgress","error":null,"startedAt":null,"completedAt":null,"durationMs":null}}}} {"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"codex_apps","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137032927}} {"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"openaiDeveloperDocs","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137032927}} {"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"node_repl","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137032927}} diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/input.ts b/apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/input.ts index 4b23f65b0df..3a66bfdc28f 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/input.ts +++ b/apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/input.ts @@ -13,18 +13,12 @@ import { * unrecognised on the next turn and gets registered as a brand new subagent — * the user sees a duplicate appear and the original frozen forever. * - * Not registered yet. The recorded transcript does contain a real reuse, but - * replaying it leaves the second run running forever so the scenario never - * reaches thread idle, and that is unrelated to the reuse routing here — it - * reproduces with those changes reverted. An advance_clock step to cross the - * 30-minute idle timeout (which is what would exercise rehydration rather than - * plain reuse) fails the same way, so it is left out until the hang is - * understood. */ export function subagentReuseAfterIdleInput(): OrchestratorFixtureInput { return { steps: [ { type: "message", text: SUBAGENT_REUSE_AFTER_IDLE_PROMPT }, + { type: "advance_clock", duration: "31 minutes" }, { type: "message", text: SUBAGENT_REUSE_AFTER_IDLE_RESUME_PROMPT }, ], }; From d453837b42768190e8fbd6a0fe66edb9bf5af0a9 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Thu, 30 Jul 2026 06:14:36 +0530 Subject: [PATCH 6/8] test(orchestration-v2): register idle reuse replay expectations --- .../testkit/CodexReplayFixtures.integration.test.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts b/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts index 9c0b279332d..4b9625f4caf 100644 --- a/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts @@ -136,6 +136,13 @@ const scenarioExpectations = { turnCompletedCount: 0, approvalRequestCount: 0, }, + subagent_reuse_after_idle: { + outgoing: ["initialize", "initialized", "thread/start", "thread/resume", "turn/start"], + incoming: ["thread/started", "thread/resume", "turn/started", "turn/completed"], + turnStartCount: 2, + turnCompletedCount: 4, + approvalRequestCount: 0, + }, subagent_v2: { outgoing: ["initialize", "initialized", "thread/start", "turn/start"], incoming: [ From 327de5024bf8b47d05ac35b9de95a7e4b76989a2 Mon Sep 17 00:00:00 2001 From: T3 Code Test Date: Fri, 31 Jul 2026 04:49:28 +0530 Subject: [PATCH 7/8] fix(orchestration-v2): leave zero-activation leftovers unseeded in turn ordinals The rehydration seed stored Math.max(1, activationCount) in a map that tracks the last issued ordinal, so a leftover with no recorded activations started its first turn at ordinal 2 and skipped childRootNodeId. Skip seeding those entries so the first turn takes ordinal 1. Co-Authored-By: Claude Fable 5 --- apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 757f26b28e4..2898a8dd67b 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -1613,7 +1613,11 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi for (const entry of existing) { const key = String(entry.childProviderThread.id); if (updated.has(key)) continue; - updated.set(key, Math.max(1, entry.subagent.activationCount)); + // The map stores the last issued ordinal. A leftover with no + // recorded activations must stay unseeded so its first turn + // takes ordinal 1 and maps onto childRootNodeId. + if (entry.subagent.activationCount <= 0) continue; + updated.set(key, entry.subagent.activationCount); } return updated; }); From b0398e1ae362af183db28d9dfa164fcfff41978a Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Sat, 1 Aug 2026 13:39:49 +0530 Subject: [PATCH 8/8] fix(orchestration-v2): ignore settled subagent noise Co-Authored-By: Claude Fable 5 --- .../Adapters/CodexAdapterV2.test.ts | 124 +++++++++++++++++- .../Adapters/CodexAdapterV2.ts | 30 +++-- 2 files changed, 134 insertions(+), 20 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index 83b3a306de9..daba8b2e144 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -3577,7 +3577,7 @@ describe("CodexAdapterV2 post-settle continuation", () => { assert.equal(firstCompletion?.subagent.status, "idle"); assert.equal(firstCompletion?.subagent.result, "CODEX_FIRST_DONE"); assert.equal(firstCompletion?.subagent.activationCount, 1); - assert.equal(firstCompletion?.subagent.usage?.totalTokens, 100); + assert.isNull(firstCompletion?.subagent.usage); assert.isNull(firstCompletion?.subagent.currentActivationId); const firstActivationUpdates = harness .subagentActivationUpdates() @@ -3627,7 +3627,7 @@ describe("CodexAdapterV2 post-settle continuation", () => { }, "resumed subagent completion"); const finalSubagent = harness.subagentUpdates().at(-1)?.subagent; assert.equal(finalSubagent?.activationCount, 2); - assert.equal(finalSubagent?.usage?.totalTokens, 180); + assert.isNull(finalSubagent?.usage); assert.isNull(finalSubagent?.currentActivationId); const activations = harness.subagentActivationUpdates(); assert.lengthOf(new Set(activations.map((event) => event.activation.id)), 2); @@ -3777,6 +3777,91 @@ describe("CodexAdapterV2 post-settle continuation", () => { ], }); + const restartSettledNoiseTranscript = makeCodexReplayTranscript({ + scenario: `${RESTART_SCENARIO}-settled-noise`, + entries: [ + ...codexReplayPreamble({ + nativeThreadId: RESTART_NATIVE_THREAD, + nativeTurnId: RESTART_NATIVE_TURN, + prompt: RESTART_PROMPT, + }), + restartChildFrame({ + label: "thread/tokenUsage/updated/settled-child", + method: "thread/tokenUsage/updated", + params: { + threadId: RESTART_CHILD_THREAD, + turnId: "native-codex-restart-settled-turn", + tokenUsage: { + total: { + totalTokens: 250, + inputTokens: 200, + cachedInputTokens: 0, + outputTokens: 50, + reasoningOutputTokens: 0, + }, + last: { + totalTokens: 150, + inputTokens: 120, + cachedInputTokens: 0, + outputTokens: 30, + reasoningOutputTokens: 0, + }, + }, + }, + }), + restartChildFrame({ + label: "item/completed/settled-agent-state", + method: "item/completed", + params: { + item: { + type: "collabAgentToolCall", + id: "call-codex-restart-settled-state", + tool: "wait", + status: "completed", + senderThreadId: RESTART_NATIVE_THREAD, + receiverThreadIds: [RESTART_CHILD_THREAD], + prompt: null, + model: null, + reasoningEffort: null, + agentsStates: { + [RESTART_CHILD_THREAD]: { + status: "completed", + message: "TRAILING_SETTLED_RESULT", + }, + }, + }, + threadId: RESTART_NATIVE_THREAD, + turnId: RESTART_NATIVE_TURN, + completedAtMs: 1782622482000, + }, + }), + restartChildFrame({ + label: "item/completed/root-answer", + method: "item/completed", + params: { + item: { + type: "agentMessage", + id: "root-answer-restart-settled-noise", + text: "NO_CHANGE", + phase: "final_answer", + memoryCitation: null, + }, + threadId: RESTART_NATIVE_THREAD, + turnId: RESTART_NATIVE_TURN, + completedAtMs: 1782622483000, + }, + }), + restartChildFrame({ + label: "turn/completed/root", + method: "turn/completed", + params: { + threadId: RESTART_NATIVE_THREAD, + turn: makeCodexReplayTurn({ id: RESTART_NATIVE_TURN, status: "completed" }), + }, + }), + ], + }); + /** * The subagent as the projection holds it after a restart: settled at idle, * still bound to its child thread and native item, one activation recorded. @@ -3906,10 +3991,7 @@ describe("CodexAdapterV2 post-settle continuation", () => { assert.equal(latest?.childThreadId, existing.childThread.id); // Continues the prior activation rather than restarting the count. assert.equal(latest?.activationCount, 2); - // Usage carries across the restart instead of resetting: the seeded - // 100 is not lost, and the thread-cumulative 250 is not double-counted - // on top of it. - assert.equal(latest?.usage?.totalTokens, 250); + assert.equal(latest?.usage?.totalTokens, 100); // The new activation takes the next ordinal, not a colliding ordinal 1. const activations = harness.subagentActivationUpdates(); assert.deepStrictEqual( @@ -3920,6 +4002,36 @@ describe("CodexAdapterV2 post-settle continuation", () => { [...new Set(activations.map((event) => event.activation.subagentId))], [RESTART_SUBAGENT_ID], ); + assert.equal(activations.at(-1)?.activation.usage?.totalTokens, 150); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("drops settled identity updates until a new activation starts", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeCodexReplayHarness(restartSettledNoiseTranscript); + const now = yield* DateTime.now; + const existing = makeRestartExistingSubagent({ + parentThreadId: harness.threadId, + parentProviderThread: harness.providerThread, + now, + }); + + yield* harness.runtime.startTurn({ + ...makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-codex-restart-settled-noise"), + text: RESTART_PROMPT, + }), + existingSubagents: [existing], + }); + + yield* awaitUntil(() => harness.terminalEvents().length === 1, "root turn terminal"); + assert.lengthOf(harness.subagentUpdates(), 0); + assert.lengthOf(harness.subagentActivationUpdates(), 0); }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), ), ); diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 9cc352f1d5c..762898a1725 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -2476,7 +2476,7 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi const subagents = yield* Ref.get(subagentThreads); for (const [nativeThreadId, state] of Object.entries(input.item.agentsStates)) { const subagent = subagents.get(nativeThreadId); - if (subagent === undefined) { + if (subagent === undefined || subagent.task.currentActivationId === null) { continue; } yield* emitSubagentTaskUpdate({ @@ -3488,19 +3488,21 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi if (subagent === undefined || payload.tokenUsage.total.totalTokens <= 0) { return; } - const total = payload.tokenUsage.total; - const usage = { - totalTokens: total.totalTokens, - inputTokens: total.inputTokens, - cachedInputTokens: total.cachedInputTokens, - outputTokens: total.outputTokens, - reasoningOutputTokens: total.reasoningOutputTokens, - } satisfies OrchestrationV2SubagentUsage; - yield* emitSubagentTaskUpdate({ - subagent, - status: subagent.task.status, - usage, - }); + if (subagent.task.currentActivationId !== null) { + const total = payload.tokenUsage.total; + const usage = { + totalTokens: total.totalTokens, + inputTokens: total.inputTokens, + cachedInputTokens: total.cachedInputTokens, + outputTokens: total.outputTokens, + reasoningOutputTokens: total.reasoningOutputTokens, + } satisfies OrchestrationV2SubagentUsage; + yield* emitSubagentTaskUpdate({ + subagent, + status: subagent.task.status, + usage, + }); + } const activeContext = (yield* Ref.get(activeTurns)).get(payload.turnId); const activation =