Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion plugins/tracing/dist/index.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -46687,6 +46687,9 @@ function parseSession(lines) {
}
const ensureTurn = (ts) => turn ??= newTurn(ts);
const ensureStep = (ts) => step ??= newStep(ts);
const recordSubagentThread = (threadId) => {
if (!turn.subagentThreadIds.includes(threadId)) turn.subagentThreadIds.push(threadId);
};
const closeStep = (ts, usage) => {
if (!step) return;
step.endTime = Math.max(step.endTime, ts);
Expand Down Expand Up @@ -46835,7 +46838,8 @@ function parseSession(lines) {
aborted: true
});
else {
if (et === "collab_agent_spawn_end" && typeof p.new_thread_id === "string") turn.subagentThreadIds.push(p.new_thread_id);
if (et === "collab_agent_spawn_end" && typeof p.new_thread_id === "string") recordSubagentThread(p.new_thread_id);
if (et === "sub_agent_activity" && p.kind === "started" && typeof p.agent_thread_id === "string") recordSubagentThread(p.agent_thread_id);
if ((et === "mcp_tool_call_begin" || et === "mcp_tool_call_end") && typeof p.call_id === "string") {
const tc = toolCallsById.get(p.call_id);
const inv = p.invocation;
Expand Down
21 changes: 20 additions & 1 deletion plugins/tracing/src/parse.ts
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,14 @@ export function parseSession(lines: RolloutLine[]): {
const ensureTurn = (ts: number): MutableTurn => (turn ??= newTurn(ts));
const ensureStep = (ts: number) => (step ??= newStep(ts));

// Rollouts from the transition period can carry both spawn-event formats
// for the same child; the thread must be nested exactly once.
const recordSubagentThread = (threadId: string) => {
if (!turn!.subagentThreadIds.includes(threadId)) {
turn!.subagentThreadIds.push(threadId);
}
};

const closeStep = (ts: number, usage?: TokenUsage) => {
if (!step) return;
step.endTime = Math.max(step.endTime, ts);
Expand Down Expand Up @@ -308,7 +316,18 @@ export function parseSession(lines: RolloutLine[]): {
// A subagent spawn records the child thread *and* (since it carries a
// call_id ending in "_end") enriches the spawning tool call below.
if (et === "collab_agent_spawn_end" && typeof p.new_thread_id === "string") {
turn!.subagentThreadIds.push(p.new_thread_id);
recordSubagentThread(p.new_thread_id);
}
// Codex multi-agent v2 persists the spawn as sub_agent_activity
// instead. Only kind "started" marks a spawn — "interacted" and
// "interrupted" reference an existing child and would nest it under
// the wrong (later) turn.
if (
et === "sub_agent_activity" &&
p.kind === "started" &&
typeof p.agent_thread_id === "string"
) {
recordSubagentThread(p.agent_thread_id);
}
// MCP tool calls are function calls with a mangled name
// (`server__tool`); the begin/end events carry the clean server/tool
Expand Down
3 changes: 3 additions & 0 deletions plugins/tracing/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,9 @@ export type EventMsgPayload = {
} | null;
/** collab_agent_spawn_end */
new_thread_id?: string | null;
/** sub_agent_activity */
kind?: string;
agent_thread_id?: string | null;
/** mcp_tool_call_begin / mcp_tool_call_end */
invocation?: { server?: string; tool?: string; arguments?: unknown } | null;
/** web_search_end */
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{"timestamp":"2026-06-03T12:00:00.000Z","type":"session_meta","payload":{"id":"sess-activity","cli_version":"0.145.0","model_provider":"openai"}}
{"timestamp":"2026-06-03T12:00:01.000Z","type":"event_msg","payload":{"type":"task_started","turn_id":"turn-activity"}}
{"timestamp":"2026-06-03T12:00:01.200Z","type":"turn_context","payload":{"model":"gpt-5.4"}}
{"timestamp":"2026-06-03T12:00:01.300Z","type":"event_msg","payload":{"type":"user_message","message":"Spawn a worker to compute the answer"}}
{"timestamp":"2026-06-03T12:00:02.000Z","type":"response_item","payload":{"type":"function_call","name":"spawn_agent","call_id":"call-spawn-act","arguments":"{\"message\":\"compute the answer\"}"}}
{"timestamp":"2026-06-03T12:00:02.500Z","type":"event_msg","payload":{"type":"sub_agent_activity","event_id":"call-spawn-act","occurred_at_ms":1780488002500,"agent_thread_id":"thread-act","agent_path":"/root/worker","kind":"started"}}
{"timestamp":"2026-06-03T12:00:02.600Z","type":"event_msg","payload":{"type":"sub_agent_activity","event_id":"call-wait-act","occurred_at_ms":1780488002600,"agent_thread_id":"thread-act","agent_path":"/root/worker","kind":"interacted"}}
{"timestamp":"2026-06-03T12:00:02.800Z","type":"response_item","payload":{"type":"function_call_output","call_id":"call-spawn-act","output":"{\"agent_id\":\"thread-act\",\"nickname\":\"Worker\"}"}}
{"timestamp":"2026-06-03T12:00:04.500Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"The worker replied: 42"}]}}
{"timestamp":"2026-06-03T12:00:04.600Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":100,"output_tokens":20,"total_tokens":120,"cached_input_tokens":0,"reasoning_output_tokens":0},"total_token_usage":{"input_tokens":100,"output_tokens":20,"total_tokens":120,"cached_input_tokens":0,"reasoning_output_tokens":0}}}}
{"timestamp":"2026-06-03T12:00:05.000Z","type":"event_msg","payload":{"type":"task_complete","turn_id":"turn-activity"}}
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{"timestamp":"2026-06-03T12:00:03.000Z","type":"session_meta","payload":{"id":"thread-act","cli_version":"0.145.0","model_provider":"openai","parent_thread_id":"sess-activity","thread_source":"subagent"}}
{"timestamp":"2026-06-03T12:00:03.100Z","type":"event_msg","payload":{"type":"task_started","turn_id":"turn-act-child"}}
{"timestamp":"2026-06-03T12:00:03.200Z","type":"turn_context","payload":{"model":"gpt-5.4"}}
{"timestamp":"2026-06-03T12:00:03.300Z","type":"event_msg","payload":{"type":"user_message","message":"compute the answer"}}
{"timestamp":"2026-06-03T12:00:04.000Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"42"}]}}
{"timestamp":"2026-06-03T12:00:04.100Z","type":"event_msg","payload":{"type":"agent_message","message":"42"}}
{"timestamp":"2026-06-03T12:00:04.200Z","type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":40,"output_tokens":15,"total_tokens":55,"cached_input_tokens":0,"reasoning_output_tokens":0},"total_token_usage":{"input_tokens":40,"output_tokens":15,"total_tokens":55,"cached_input_tokens":0,"reasoning_output_tokens":0}}}}
{"timestamp":"2026-06-03T12:00:04.300Z","type":"event_msg","payload":{"type":"task_complete","turn_id":"turn-act-child"}}
53 changes: 51 additions & 2 deletions plugins/tracing/test/parse.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,6 @@ describe("parseSession", () => {
expect(turn.completed).toBe(true);
expect(turn.aborted).toBe(true);
expect(turn.userInput).toBe("Spawn a subagent to tell a joke");

// The spawn is recorded as a subagent thread...
expect(turn.subagentThreadIds).toEqual(["thread-child"]);

// ...and the failing exec is captured with its error.
Expand All @@ -81,6 +79,57 @@ describe("parseSession", () => {
expect(turn.endTime).toBe(Date.parse("2026-06-03T11:00:05.000Z"));
});

it("records subagent threads from sub_agent_activity, ignoring non-started kinds", () => {
const event = (ts: string, payload: Record<string, unknown>): RolloutLine => ({
timestamp: ts,
type: "event_msg",
payload: { ...payload },
});
const lines: RolloutLine[] = [
{ timestamp: "2026-06-03T13:00:00.000Z", type: "session_meta", payload: { id: "s" } },
event("2026-06-03T13:00:01.000Z", { type: "task_started", turn_id: "t" }),
event("2026-06-03T13:00:02.000Z", {
type: "sub_agent_activity",
event_id: "c1",
agent_thread_id: "thread-a",
agent_path: "/root/worker",
kind: "started",
}),
// The same spawn reported again — legacy format and a repeated activity.
event("2026-06-03T13:00:02.100Z", {
type: "collab_agent_spawn_end",
call_id: "c1",
new_thread_id: "thread-a",
}),
event("2026-06-03T13:00:02.200Z", {
type: "sub_agent_activity",
event_id: "c1",
agent_thread_id: "thread-a",
agent_path: "/root/worker",
kind: "started",
}),
// Later lifecycle kinds reference an existing child and must not register.
event("2026-06-03T13:00:03.000Z", {
type: "sub_agent_activity",
event_id: "c2",
agent_thread_id: "thread-b",
agent_path: "/root/other",
kind: "interacted",
}),
event("2026-06-03T13:00:03.100Z", {
type: "sub_agent_activity",
event_id: "c3",
agent_thread_id: "thread-c",
agent_path: "/root/other",
kind: "interrupted",
}),
event("2026-06-03T13:00:04.000Z", { type: "task_complete", turn_id: "t" }),
];
const { turns } = parseSession(lines);
expect(turns).toHaveLength(1);
expect(turns[0].subagentThreadIds).toEqual(["thread-a"]);
});

it("treats a trailing, never-completed turn as not completed", () => {
const lines: RolloutLine[] = [
{ timestamp: "2026-06-03T12:00:00.000Z", type: "session_meta", payload: { id: "s" } },
Expand Down
24 changes: 24 additions & 0 deletions plugins/tracing/test/trace.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,30 @@ describe("convertRollout", () => {
expect(attr(failedTool!, "langfuse.observation.status_message")).toContain("command failed");
});

it("nests subagent turns discovered via sub_agent_activity events", async () => {
const dir = stageFixtures();
await convertRollout(path.join(dir, "rollout-activity-main.jsonl"), { config: baseConfig });

const spans = exporter.getFinishedSpans();
const parent = spans.find((s) => s.name === "Codex Turn" && obsType(s) === "agent");
const childTurns = spans.filter(
(s) => s.name === "Codex Subagent Turn" && obsType(s) === "agent",
);
expect(parent).toBeDefined();
// Exactly one child (the kind filter itself is pinned at parse level,
// where non-"started" activities target distinct thread ids).
expect(childTurns).toHaveLength(1);
const child = childTurns[0];
expect(child.spanContext().traceId).toBe(parent!.spanContext().traceId);
expect(attr(child, "langfuse.observation.input")).toContain("compute the answer");

const childGeneration = spans.find(
(s) => obsType(s) === "generation" && parentId(s) === child.spanContext().spanId,
);
expect(childGeneration?.name).toBe("LLM Subagent");
expect(attr(childGeneration!, "langfuse.observation.model.name")).toBe("gpt-5.4");
});

it("captures web search, local shell, and MCP tool calls with specific names", async () => {
const dir = stageFixtures();
await convertRollout(path.join(dir, "rollout-tools-main.jsonl"), { config: baseConfig });
Expand Down