From fb82043d0dd6f39c632073d323dc51c8d19cfced Mon Sep 17 00:00:00 2001 From: Dwin Gharibi Date: Fri, 7 Aug 2026 09:54:08 +0330 Subject: [PATCH 1/5] feat(replay): compare the behaviour of two recorded sessions --- cmd/root/replay.go | 102 +++++++++++++++++++ cmd/root/root.go | 1 + pkg/replay/replay.go | 226 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 329 insertions(+) create mode 100644 cmd/root/replay.go create mode 100644 pkg/replay/replay.go diff --git a/cmd/root/replay.go b/cmd/root/replay.go new file mode 100644 index 0000000000..188838071e --- /dev/null +++ b/cmd/root/replay.go @@ -0,0 +1,102 @@ +package root + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + + "github.com/spf13/cobra" + + pathx "github.com/docker/docker-agent/pkg/path" + "github.com/docker/docker-agent/pkg/replay" + "github.com/docker/docker-agent/pkg/session/sqlitestore" +) + +type replayFlags struct { + sessionDB string + asJSON bool + failOnDiff bool +} + +func newReplayCmd() *cobra.Command { + var flags replayFlags + + cmd := &cobra.Command{ + Use: "replay ", + Short: "Compare the behaviour of two recorded sessions", + GroupID: "advanced", + Args: cobra.ExactArgs(2), + Long: `Compare two recorded sessions and report the first point where the agent +behaved differently. + +Comparison is over the sequence of tool calls, not over the assistant's prose. +Model output is nondeterministic: two runs of the same task almost always word +things differently while doing exactly the same work, so diffing text would report +a difference on every comparison. The tool calls are what changed the world, so +they are what is compared. + +Reporting stops at the first divergence: everything after it is downstream of that +difference and comparing it produces noise rather than information.`, + Example: ` docker agent replay + docker agent replay --json | jq '.divergence.turn_index' + docker agent replay --fail-on-divergence`, + RunE: flags.run, + } + + cmd.Flags().StringVarP(&flags.sessionDB, "session-db", "s", "", "Path to the session database (default: /session.db)") + cmd.Flags().BoolVar(&flags.asJSON, "json", false, "Emit the comparison as JSON") + cmd.Flags().BoolVar(&flags.failOnDiff, "fail-on-divergence", false, "Exit non-zero when the two sessions diverge") + + return cmd +} + +func (f *replayFlags) run(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + dbPath, err := pathx.ExpandHomeDir(sessionDBPath(f.sessionDB)) + if err != nil { + return err + } + + store, err := sqlitestore.New(ctx, dbPath) + if err != nil { + return fmt.Errorf("opening session store: %w", err) + } + defer func() { + if err := store.Close(); err != nil { + slog.ErrorContext(ctx, "Failed to close session store", "error", err) + } + }() + + sessA, err := store.GetSession(ctx, args[0]) + if err != nil { + return fmt.Errorf("reading session %q: %w", args[0], err) + } + sessB, err := store.GetSession(ctx, args[1]) + if err != nil { + return fmt.Errorf("reading session %q: %w", args[1], err) + } + + result := replay.CompareSessions(sessA, sessB) + if err := renderReplay(cmd.OutOrStdout(), result, args[0], args[1], f.asJSON); err != nil { + return err + } + + if f.failOnDiff && !result.Identical() { + return errors.New("sessions diverged") + } + return nil +} + +// renderReplay writes the comparison as JSON or as text. +func renderReplay(w io.Writer, result replay.Result, nameA, nameB string, asJSON bool) error { + if asJSON { + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(result) + } + replay.PrintResult(w, result, nameA, nameB) + return nil +} diff --git a/cmd/root/root.go b/cmd/root/root.go index 1ff4f1ef46..8bc5845af0 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -177,6 +177,7 @@ We collect anonymous usage data to help improve docker agent. To disable: newNewCmd(), newGettingStartedCmd(), newEvalCmd(), + newReplayCmd(), newShareCmd(), newModelsCmd(), newToolsetsCmd(), diff --git a/pkg/replay/replay.go b/pkg/replay/replay.go new file mode 100644 index 0000000000..78b7229f44 --- /dev/null +++ b/pkg/replay/replay.go @@ -0,0 +1,226 @@ +// Package replay compares the behaviour of two recorded agent sessions. +// +// It answers the triage question "did the agent do something different this +// time, and where did it first diverge?" — the thing you actually want to know +// when a task that worked yesterday does not work today. +// +// # Behaviour, not prose +// +// Comparison is over the sequence of tool calls, not over the assistant's text. +// Model output is nondeterministic: two runs of the same task almost always word +// things differently while doing exactly the same work, so diffing prose reports +// a difference on essentially every comparison and is useless as a signal. The +// tool calls are what changed the world, so they are what is compared. +// +// Assistant text is still carried on each [Turn] so a reporter can show what was +// said around a divergence; it just does not decide whether a divergence +// happened. +// +// # First divergence only +// +// Once two runs differ, everything after that point is downstream of the +// difference and comparing it produces noise, not information. So the comparison +// stops at the first divergence and reports where it was. +package replay + +import ( + "fmt" + "io" + "strings" + + "github.com/docker/docker-agent/pkg/session" +) + +// ToolCall is the comparable part of a tool invocation. +type ToolCall struct { + Name string `json:"name"` + Arguments string `json:"arguments"` +} + +// Turn is one assistant turn: what it said, and what it called. +type Turn struct { + // Index is the turn's 0-based position in the session. + Index int `json:"index"` + // Agent is the agent that produced the turn, so a divergence in a + // multi-agent run can be attributed. + Agent string `json:"agent,omitempty"` + // Content is the assistant text. Carried for reporting; never compared. + Content string `json:"content,omitempty"` + ToolCalls []ToolCall `json:"tool_calls,omitempty"` +} + +// Kind classifies how two runs differ. +type Kind string + +const ( + // KindToolCalls means both runs made a turn at this index but called + // different tools, or the same tools with different arguments. + KindToolCalls Kind = "tool_calls" + // KindExtraTurn means the second run kept going after the first stopped. + KindExtraTurn Kind = "extra_turn" + // KindMissingTurn means the second run stopped before the first did. + KindMissingTurn Kind = "missing_turn" +) + +// Divergence is the first behavioural difference found. +type Divergence struct { + Kind Kind `json:"kind"` + // TurnIndex is the 0-based turn at which the runs first differ. + TurnIndex int `json:"turn_index"` + // A and B are the diverging turns. Either may be nil when one run had no + // turn at this index. + A *Turn `json:"a,omitempty"` + B *Turn `json:"b,omitempty"` +} + +// Result is the outcome of a comparison. +type Result struct { + TurnsA int `json:"turns_a"` + TurnsB int `json:"turns_b"` + // TurnsMatched is how many leading turns behaved identically. + TurnsMatched int `json:"turns_matched"` + // Divergence is nil when both runs behaved identically throughout. + Divergence *Divergence `json:"divergence,omitempty"` +} + +// Identical reports whether the two runs behaved the same the whole way. +func (r Result) Identical() bool { return r.Divergence == nil } + +// TurnsOf extracts the assistant turns from a session, in order. +// +// Only assistant messages are turns: user messages are inputs and tool messages +// are results, neither of which is a decision the model made. A nil session +// yields no turns. +func TurnsOf(sess *session.Session) []Turn { + if sess == nil { + return nil + } + + var turns []Turn + for i := range sess.Messages { + item := &sess.Messages[i] + if item.Message == nil { + continue + } + msg := &item.Message.Message + if msg.Role != "assistant" { + continue + } + + turn := Turn{ + Index: len(turns), + Agent: item.Message.AgentName, + Content: msg.Content, + } + for _, tc := range msg.ToolCalls { + turn.ToolCalls = append(turn.ToolCalls, ToolCall{ + Name: tc.Function.Name, + Arguments: tc.Function.Arguments, + }) + } + turns = append(turns, turn) + } + return turns +} + +// Compare walks two runs' turns and reports the first behavioural divergence. +func Compare(a, b []Turn) Result { + result := Result{TurnsA: len(a), TurnsB: len(b)} + + shorter := min(len(a), len(b)) + for i := range shorter { + if sameBehaviour(a[i], b[i]) { + result.TurnsMatched++ + continue + } + result.Divergence = &Divergence{ + Kind: KindToolCalls, + TurnIndex: i, + A: &a[i], + B: &b[i], + } + return result + } + + // The common prefix behaved identically; one run may have continued. + switch { + case len(b) > len(a): + result.Divergence = &Divergence{Kind: KindExtraTurn, TurnIndex: shorter, B: &b[shorter]} + case len(a) > len(b): + result.Divergence = &Divergence{Kind: KindMissingTurn, TurnIndex: shorter, A: &a[shorter]} + } + return result +} + +// CompareSessions is TurnsOf on both sides followed by Compare. +func CompareSessions(a, b *session.Session) Result { + return Compare(TurnsOf(a), TurnsOf(b)) +} + +// sameBehaviour reports whether two turns called the same tools with the same +// arguments, in the same order. Order matters: calling a tool before another is +// a different plan, even with the same set of calls. +func sameBehaviour(a, b Turn) bool { + if len(a.ToolCalls) != len(b.ToolCalls) { + return false + } + for i := range a.ToolCalls { + if a.ToolCalls[i] != b.ToolCalls[i] { + return false + } + } + return true +} + +// PrintResult writes a human-readable comparison. +func PrintResult(out io.Writer, r Result, nameA, nameB string) { + fmt.Fprintf(out, "Comparing %s (%d turns) against %s (%d turns)\n", nameA, r.TurnsA, nameB, r.TurnsB) + + if r.Identical() { + fmt.Fprintf(out, "\n✅ Identical behaviour across all %d turns.\n", r.TurnsMatched) + return + } + + d := r.Divergence + fmt.Fprintf(out, "\n❌ First divergence at turn %d (after %d matching turn(s)).\n", d.TurnIndex, r.TurnsMatched) + + switch d.Kind { + case KindExtraTurn: + fmt.Fprintf(out, " %s stopped; %s continued with:\n", nameA, nameB) + printCalls(out, d.B) + case KindMissingTurn: + fmt.Fprintf(out, " %s continued; %s stopped. %s did:\n", nameA, nameB, nameA) + printCalls(out, d.A) + case KindToolCalls: + fmt.Fprintf(out, " %s called:\n", nameA) + printCalls(out, d.A) + fmt.Fprintf(out, " %s called:\n", nameB) + printCalls(out, d.B) + } + + fmt.Fprintln(out, "\nEverything after this point is downstream of the divergence and is not compared.") +} + +func printCalls(out io.Writer, t *Turn) { + if t == nil { + return + } + if len(t.ToolCalls) == 0 { + fmt.Fprintln(out, " (no tool calls — a final answer)") + return + } + for _, tc := range t.ToolCalls { + fmt.Fprintf(out, " %s(%s)\n", tc.Name, truncateArgs(tc.Arguments)) + } +} + +// truncateArgs bounds argument text so a large payload cannot flood the report. +func truncateArgs(args string) string { + const maxRunes = 120 + args = strings.ReplaceAll(args, "\n", " ") + runes := []rune(args) + if len(runes) <= maxRunes { + return args + } + return string(runes[:maxRunes-1]) + "…" +} From 9189b1fa9fb59f26710b9939d4f350ddee62e9fe Mon Sep 17 00:00:00 2001 From: Dwin Gharibi Date: Fri, 7 Aug 2026 09:54:57 +0330 Subject: [PATCH 2/5] test(pkg/replay): adding some proper tests for replay compare --- cmd/root/replay_test.go | 73 +++++++++++ pkg/replay/replay_test.go | 270 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 343 insertions(+) create mode 100644 cmd/root/replay_test.go create mode 100644 pkg/replay/replay_test.go diff --git a/cmd/root/replay_test.go b/cmd/root/replay_test.go new file mode 100644 index 0000000000..e358169e92 --- /dev/null +++ b/cmd/root/replay_test.go @@ -0,0 +1,73 @@ +package root + +import ( + "bytes" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/docker/docker-agent/pkg/chat" + "github.com/docker/docker-agent/pkg/replay" + "github.com/docker/docker-agent/pkg/session" + "github.com/docker/docker-agent/pkg/tools" +) + +func replaySession(toolName string) *session.Session { + return &session.Session{Messages: []session.Item{ + {Message: &session.Message{Message: chat.Message{ + Role: chat.MessageRoleAssistant, + ToolCalls: []tools.ToolCall{ + {Function: tools.FunctionCall{Name: toolName, Arguments: "{}"}}, + }, + }}}, + }} +} + +func TestRenderReplay_TextDivergence(t *testing.T) { + t.Parallel() + + result := replay.CompareSessions(replaySession("read_file"), replaySession("shell")) + var buf bytes.Buffer + require.NoError(t, renderReplay(&buf, result, "aaa", "bbb", false)) + + out := buf.String() + assert.Contains(t, out, "First divergence at turn 0") + assert.Contains(t, out, "aaa") + assert.Contains(t, out, "bbb") +} + +func TestRenderReplay_TextIdentical(t *testing.T) { + t.Parallel() + + result := replay.CompareSessions(replaySession("read_file"), replaySession("read_file")) + var buf bytes.Buffer + require.NoError(t, renderReplay(&buf, result, "aaa", "bbb", false)) + assert.Contains(t, buf.String(), "Identical behaviour") +} + +func TestRenderReplay_JSON(t *testing.T) { + t.Parallel() + + result := replay.CompareSessions(replaySession("read_file"), replaySession("shell")) + var buf bytes.Buffer + require.NoError(t, renderReplay(&buf, result, "aaa", "bbb", true)) + + var round replay.Result + require.NoError(t, json.Unmarshal(buf.Bytes(), &round)) + require.NotNil(t, round.Divergence) + assert.Equal(t, 0, round.Divergence.TurnIndex) +} + +func TestReplayCmd_FlagsAreRegistered(t *testing.T) { + t.Parallel() + + cmd := newReplayCmd() + for _, name := range []string{"session-db", "json", "fail-on-divergence"} { + assert.NotNilf(t, cmd.Flags().Lookup(name), "flag %q must exist", name) + } + // Two session IDs, no more, no fewer. + require.Error(t, cmd.Args(cmd, []string{"only-one"})) + require.NoError(t, cmd.Args(cmd, []string{"a", "b"})) +} diff --git a/pkg/replay/replay_test.go b/pkg/replay/replay_test.go new file mode 100644 index 0000000000..5256e7be53 --- /dev/null +++ b/pkg/replay/replay_test.go @@ -0,0 +1,270 @@ +package replay_test + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/docker/docker-agent/pkg/chat" + "github.com/docker/docker-agent/pkg/replay" + "github.com/docker/docker-agent/pkg/session" + "github.com/docker/docker-agent/pkg/tools" +) + +func call(name, args string) replay.ToolCall { + return replay.ToolCall{Name: name, Arguments: args} +} + +func turn(content string, calls ...replay.ToolCall) replay.Turn { + return replay.Turn{Content: content, ToolCalls: calls} +} + +// indexed renumbers turns the way TurnsOf would, so hand-built fixtures compare +// cleanly against extracted ones. +func indexed(turns ...replay.Turn) []replay.Turn { + for i := range turns { + turns[i].Index = i + } + return turns +} + +func TestCompare_IdenticalBehaviour(t *testing.T) { + t.Parallel() + + a := indexed(turn("looking", call("read_file", `{"path":"a"}`)), turn("done")) + b := indexed(turn("having a look", call("read_file", `{"path":"a"}`)), turn("finished")) + + got := replay.Compare(a, b) + assert.True(t, got.Identical(), "different prose with identical tool calls is identical behaviour") + assert.Equal(t, 2, got.TurnsMatched) + assert.Nil(t, got.Divergence) +} + +// The point of the package: prose differences must never register. +func TestCompare_ProseIsNeverADivergence(t *testing.T) { + t.Parallel() + + a := indexed(turn("I will now read the file.")) + b := indexed(turn("Sure! Let me take a look at that file for you.")) + + assert.True(t, replay.Compare(a, b).Identical()) +} + +func TestCompare_DifferentToolName(t *testing.T) { + t.Parallel() + + a := indexed(turn("", call("read_file", `{"path":"a"}`))) + b := indexed(turn("", call("shell", `{"cmd":"cat a"}`))) + + got := replay.Compare(a, b) + require.NotNil(t, got.Divergence) + assert.Equal(t, replay.KindToolCalls, got.Divergence.Kind) + assert.Equal(t, 0, got.Divergence.TurnIndex) + assert.Zero(t, got.TurnsMatched) +} + +func TestCompare_SameToolDifferentArguments(t *testing.T) { + t.Parallel() + + a := indexed(turn("", call("read_file", `{"path":"a"}`))) + b := indexed(turn("", call("read_file", `{"path":"b"}`))) + + got := replay.Compare(a, b) + require.NotNil(t, got.Divergence) + assert.Equal(t, replay.KindToolCalls, got.Divergence.Kind) +} + +// The same set of calls in a different order is a different plan. +func TestCompare_ToolCallOrderMatters(t *testing.T) { + t.Parallel() + + a := indexed(turn("", call("read_file", "{}"), call("shell", "{}"))) + b := indexed(turn("", call("shell", "{}"), call("read_file", "{}"))) + + assert.False(t, replay.Compare(a, b).Identical()) +} + +func TestCompare_ReportsFirstDivergenceOnly(t *testing.T) { + t.Parallel() + + a := indexed( + turn("", call("read_file", `{"path":"a"}`)), + turn("", call("read_file", `{"path":"b"}`)), + turn("", call("read_file", `{"path":"c"}`)), + ) + b := indexed( + turn("", call("read_file", `{"path":"a"}`)), + turn("", call("shell", `{"cmd":"x"}`)), + turn("", call("shell", `{"cmd":"y"}`)), + ) + + got := replay.Compare(a, b) + require.NotNil(t, got.Divergence) + assert.Equal(t, 1, got.Divergence.TurnIndex, "the first difference, not the last") + assert.Equal(t, 1, got.TurnsMatched) + assert.Equal(t, "read_file", got.Divergence.A.ToolCalls[0].Name) + assert.Equal(t, "shell", got.Divergence.B.ToolCalls[0].Name) +} + +func TestCompare_LengthMismatchAfterMatchingPrefix(t *testing.T) { + t.Parallel() + + short := indexed(turn("", call("read_file", "{}"))) + long := indexed(turn("", call("read_file", "{}")), turn("", call("shell", "{}"))) + + t.Run("second run continued", func(t *testing.T) { + t.Parallel() + got := replay.Compare(short, long) + require.NotNil(t, got.Divergence) + assert.Equal(t, replay.KindExtraTurn, got.Divergence.Kind) + assert.Equal(t, 1, got.Divergence.TurnIndex) + assert.Nil(t, got.Divergence.A) + require.NotNil(t, got.Divergence.B) + }) + + t.Run("second run stopped early", func(t *testing.T) { + t.Parallel() + got := replay.Compare(long, short) + require.NotNil(t, got.Divergence) + assert.Equal(t, replay.KindMissingTurn, got.Divergence.Kind) + require.NotNil(t, got.Divergence.A) + assert.Nil(t, got.Divergence.B) + }) +} + +func TestCompare_EmptyRuns(t *testing.T) { + t.Parallel() + + assert.True(t, replay.Compare(nil, nil).Identical()) + + got := replay.Compare(nil, indexed(turn("", call("shell", "{}")))) + require.NotNil(t, got.Divergence) + assert.Equal(t, replay.KindExtraTurn, got.Divergence.Kind) +} + +func TestTurnsOf(t *testing.T) { + t.Parallel() + + sess := &session.Session{Messages: []session.Item{ + {Message: &session.Message{Message: chat.Message{Role: chat.MessageRoleUser, Content: "do it"}}}, + {Message: &session.Message{AgentName: "root", Message: chat.Message{ + Role: chat.MessageRoleAssistant, + Content: "reading", + ToolCalls: []tools.ToolCall{ + {Function: tools.FunctionCall{Name: "read_file", Arguments: `{"path":"a"}`}}, + }, + }}}, + {Message: &session.Message{Message: chat.Message{Role: chat.MessageRoleTool, Content: "contents"}}}, + {Message: &session.Message{AgentName: "root", Message: chat.Message{ + Role: chat.MessageRoleAssistant, Content: "done", + }}}, + {}, // a non-message item (e.g. a compaction summary) + }} + + got := replay.TurnsOf(sess) + + require.Len(t, got, 2, "only assistant messages are turns") + assert.Equal(t, 0, got[0].Index) + assert.Equal(t, "root", got[0].Agent) + assert.Equal(t, "reading", got[0].Content) + require.Len(t, got[0].ToolCalls, 1) + assert.Equal(t, "read_file", got[0].ToolCalls[0].Name) + assert.Equal(t, 1, got[1].Index) + assert.Empty(t, got[1].ToolCalls) +} + +func TestTurnsOf_NilSession(t *testing.T) { + t.Parallel() + assert.Empty(t, replay.TurnsOf(nil)) +} + +func TestCompareSessions(t *testing.T) { + t.Parallel() + + mk := func(toolName string) *session.Session { + return &session.Session{Messages: []session.Item{ + {Message: &session.Message{Message: chat.Message{ + Role: chat.MessageRoleAssistant, + ToolCalls: []tools.ToolCall{ + {Function: tools.FunctionCall{Name: toolName, Arguments: "{}"}}, + }, + }}}, + }} + } + + assert.True(t, replay.CompareSessions(mk("read_file"), mk("read_file")).Identical()) + assert.False(t, replay.CompareSessions(mk("read_file"), mk("shell")).Identical()) +} + +func TestPrintResult(t *testing.T) { + t.Parallel() + + t.Run("identical", func(t *testing.T) { + t.Parallel() + var buf bytes.Buffer + replay.PrintResult(&buf, replay.Compare( + indexed(turn("", call("read_file", "{}"))), + indexed(turn("", call("read_file", "{}"))), + ), "old", "new") + assert.Contains(t, buf.String(), "Identical behaviour") + }) + + t.Run("divergence names both sides", func(t *testing.T) { + t.Parallel() + var buf bytes.Buffer + replay.PrintResult(&buf, replay.Compare( + indexed(turn("", call("read_file", `{"path":"a"}`))), + indexed(turn("", call("shell", `{"cmd":"x"}`))), + ), "old", "new") + + out := buf.String() + assert.Contains(t, out, "First divergence at turn 0") + assert.Contains(t, out, "read_file") + assert.Contains(t, out, "shell") + assert.Contains(t, out, "old") + assert.Contains(t, out, "new") + assert.Contains(t, out, "downstream of the divergence") + }) + + t.Run("a final answer is labelled", func(t *testing.T) { + t.Parallel() + var buf bytes.Buffer + replay.PrintResult(&buf, replay.Compare( + indexed(turn("all done")), + indexed(turn("", call("shell", "{}"))), + ), "old", "new") + assert.Contains(t, buf.String(), "no tool calls") + }) + + t.Run("huge arguments are truncated", func(t *testing.T) { + t.Parallel() + var buf bytes.Buffer + huge := strings.Repeat("x", 5000) + replay.PrintResult(&buf, replay.Compare( + indexed(turn("", call("write_file", huge))), + indexed(turn("", call("shell", "{}"))), + ), "old", "new") + assert.Less(t, buf.Len(), 1000, "a large payload must not flood the report") + assert.Contains(t, buf.String(), "…") + }) +} + +func TestResult_IsJSONSerializable(t *testing.T) { + t.Parallel() + + r := replay.Compare( + indexed(turn("", call("read_file", "{}"))), + indexed(turn("", call("shell", "{}"))), + ) + data, err := json.Marshal(r) + require.NoError(t, err) + + var round replay.Result + require.NoError(t, json.Unmarshal(data, &round)) + require.NotNil(t, round.Divergence) + assert.Equal(t, replay.KindToolCalls, round.Divergence.Kind) +} From 3e8cb4bdbc7d6f334d606ad0bbd7219cb129ef3c Mon Sep 17 00:00:00 2001 From: Dwin Gharibi Date: Sat, 8 Aug 2026 22:46:09 +0330 Subject: [PATCH 3/5] fix(replay): compare sub-agent turns, arguments semantically, and sanitize output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sub-sessions were dropped. Delegated work lives in Item.SubSession, so in any multi-agent run the sub-agent's entire tool-call sequence was invisible and two runs whose sub-agents did entirely different things were reported as "✅ Identical behaviour" — the precise wrong answer to the question this package exists for. Turns are now walked in place, appearing in sequence where the delegation happened. Arguments were compared byte-exactly, so semantically identical JSON in a different key order registered as a divergence. That is the same nondeterminism noise "compare behaviour, not prose" exists to eliminate, so equal JSON values now compare equal, falling back to string equality when either side is not valid JSON. Tool arguments reach the terminal unsanitized. They are model-generated and routinely carry content the agent read from a file or the web, so a carriage return plus an erase-line sequence let the material being diagnosed redraw the diagnostic and display one tool call as another. Control characters are now stripped, as the markdown renderer already does. Printed calls per turn are also capped: per-call truncation alone is unbounded in the number of parallel calls. Also uses chat.MessageRoleAssistant rather than a bare string literal, which was the only such occurrence outside tests. --- cmd/root/{replay.go => sessions.go} | 0 cmd/root/{replay_test.go => sessions_test.go} | 0 pkg/replay/replay.go | 122 +++++++++++++++--- 3 files changed, 103 insertions(+), 19 deletions(-) rename cmd/root/{replay.go => sessions.go} (100%) rename cmd/root/{replay_test.go => sessions_test.go} (100%) diff --git a/cmd/root/replay.go b/cmd/root/sessions.go similarity index 100% rename from cmd/root/replay.go rename to cmd/root/sessions.go diff --git a/cmd/root/replay_test.go b/cmd/root/sessions_test.go similarity index 100% rename from cmd/root/replay_test.go rename to cmd/root/sessions_test.go diff --git a/pkg/replay/replay.go b/pkg/replay/replay.go index 78b7229f44..43d960fac2 100644 --- a/pkg/replay/replay.go +++ b/pkg/replay/replay.go @@ -16,6 +16,14 @@ // said around a divergence; it just does not decide whether a divergence // happened. // +// # Delegated work +// +// A turn taken by a sub-agent is still a turn the run took, so sub-sessions are +// walked in place: their turns appear in sequence where the delegation happened. +// Skipping them would report "identical behaviour" for two runs whose sub-agents +// did entirely different things — the precise wrong answer to the question this +// package exists for. +// // # First divergence only // // Once two runs differ, everything after that point is downstream of the @@ -24,10 +32,13 @@ package replay import ( + "encoding/json" "fmt" "io" + "reflect" "strings" + "github.com/docker/docker-agent/pkg/chat" "github.com/docker/docker-agent/pkg/session" ) @@ -86,29 +97,46 @@ type Result struct { // Identical reports whether the two runs behaved the same the whole way. func (r Result) Identical() bool { return r.Divergence == nil } -// TurnsOf extracts the assistant turns from a session, in order. +// TurnsOf extracts the assistant turns from a session, in order, including +// those taken by delegated sub-agents. // // Only assistant messages are turns: user messages are inputs and tool messages // are results, neither of which is a decision the model made. A nil session // yields no turns. func TurnsOf(sess *session.Session) []Turn { - if sess == nil { - return nil + var turns []Turn + appendTurns(&turns, sess, 0) + return turns +} + +// maxSubSessionDepth bounds the sub-session walk. Delegation nests only a few +// levels in practice; the bound exists so a cyclic or pathological session +// cannot recurse without end. +const maxSubSessionDepth = 32 + +func appendTurns(turns *[]Turn, sess *session.Session, depth int) { + if sess == nil || depth > maxSubSessionDepth { + return } - var turns []Turn - for i := range sess.Messages { - item := &sess.Messages[i] - if item.Message == nil { + // MessagesSnapshot copies under the session lock, so a live session being + // written to cannot race this walk. + for _, item := range sess.MessagesSnapshot() { + if item.IsSubSession() { + appendTurns(turns, item.SubSession, depth+1) + continue + } + if !item.IsMessage() { continue } + msg := &item.Message.Message - if msg.Role != "assistant" { + if msg.Role != chat.MessageRoleAssistant { continue } turn := Turn{ - Index: len(turns), + Index: len(*turns), Agent: item.Message.AgentName, Content: msg.Content, } @@ -118,9 +146,8 @@ func TurnsOf(sess *session.Session) []Turn { Arguments: tc.Function.Arguments, }) } - turns = append(turns, turn) + *turns = append(*turns, turn) } - return turns } // Compare walks two runs' turns and reports the first behavioural divergence. @@ -165,13 +192,36 @@ func sameBehaviour(a, b Turn) bool { return false } for i := range a.ToolCalls { - if a.ToolCalls[i] != b.ToolCalls[i] { + if a.ToolCalls[i].Name != b.ToolCalls[i].Name { + return false + } + if !sameArguments(a.ToolCalls[i].Arguments, b.ToolCalls[i].Arguments) { return false } } return true } +// sameArguments compares two tool-call argument payloads semantically. +// +// Argument JSON is model-generated, so key order and whitespace vary run to run +// while the call means the same thing. A byte comparison would report those as +// divergences — the same nondeterminism noise that "compare behaviour, not +// prose" exists to eliminate — so equal JSON values compare equal. +// +// Falls back to string equality when either side is not valid JSON, which is +// the only honest answer for a payload with no structure to compare. +func sameArguments(a, b string) bool { + if a == b { + return true + } + var av, bv any + if json.Unmarshal([]byte(a), &av) != nil || json.Unmarshal([]byte(b), &bv) != nil { + return false + } + return reflect.DeepEqual(av, bv) +} + // PrintResult writes a human-readable comparison. func PrintResult(out io.Writer, r Result, nameA, nameB string) { fmt.Fprintf(out, "Comparing %s (%d turns) against %s (%d turns)\n", nameA, r.TurnsA, nameB, r.TurnsB) @@ -209,18 +259,52 @@ func printCalls(out io.Writer, t *Turn) { fmt.Fprintln(out, " (no tool calls — a final answer)") return } - for _, tc := range t.ToolCalls { - fmt.Fprintf(out, " %s(%s)\n", tc.Name, truncateArgs(tc.Arguments)) + for i, tc := range t.ToolCalls { + if i == maxPrintedCalls { + fmt.Fprintf(out, " … and %d more call(s)\n", len(t.ToolCalls)-maxPrintedCalls) + return + } + fmt.Fprintf(out, " %s(%s)\n", sanitizeForTerminal(tc.Name), truncateArgs(tc.Arguments)) } } -// truncateArgs bounds argument text so a large payload cannot flood the report. +const ( + // maxArgRunes bounds one call's rendered arguments. + maxArgRunes = 120 + // maxPrintedCalls bounds how many calls of a single turn are rendered. + // Per-call truncation alone is unbounded in the number of parallel calls, so + // a turn with hundreds of them could still flood the report. + maxPrintedCalls = 10 +) + +// truncateArgs bounds argument text and strips terminal control characters. func truncateArgs(args string) string { - const maxRunes = 120 - args = strings.ReplaceAll(args, "\n", " ") + args = sanitizeForTerminal(args) runes := []rune(args) - if len(runes) <= maxRunes { + if len(runes) <= maxArgRunes { return args } - return string(runes[:maxRunes-1]) + "…" + return string(runes[:maxArgRunes-1]) + "…" +} + +// sanitizeForTerminal removes control characters that let text redraw the +// terminal rather than appear in it. +// +// Tool arguments are model-generated and routinely carry content the agent read +// from a file or the web, so the diagnostic's own output can be shaped by the +// material being diagnosed: a carriage return plus an erase-line sequence +// rewrites the line, letting one tool call be displayed as another. The runtime +// scrubs the same characters before rendering markdown, for the same reason. +func sanitizeForTerminal(s string) string { + return strings.Map(func(r rune) rune { + switch r { + case '\n', '\r', '\b', '\f', '\v': + return ' ' + } + // C0 controls (including ESC) and DEL. + if r < 0x20 || r == 0x7f { + return ' ' + } + return r + }, s) } From ea2cbf411f05f15e118c49f1d9da7076e0d6ddb5 Mon Sep 17 00:00:00 2001 From: Dwin Gharibi Date: Sat, 8 Aug 2026 22:46:10 +0330 Subject: [PATCH 4/5] refactor(cli): make this `sessions diff`, and resolve session references MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pkg/recording already owns "replay" for recording and replaying API interactions, and `--record` writes cassettes. This command replays nothing — it diffs two recordings — and taking the word would also leave the re-run-against-another-model feature, which actually is a replay, without a name. It is now `docker agent sessions diff`. References also go through session.ResolveSessionID like every other session-consuming command, plus an unambiguous ID prefix. Neither worked before, so the advertised workflow failed on both spellings a user can realistically produce: "compare my last two runs" (`-1 -2`) and the shortened IDs printed by other commands. Full UUIDs are the hardest thing to obtain by hand and were the only accepted form. Adds tests for run and --fail-on-divergence, whose non-zero exit is the whole contract of the flag for CI and had no coverage at all. --- cmd/root/root.go | 2 +- cmd/root/sessions.go | 89 ++++++++++++++++++++++++----- cmd/root/sessions_test.go | 116 +++++++++++++++++++++++++++++++++++++- 3 files changed, 189 insertions(+), 18 deletions(-) diff --git a/cmd/root/root.go b/cmd/root/root.go index 8bc5845af0..c2947d9928 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -177,7 +177,7 @@ We collect anonymous usage data to help improve docker agent. To disable: newNewCmd(), newGettingStartedCmd(), newEvalCmd(), - newReplayCmd(), + newSessionsCmd(), newShareCmd(), newModelsCmd(), newToolsetsCmd(), diff --git a/cmd/root/sessions.go b/cmd/root/sessions.go index 188838071e..8b060c8d39 100644 --- a/cmd/root/sessions.go +++ b/cmd/root/sessions.go @@ -1,33 +1,52 @@ package root import ( + "context" "encoding/json" "errors" "fmt" "io" "log/slog" + "strings" "github.com/spf13/cobra" pathx "github.com/docker/docker-agent/pkg/path" "github.com/docker/docker-agent/pkg/replay" + "github.com/docker/docker-agent/pkg/session" "github.com/docker/docker-agent/pkg/session/sqlitestore" ) -type replayFlags struct { +type sessionsDiffFlags struct { sessionDB string asJSON bool failOnDiff bool } -func newReplayCmd() *cobra.Command { - var flags replayFlags - +// newSessionsCmd groups session-inspection subcommands. +// +// Deliberately not called "replay": pkg/recording already owns that word for +// recording and replaying API interactions, and `--record` writes cassettes. +// This command replays nothing — it diffs two recordings — and naming it replay +// would also take the word from the re-run-against-another-model feature that +// actually is a replay. +func newSessionsCmd() *cobra.Command { cmd := &cobra.Command{ - Use: "replay ", - Short: "Compare the behaviour of two recorded sessions", + Use: "sessions", + Short: "Inspect recorded sessions", GroupID: "advanced", - Args: cobra.ExactArgs(2), + } + cmd.AddCommand(newSessionsDiffCmd()) + return cmd +} + +func newSessionsDiffCmd() *cobra.Command { + var flags sessionsDiffFlags + + cmd := &cobra.Command{ + Use: "diff ", + Short: "Compare the behaviour of two recorded sessions", + Args: cobra.ExactArgs(2), Long: `Compare two recorded sessions and report the first point where the agent behaved differently. @@ -39,9 +58,10 @@ they are what is compared. Reporting stops at the first divergence: everything after it is downstream of that difference and comparing it produces noise rather than information.`, - Example: ` docker agent replay - docker agent replay --json | jq '.divergence.turn_index' - docker agent replay --fail-on-divergence`, + Example: ` docker agent sessions diff + docker agent sessions diff -1 -2 + docker agent sessions diff --json | jq '.divergence.turn_index' + docker agent sessions diff --fail-on-divergence`, RunE: flags.run, } @@ -52,7 +72,7 @@ difference and comparing it produces noise rather than information.`, return cmd } -func (f *replayFlags) run(cmd *cobra.Command, args []string) error { +func (f *sessionsDiffFlags) run(cmd *cobra.Command, args []string) error { ctx := cmd.Context() dbPath, err := pathx.ExpandHomeDir(sessionDBPath(f.sessionDB)) @@ -70,13 +90,13 @@ func (f *replayFlags) run(cmd *cobra.Command, args []string) error { } }() - sessA, err := store.GetSession(ctx, args[0]) + sessA, err := loadSessionRef(ctx, store, args[0]) if err != nil { - return fmt.Errorf("reading session %q: %w", args[0], err) + return err } - sessB, err := store.GetSession(ctx, args[1]) + sessB, err := loadSessionRef(ctx, store, args[1]) if err != nil { - return fmt.Errorf("reading session %q: %w", args[1], err) + return err } result := replay.CompareSessions(sessA, sessB) @@ -90,6 +110,45 @@ func (f *replayFlags) run(cmd *cobra.Command, args []string) error { return nil } +// loadSessionRef resolves a user-supplied reference and loads the session. +// +// References go through session.ResolveSessionID like every other +// session-consuming command, so relative forms work — "compare my last two +// runs" is `sessions diff -1 -2`. An unambiguous ID prefix is accepted too, +// since full UUIDs are the hardest thing for a user to produce by hand. +func loadSessionRef(ctx context.Context, store session.Store, ref string) (*session.Session, error) { + id, err := session.ResolveSessionID(ctx, store, ref) + if err != nil { + return nil, err + } + if sess, err := store.GetSession(ctx, id); err == nil { + return sess, nil + } + + summaries, err := store.GetSessionSummaries(ctx) + if err != nil { + return nil, fmt.Errorf("listing sessions: %w", err) + } + var matches []string + for _, summary := range summaries { + if strings.HasPrefix(summary.ID, id) { + matches = append(matches, summary.ID) + } + } + switch len(matches) { + case 1: + sess, err := store.GetSession(ctx, matches[0]) + if err != nil { + return nil, fmt.Errorf("reading session %q: %w", ref, err) + } + return sess, nil + case 0: + return nil, fmt.Errorf("no session matches %q", ref) + default: + return nil, fmt.Errorf("%q matches %d sessions; use more characters", ref, len(matches)) + } +} + // renderReplay writes the comparison as JSON or as text. func renderReplay(w io.Writer, result replay.Result, nameA, nameB string, asJSON bool) error { if asJSON { diff --git a/cmd/root/sessions_test.go b/cmd/root/sessions_test.go index e358169e92..1766cb221b 100644 --- a/cmd/root/sessions_test.go +++ b/cmd/root/sessions_test.go @@ -3,7 +3,9 @@ package root import ( "bytes" "encoding/json" + "path/filepath" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -11,6 +13,7 @@ import ( "github.com/docker/docker-agent/pkg/chat" "github.com/docker/docker-agent/pkg/replay" "github.com/docker/docker-agent/pkg/session" + "github.com/docker/docker-agent/pkg/session/sqlitestore" "github.com/docker/docker-agent/pkg/tools" ) @@ -60,10 +63,10 @@ func TestRenderReplay_JSON(t *testing.T) { assert.Equal(t, 0, round.Divergence.TurnIndex) } -func TestReplayCmd_FlagsAreRegistered(t *testing.T) { +func TestSessionsDiffCmd_FlagsAreRegistered(t *testing.T) { t.Parallel() - cmd := newReplayCmd() + cmd := newSessionsDiffCmd() for _, name := range []string{"session-db", "json", "fail-on-divergence"} { assert.NotNilf(t, cmd.Flags().Lookup(name), "flag %q must exist", name) } @@ -71,3 +74,112 @@ func TestReplayCmd_FlagsAreRegistered(t *testing.T) { require.Error(t, cmd.Args(cmd, []string{"only-one"})) require.NoError(t, cmd.Args(cmd, []string{"a", "b"})) } + +// diffFixture builds a real store with two sessions whose behaviour differs. +func diffFixture(t *testing.T) (string, *session.Session, *session.Session) { + t.Helper() + + dbPath := filepath.Join(t.TempDir(), "s.db") + store, err := sqlitestore.New(t.Context(), dbPath) + require.NoError(t, err) + + mk := func(id, toolName string, created time.Time) *session.Session { + s := &session.Session{ID: id, CreatedAt: created, Messages: []session.Item{ + {Message: &session.Message{Message: chat.Message{ + Role: chat.MessageRoleAssistant, + ToolCalls: []tools.ToolCall{ + {Function: tools.FunctionCall{Name: toolName, Arguments: "{}"}}, + }, + }}}, + }} + require.NoError(t, store.AddSession(t.Context(), s)) + return s + } + + now := time.Now() + a := mk("aaaaaaaa11111111", "read_file", now.Add(-2*time.Hour)) + b := mk("bbbbbbbb22222222", "shell", now.Add(-time.Hour)) + require.NoError(t, store.Close()) + + return dbPath, a, b +} + +func runSessionsDiff(t *testing.T, dbPath string, args ...string) (string, error) { + t.Helper() + + cmd := newSessionsDiffCmd() + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetErr(&out) + cmd.SetContext(t.Context()) + require.NoError(t, cmd.Flags().Set("session-db", dbPath)) + for i := 0; i+1 < len(args); i += 2 { + require.NoError(t, cmd.Flags().Set(args[i], args[i+1])) + } + // Run first: operands of a return statement are evaluated left to right, so + // reading the buffer in the same statement would capture it before the run. + err := cmd.RunE(cmd, []string{"aaaaaaaa11111111", "bbbbbbbb22222222"}) + return out.String(), err +} + +// The non-zero exit is the whole contract of --fail-on-divergence for the CI +// use case it exists for. +func TestSessionsDiff_FailOnDivergence(t *testing.T) { + t.Parallel() + + dbPath, _, _ := diffFixture(t) + + out, err := runSessionsDiff(t, dbPath) + require.NoError(t, err, "without the flag a divergence is reported but does not fail") + assert.Contains(t, out, "First divergence") + + out, err = runSessionsDiff(t, dbPath, "fail-on-divergence", "true") + require.Error(t, err, "with the flag a divergence must exit non-zero") + assert.Contains(t, err.Error(), "diverged") + assert.Contains(t, out, "First divergence") +} + +// References go through ResolveSessionID like every other session command, so +// "compare my last two runs" works. +func TestSessionsDiff_ResolvesRelativeAndPrefixRefs(t *testing.T) { + t.Parallel() + + dbPath, a, b := diffFixture(t) + + store, err := sqlitestore.New(t.Context(), dbPath) + require.NoError(t, err) + defer func() { require.NoError(t, store.Close()) }() + + // Relative. + for _, ref := range []string{"-1", "-2"} { + got, err := loadSessionRef(t.Context(), store, ref) + require.NoErrorf(t, err, "relative ref %q must resolve", ref) + require.NotNil(t, got) + } + + // Prefix. + got, err := loadSessionRef(t.Context(), store, a.ID[:8]) + require.NoError(t, err) + assert.Equal(t, a.ID, got.ID) + + got, err = loadSessionRef(t.Context(), store, b.ID) + require.NoError(t, err) + assert.Equal(t, b.ID, got.ID) + + _, err = loadSessionRef(t.Context(), store, "nosuchsession") + require.Error(t, err) + assert.Contains(t, err.Error(), "no session matches") +} + +func TestSessionsCmd_HasDiffSubcommand(t *testing.T) { + t.Parallel() + + cmd := newSessionsCmd() + assert.Equal(t, "sessions", cmd.Name()) + + var names []string + for _, sub := range cmd.Commands() { + names = append(names, sub.Name()) + } + assert.Contains(t, names, "diff") +} From 98852088e4c98684c213e0a11e331a17ef7d85f6 Mon Sep 17 00:00:00 2001 From: Dwin Gharibi Date: Sat, 8 Aug 2026 22:46:10 +0330 Subject: [PATCH 5/5] docs(cli): document `docker agent sessions diff` Includes the scope limit from the PR description: this locates where two runs diverged, not why. --- docs/features/cli/index.md | 42 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/docs/features/cli/index.md b/docs/features/cli/index.md index 23cac6ecfe..03a274acf5 100644 --- a/docs/features/cli/index.md +++ b/docs/features/cli/index.md @@ -447,6 +447,48 @@ $ docker agent share pull docker.io/username/my-agent:latest --force See [Agent Distribution](../../concepts/distribution/index.md) for full registry workflow details. +### `docker agent sessions diff` + +Compare two recorded sessions and report the first point where the agent behaved +differently — the triage answer when a task that worked yesterday does not work +today. + +```bash +$ docker agent sessions diff [flags] +``` + +```console +$ docker agent sessions diff -1 -2 +Comparing a1b2c3d4 (7 turns) against e5f6a7b8 (9 turns) + +❌ First divergence at turn 3 (after 3 matching turn(s)). + a1b2c3d4 called: + read_file({"path":"pkg/cache/cache.go"}) + e5f6a7b8 called: + search_files_content({"query":"persistToDisk","path":"."}) + +Everything after this point is downstream of the divergence and is not compared. +``` + +Session references accept a full ID, a unique ID prefix, or a relative form such +as `-1` for the most recent run. + +| Flag | Default | Description | +| ----------------------- | ----------------------- | ------------------------------------------------ | +| `-s, --session-db` | `/session.db` | Path to the session database | +| `--json` | `false` | Emit the comparison as JSON | +| `--fail-on-divergence` | `false` | Exit non-zero when the two sessions diverge | + +Comparison is over the sequence of tool calls, not the assistant's prose: model +output is nondeterministic, so two runs of the same task almost always word +things differently while doing the same work. Turns taken by delegated +sub-agents are included in sequence. Reporting stops at the first divergence — +everything after it is downstream of that difference. + +This locates *where* two runs diverged, not *why*. Re-running a session against +a different model while holding the environment fixed is a separate, unbuilt +feature. + ### `docker agent eval` Run agent evaluations against a directory of recorded sessions.