diff --git a/cmd/root/root.go b/cmd/root/root.go index 1ff4f1ef46..c2947d9928 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -177,6 +177,7 @@ We collect anonymous usage data to help improve docker agent. To disable: newNewCmd(), newGettingStartedCmd(), newEvalCmd(), + newSessionsCmd(), newShareCmd(), newModelsCmd(), newToolsetsCmd(), diff --git a/cmd/root/sessions.go b/cmd/root/sessions.go new file mode 100644 index 0000000000..8b060c8d39 --- /dev/null +++ b/cmd/root/sessions.go @@ -0,0 +1,161 @@ +package root + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "strings" + + "github.com/spf13/cobra" + + pathx "github.com/docker/docker-agent/pkg/path" + "github.com/docker/docker-agent/pkg/replay" + "github.com/docker/docker-agent/pkg/session" + "github.com/docker/docker-agent/pkg/session/sqlitestore" +) + +type sessionsDiffFlags struct { + sessionDB string + asJSON bool + failOnDiff bool +} + +// newSessionsCmd groups session-inspection subcommands. +// +// Deliberately not called "replay": pkg/recording already owns that word for +// recording and replaying API interactions, and `--record` writes cassettes. +// This command replays nothing — it diffs two recordings — and naming it replay +// would also take the word from the re-run-against-another-model feature that +// actually is a replay. +func newSessionsCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "sessions", + Short: "Inspect recorded sessions", + GroupID: "advanced", + } + cmd.AddCommand(newSessionsDiffCmd()) + return cmd +} + +func newSessionsDiffCmd() *cobra.Command { + var flags sessionsDiffFlags + + cmd := &cobra.Command{ + Use: "diff ", + Short: "Compare the behaviour of two recorded sessions", + Args: cobra.ExactArgs(2), + Long: `Compare two recorded sessions and report the first point where the agent +behaved differently. + +Comparison is over the sequence of tool calls, not over the assistant's prose. +Model output is nondeterministic: two runs of the same task almost always word +things differently while doing exactly the same work, so diffing text would report +a difference on every comparison. The tool calls are what changed the world, so +they are what is compared. + +Reporting stops at the first divergence: everything after it is downstream of that +difference and comparing it produces noise rather than information.`, + Example: ` docker agent sessions diff + docker agent sessions diff -1 -2 + docker agent sessions diff --json | jq '.divergence.turn_index' + docker agent sessions diff --fail-on-divergence`, + RunE: flags.run, + } + + cmd.Flags().StringVarP(&flags.sessionDB, "session-db", "s", "", "Path to the session database (default: /session.db)") + cmd.Flags().BoolVar(&flags.asJSON, "json", false, "Emit the comparison as JSON") + cmd.Flags().BoolVar(&flags.failOnDiff, "fail-on-divergence", false, "Exit non-zero when the two sessions diverge") + + return cmd +} + +func (f *sessionsDiffFlags) run(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + dbPath, err := pathx.ExpandHomeDir(sessionDBPath(f.sessionDB)) + if err != nil { + return err + } + + store, err := sqlitestore.New(ctx, dbPath) + if err != nil { + return fmt.Errorf("opening session store: %w", err) + } + defer func() { + if err := store.Close(); err != nil { + slog.ErrorContext(ctx, "Failed to close session store", "error", err) + } + }() + + sessA, err := loadSessionRef(ctx, store, args[0]) + if err != nil { + return err + } + sessB, err := loadSessionRef(ctx, store, args[1]) + if err != nil { + return err + } + + result := replay.CompareSessions(sessA, sessB) + if err := renderReplay(cmd.OutOrStdout(), result, args[0], args[1], f.asJSON); err != nil { + return err + } + + if f.failOnDiff && !result.Identical() { + return errors.New("sessions diverged") + } + return nil +} + +// loadSessionRef resolves a user-supplied reference and loads the session. +// +// References go through session.ResolveSessionID like every other +// session-consuming command, so relative forms work — "compare my last two +// runs" is `sessions diff -1 -2`. An unambiguous ID prefix is accepted too, +// since full UUIDs are the hardest thing for a user to produce by hand. +func loadSessionRef(ctx context.Context, store session.Store, ref string) (*session.Session, error) { + id, err := session.ResolveSessionID(ctx, store, ref) + if err != nil { + return nil, err + } + if sess, err := store.GetSession(ctx, id); err == nil { + return sess, nil + } + + summaries, err := store.GetSessionSummaries(ctx) + if err != nil { + return nil, fmt.Errorf("listing sessions: %w", err) + } + var matches []string + for _, summary := range summaries { + if strings.HasPrefix(summary.ID, id) { + matches = append(matches, summary.ID) + } + } + switch len(matches) { + case 1: + sess, err := store.GetSession(ctx, matches[0]) + if err != nil { + return nil, fmt.Errorf("reading session %q: %w", ref, err) + } + return sess, nil + case 0: + return nil, fmt.Errorf("no session matches %q", ref) + default: + return nil, fmt.Errorf("%q matches %d sessions; use more characters", ref, len(matches)) + } +} + +// renderReplay writes the comparison as JSON or as text. +func renderReplay(w io.Writer, result replay.Result, nameA, nameB string, asJSON bool) error { + if asJSON { + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(result) + } + replay.PrintResult(w, result, nameA, nameB) + return nil +} diff --git a/cmd/root/sessions_test.go b/cmd/root/sessions_test.go new file mode 100644 index 0000000000..1766cb221b --- /dev/null +++ b/cmd/root/sessions_test.go @@ -0,0 +1,185 @@ +package root + +import ( + "bytes" + "encoding/json" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/docker/docker-agent/pkg/chat" + "github.com/docker/docker-agent/pkg/replay" + "github.com/docker/docker-agent/pkg/session" + "github.com/docker/docker-agent/pkg/session/sqlitestore" + "github.com/docker/docker-agent/pkg/tools" +) + +func replaySession(toolName string) *session.Session { + return &session.Session{Messages: []session.Item{ + {Message: &session.Message{Message: chat.Message{ + Role: chat.MessageRoleAssistant, + ToolCalls: []tools.ToolCall{ + {Function: tools.FunctionCall{Name: toolName, Arguments: "{}"}}, + }, + }}}, + }} +} + +func TestRenderReplay_TextDivergence(t *testing.T) { + t.Parallel() + + result := replay.CompareSessions(replaySession("read_file"), replaySession("shell")) + var buf bytes.Buffer + require.NoError(t, renderReplay(&buf, result, "aaa", "bbb", false)) + + out := buf.String() + assert.Contains(t, out, "First divergence at turn 0") + assert.Contains(t, out, "aaa") + assert.Contains(t, out, "bbb") +} + +func TestRenderReplay_TextIdentical(t *testing.T) { + t.Parallel() + + result := replay.CompareSessions(replaySession("read_file"), replaySession("read_file")) + var buf bytes.Buffer + require.NoError(t, renderReplay(&buf, result, "aaa", "bbb", false)) + assert.Contains(t, buf.String(), "Identical behaviour") +} + +func TestRenderReplay_JSON(t *testing.T) { + t.Parallel() + + result := replay.CompareSessions(replaySession("read_file"), replaySession("shell")) + var buf bytes.Buffer + require.NoError(t, renderReplay(&buf, result, "aaa", "bbb", true)) + + var round replay.Result + require.NoError(t, json.Unmarshal(buf.Bytes(), &round)) + require.NotNil(t, round.Divergence) + assert.Equal(t, 0, round.Divergence.TurnIndex) +} + +func TestSessionsDiffCmd_FlagsAreRegistered(t *testing.T) { + t.Parallel() + + cmd := newSessionsDiffCmd() + for _, name := range []string{"session-db", "json", "fail-on-divergence"} { + assert.NotNilf(t, cmd.Flags().Lookup(name), "flag %q must exist", name) + } + // Two session IDs, no more, no fewer. + require.Error(t, cmd.Args(cmd, []string{"only-one"})) + require.NoError(t, cmd.Args(cmd, []string{"a", "b"})) +} + +// diffFixture builds a real store with two sessions whose behaviour differs. +func diffFixture(t *testing.T) (string, *session.Session, *session.Session) { + t.Helper() + + dbPath := filepath.Join(t.TempDir(), "s.db") + store, err := sqlitestore.New(t.Context(), dbPath) + require.NoError(t, err) + + mk := func(id, toolName string, created time.Time) *session.Session { + s := &session.Session{ID: id, CreatedAt: created, Messages: []session.Item{ + {Message: &session.Message{Message: chat.Message{ + Role: chat.MessageRoleAssistant, + ToolCalls: []tools.ToolCall{ + {Function: tools.FunctionCall{Name: toolName, Arguments: "{}"}}, + }, + }}}, + }} + require.NoError(t, store.AddSession(t.Context(), s)) + return s + } + + now := time.Now() + a := mk("aaaaaaaa11111111", "read_file", now.Add(-2*time.Hour)) + b := mk("bbbbbbbb22222222", "shell", now.Add(-time.Hour)) + require.NoError(t, store.Close()) + + return dbPath, a, b +} + +func runSessionsDiff(t *testing.T, dbPath string, args ...string) (string, error) { + t.Helper() + + cmd := newSessionsDiffCmd() + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetErr(&out) + cmd.SetContext(t.Context()) + require.NoError(t, cmd.Flags().Set("session-db", dbPath)) + for i := 0; i+1 < len(args); i += 2 { + require.NoError(t, cmd.Flags().Set(args[i], args[i+1])) + } + // Run first: operands of a return statement are evaluated left to right, so + // reading the buffer in the same statement would capture it before the run. + err := cmd.RunE(cmd, []string{"aaaaaaaa11111111", "bbbbbbbb22222222"}) + return out.String(), err +} + +// The non-zero exit is the whole contract of --fail-on-divergence for the CI +// use case it exists for. +func TestSessionsDiff_FailOnDivergence(t *testing.T) { + t.Parallel() + + dbPath, _, _ := diffFixture(t) + + out, err := runSessionsDiff(t, dbPath) + require.NoError(t, err, "without the flag a divergence is reported but does not fail") + assert.Contains(t, out, "First divergence") + + out, err = runSessionsDiff(t, dbPath, "fail-on-divergence", "true") + require.Error(t, err, "with the flag a divergence must exit non-zero") + assert.Contains(t, err.Error(), "diverged") + assert.Contains(t, out, "First divergence") +} + +// References go through ResolveSessionID like every other session command, so +// "compare my last two runs" works. +func TestSessionsDiff_ResolvesRelativeAndPrefixRefs(t *testing.T) { + t.Parallel() + + dbPath, a, b := diffFixture(t) + + store, err := sqlitestore.New(t.Context(), dbPath) + require.NoError(t, err) + defer func() { require.NoError(t, store.Close()) }() + + // Relative. + for _, ref := range []string{"-1", "-2"} { + got, err := loadSessionRef(t.Context(), store, ref) + require.NoErrorf(t, err, "relative ref %q must resolve", ref) + require.NotNil(t, got) + } + + // Prefix. + got, err := loadSessionRef(t.Context(), store, a.ID[:8]) + require.NoError(t, err) + assert.Equal(t, a.ID, got.ID) + + got, err = loadSessionRef(t.Context(), store, b.ID) + require.NoError(t, err) + assert.Equal(t, b.ID, got.ID) + + _, err = loadSessionRef(t.Context(), store, "nosuchsession") + require.Error(t, err) + assert.Contains(t, err.Error(), "no session matches") +} + +func TestSessionsCmd_HasDiffSubcommand(t *testing.T) { + t.Parallel() + + cmd := newSessionsCmd() + assert.Equal(t, "sessions", cmd.Name()) + + var names []string + for _, sub := range cmd.Commands() { + names = append(names, sub.Name()) + } + assert.Contains(t, names, "diff") +} diff --git a/docs/features/cli/index.md b/docs/features/cli/index.md index 23cac6ecfe..03a274acf5 100644 --- a/docs/features/cli/index.md +++ b/docs/features/cli/index.md @@ -447,6 +447,48 @@ $ docker agent share pull docker.io/username/my-agent:latest --force See [Agent Distribution](../../concepts/distribution/index.md) for full registry workflow details. +### `docker agent sessions diff` + +Compare two recorded sessions and report the first point where the agent behaved +differently — the triage answer when a task that worked yesterday does not work +today. + +```bash +$ docker agent sessions diff [flags] +``` + +```console +$ docker agent sessions diff -1 -2 +Comparing a1b2c3d4 (7 turns) against e5f6a7b8 (9 turns) + +❌ First divergence at turn 3 (after 3 matching turn(s)). + a1b2c3d4 called: + read_file({"path":"pkg/cache/cache.go"}) + e5f6a7b8 called: + search_files_content({"query":"persistToDisk","path":"."}) + +Everything after this point is downstream of the divergence and is not compared. +``` + +Session references accept a full ID, a unique ID prefix, or a relative form such +as `-1` for the most recent run. + +| Flag | Default | Description | +| ----------------------- | ----------------------- | ------------------------------------------------ | +| `-s, --session-db` | `/session.db` | Path to the session database | +| `--json` | `false` | Emit the comparison as JSON | +| `--fail-on-divergence` | `false` | Exit non-zero when the two sessions diverge | + +Comparison is over the sequence of tool calls, not the assistant's prose: model +output is nondeterministic, so two runs of the same task almost always word +things differently while doing the same work. Turns taken by delegated +sub-agents are included in sequence. Reporting stops at the first divergence — +everything after it is downstream of that difference. + +This locates *where* two runs diverged, not *why*. Re-running a session against +a different model while holding the environment fixed is a separate, unbuilt +feature. + ### `docker agent eval` Run agent evaluations against a directory of recorded sessions. diff --git a/pkg/replay/replay.go b/pkg/replay/replay.go new file mode 100644 index 0000000000..43d960fac2 --- /dev/null +++ b/pkg/replay/replay.go @@ -0,0 +1,310 @@ +// Package replay compares the behaviour of two recorded agent sessions. +// +// It answers the triage question "did the agent do something different this +// time, and where did it first diverge?" — the thing you actually want to know +// when a task that worked yesterday does not work today. +// +// # Behaviour, not prose +// +// Comparison is over the sequence of tool calls, not over the assistant's text. +// Model output is nondeterministic: two runs of the same task almost always word +// things differently while doing exactly the same work, so diffing prose reports +// a difference on essentially every comparison and is useless as a signal. The +// tool calls are what changed the world, so they are what is compared. +// +// Assistant text is still carried on each [Turn] so a reporter can show what was +// said around a divergence; it just does not decide whether a divergence +// happened. +// +// # Delegated work +// +// A turn taken by a sub-agent is still a turn the run took, so sub-sessions are +// walked in place: their turns appear in sequence where the delegation happened. +// Skipping them would report "identical behaviour" for two runs whose sub-agents +// did entirely different things — the precise wrong answer to the question this +// package exists for. +// +// # First divergence only +// +// Once two runs differ, everything after that point is downstream of the +// difference and comparing it produces noise, not information. So the comparison +// stops at the first divergence and reports where it was. +package replay + +import ( + "encoding/json" + "fmt" + "io" + "reflect" + "strings" + + "github.com/docker/docker-agent/pkg/chat" + "github.com/docker/docker-agent/pkg/session" +) + +// ToolCall is the comparable part of a tool invocation. +type ToolCall struct { + Name string `json:"name"` + Arguments string `json:"arguments"` +} + +// Turn is one assistant turn: what it said, and what it called. +type Turn struct { + // Index is the turn's 0-based position in the session. + Index int `json:"index"` + // Agent is the agent that produced the turn, so a divergence in a + // multi-agent run can be attributed. + Agent string `json:"agent,omitempty"` + // Content is the assistant text. Carried for reporting; never compared. + Content string `json:"content,omitempty"` + ToolCalls []ToolCall `json:"tool_calls,omitempty"` +} + +// Kind classifies how two runs differ. +type Kind string + +const ( + // KindToolCalls means both runs made a turn at this index but called + // different tools, or the same tools with different arguments. + KindToolCalls Kind = "tool_calls" + // KindExtraTurn means the second run kept going after the first stopped. + KindExtraTurn Kind = "extra_turn" + // KindMissingTurn means the second run stopped before the first did. + KindMissingTurn Kind = "missing_turn" +) + +// Divergence is the first behavioural difference found. +type Divergence struct { + Kind Kind `json:"kind"` + // TurnIndex is the 0-based turn at which the runs first differ. + TurnIndex int `json:"turn_index"` + // A and B are the diverging turns. Either may be nil when one run had no + // turn at this index. + A *Turn `json:"a,omitempty"` + B *Turn `json:"b,omitempty"` +} + +// Result is the outcome of a comparison. +type Result struct { + TurnsA int `json:"turns_a"` + TurnsB int `json:"turns_b"` + // TurnsMatched is how many leading turns behaved identically. + TurnsMatched int `json:"turns_matched"` + // Divergence is nil when both runs behaved identically throughout. + Divergence *Divergence `json:"divergence,omitempty"` +} + +// Identical reports whether the two runs behaved the same the whole way. +func (r Result) Identical() bool { return r.Divergence == nil } + +// TurnsOf extracts the assistant turns from a session, in order, including +// those taken by delegated sub-agents. +// +// Only assistant messages are turns: user messages are inputs and tool messages +// are results, neither of which is a decision the model made. A nil session +// yields no turns. +func TurnsOf(sess *session.Session) []Turn { + var turns []Turn + appendTurns(&turns, sess, 0) + return turns +} + +// maxSubSessionDepth bounds the sub-session walk. Delegation nests only a few +// levels in practice; the bound exists so a cyclic or pathological session +// cannot recurse without end. +const maxSubSessionDepth = 32 + +func appendTurns(turns *[]Turn, sess *session.Session, depth int) { + if sess == nil || depth > maxSubSessionDepth { + return + } + + // MessagesSnapshot copies under the session lock, so a live session being + // written to cannot race this walk. + for _, item := range sess.MessagesSnapshot() { + if item.IsSubSession() { + appendTurns(turns, item.SubSession, depth+1) + continue + } + if !item.IsMessage() { + continue + } + + msg := &item.Message.Message + if msg.Role != chat.MessageRoleAssistant { + continue + } + + turn := Turn{ + Index: len(*turns), + Agent: item.Message.AgentName, + Content: msg.Content, + } + for _, tc := range msg.ToolCalls { + turn.ToolCalls = append(turn.ToolCalls, ToolCall{ + Name: tc.Function.Name, + Arguments: tc.Function.Arguments, + }) + } + *turns = append(*turns, turn) + } +} + +// Compare walks two runs' turns and reports the first behavioural divergence. +func Compare(a, b []Turn) Result { + result := Result{TurnsA: len(a), TurnsB: len(b)} + + shorter := min(len(a), len(b)) + for i := range shorter { + if sameBehaviour(a[i], b[i]) { + result.TurnsMatched++ + continue + } + result.Divergence = &Divergence{ + Kind: KindToolCalls, + TurnIndex: i, + A: &a[i], + B: &b[i], + } + return result + } + + // The common prefix behaved identically; one run may have continued. + switch { + case len(b) > len(a): + result.Divergence = &Divergence{Kind: KindExtraTurn, TurnIndex: shorter, B: &b[shorter]} + case len(a) > len(b): + result.Divergence = &Divergence{Kind: KindMissingTurn, TurnIndex: shorter, A: &a[shorter]} + } + return result +} + +// CompareSessions is TurnsOf on both sides followed by Compare. +func CompareSessions(a, b *session.Session) Result { + return Compare(TurnsOf(a), TurnsOf(b)) +} + +// sameBehaviour reports whether two turns called the same tools with the same +// arguments, in the same order. Order matters: calling a tool before another is +// a different plan, even with the same set of calls. +func sameBehaviour(a, b Turn) bool { + if len(a.ToolCalls) != len(b.ToolCalls) { + return false + } + for i := range a.ToolCalls { + if a.ToolCalls[i].Name != b.ToolCalls[i].Name { + return false + } + if !sameArguments(a.ToolCalls[i].Arguments, b.ToolCalls[i].Arguments) { + return false + } + } + return true +} + +// sameArguments compares two tool-call argument payloads semantically. +// +// Argument JSON is model-generated, so key order and whitespace vary run to run +// while the call means the same thing. A byte comparison would report those as +// divergences — the same nondeterminism noise that "compare behaviour, not +// prose" exists to eliminate — so equal JSON values compare equal. +// +// Falls back to string equality when either side is not valid JSON, which is +// the only honest answer for a payload with no structure to compare. +func sameArguments(a, b string) bool { + if a == b { + return true + } + var av, bv any + if json.Unmarshal([]byte(a), &av) != nil || json.Unmarshal([]byte(b), &bv) != nil { + return false + } + return reflect.DeepEqual(av, bv) +} + +// PrintResult writes a human-readable comparison. +func PrintResult(out io.Writer, r Result, nameA, nameB string) { + fmt.Fprintf(out, "Comparing %s (%d turns) against %s (%d turns)\n", nameA, r.TurnsA, nameB, r.TurnsB) + + if r.Identical() { + fmt.Fprintf(out, "\n✅ Identical behaviour across all %d turns.\n", r.TurnsMatched) + return + } + + d := r.Divergence + fmt.Fprintf(out, "\n❌ First divergence at turn %d (after %d matching turn(s)).\n", d.TurnIndex, r.TurnsMatched) + + switch d.Kind { + case KindExtraTurn: + fmt.Fprintf(out, " %s stopped; %s continued with:\n", nameA, nameB) + printCalls(out, d.B) + case KindMissingTurn: + fmt.Fprintf(out, " %s continued; %s stopped. %s did:\n", nameA, nameB, nameA) + printCalls(out, d.A) + case KindToolCalls: + fmt.Fprintf(out, " %s called:\n", nameA) + printCalls(out, d.A) + fmt.Fprintf(out, " %s called:\n", nameB) + printCalls(out, d.B) + } + + fmt.Fprintln(out, "\nEverything after this point is downstream of the divergence and is not compared.") +} + +func printCalls(out io.Writer, t *Turn) { + if t == nil { + return + } + if len(t.ToolCalls) == 0 { + fmt.Fprintln(out, " (no tool calls — a final answer)") + return + } + for i, tc := range t.ToolCalls { + if i == maxPrintedCalls { + fmt.Fprintf(out, " … and %d more call(s)\n", len(t.ToolCalls)-maxPrintedCalls) + return + } + fmt.Fprintf(out, " %s(%s)\n", sanitizeForTerminal(tc.Name), truncateArgs(tc.Arguments)) + } +} + +const ( + // maxArgRunes bounds one call's rendered arguments. + maxArgRunes = 120 + // maxPrintedCalls bounds how many calls of a single turn are rendered. + // Per-call truncation alone is unbounded in the number of parallel calls, so + // a turn with hundreds of them could still flood the report. + maxPrintedCalls = 10 +) + +// truncateArgs bounds argument text and strips terminal control characters. +func truncateArgs(args string) string { + args = sanitizeForTerminal(args) + runes := []rune(args) + if len(runes) <= maxArgRunes { + return args + } + return string(runes[:maxArgRunes-1]) + "…" +} + +// sanitizeForTerminal removes control characters that let text redraw the +// terminal rather than appear in it. +// +// Tool arguments are model-generated and routinely carry content the agent read +// from a file or the web, so the diagnostic's own output can be shaped by the +// material being diagnosed: a carriage return plus an erase-line sequence +// rewrites the line, letting one tool call be displayed as another. The runtime +// scrubs the same characters before rendering markdown, for the same reason. +func sanitizeForTerminal(s string) string { + return strings.Map(func(r rune) rune { + switch r { + case '\n', '\r', '\b', '\f', '\v': + return ' ' + } + // C0 controls (including ESC) and DEL. + if r < 0x20 || r == 0x7f { + return ' ' + } + return r + }, s) +} diff --git a/pkg/replay/replay_test.go b/pkg/replay/replay_test.go new file mode 100644 index 0000000000..5256e7be53 --- /dev/null +++ b/pkg/replay/replay_test.go @@ -0,0 +1,270 @@ +package replay_test + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/docker/docker-agent/pkg/chat" + "github.com/docker/docker-agent/pkg/replay" + "github.com/docker/docker-agent/pkg/session" + "github.com/docker/docker-agent/pkg/tools" +) + +func call(name, args string) replay.ToolCall { + return replay.ToolCall{Name: name, Arguments: args} +} + +func turn(content string, calls ...replay.ToolCall) replay.Turn { + return replay.Turn{Content: content, ToolCalls: calls} +} + +// indexed renumbers turns the way TurnsOf would, so hand-built fixtures compare +// cleanly against extracted ones. +func indexed(turns ...replay.Turn) []replay.Turn { + for i := range turns { + turns[i].Index = i + } + return turns +} + +func TestCompare_IdenticalBehaviour(t *testing.T) { + t.Parallel() + + a := indexed(turn("looking", call("read_file", `{"path":"a"}`)), turn("done")) + b := indexed(turn("having a look", call("read_file", `{"path":"a"}`)), turn("finished")) + + got := replay.Compare(a, b) + assert.True(t, got.Identical(), "different prose with identical tool calls is identical behaviour") + assert.Equal(t, 2, got.TurnsMatched) + assert.Nil(t, got.Divergence) +} + +// The point of the package: prose differences must never register. +func TestCompare_ProseIsNeverADivergence(t *testing.T) { + t.Parallel() + + a := indexed(turn("I will now read the file.")) + b := indexed(turn("Sure! Let me take a look at that file for you.")) + + assert.True(t, replay.Compare(a, b).Identical()) +} + +func TestCompare_DifferentToolName(t *testing.T) { + t.Parallel() + + a := indexed(turn("", call("read_file", `{"path":"a"}`))) + b := indexed(turn("", call("shell", `{"cmd":"cat a"}`))) + + got := replay.Compare(a, b) + require.NotNil(t, got.Divergence) + assert.Equal(t, replay.KindToolCalls, got.Divergence.Kind) + assert.Equal(t, 0, got.Divergence.TurnIndex) + assert.Zero(t, got.TurnsMatched) +} + +func TestCompare_SameToolDifferentArguments(t *testing.T) { + t.Parallel() + + a := indexed(turn("", call("read_file", `{"path":"a"}`))) + b := indexed(turn("", call("read_file", `{"path":"b"}`))) + + got := replay.Compare(a, b) + require.NotNil(t, got.Divergence) + assert.Equal(t, replay.KindToolCalls, got.Divergence.Kind) +} + +// The same set of calls in a different order is a different plan. +func TestCompare_ToolCallOrderMatters(t *testing.T) { + t.Parallel() + + a := indexed(turn("", call("read_file", "{}"), call("shell", "{}"))) + b := indexed(turn("", call("shell", "{}"), call("read_file", "{}"))) + + assert.False(t, replay.Compare(a, b).Identical()) +} + +func TestCompare_ReportsFirstDivergenceOnly(t *testing.T) { + t.Parallel() + + a := indexed( + turn("", call("read_file", `{"path":"a"}`)), + turn("", call("read_file", `{"path":"b"}`)), + turn("", call("read_file", `{"path":"c"}`)), + ) + b := indexed( + turn("", call("read_file", `{"path":"a"}`)), + turn("", call("shell", `{"cmd":"x"}`)), + turn("", call("shell", `{"cmd":"y"}`)), + ) + + got := replay.Compare(a, b) + require.NotNil(t, got.Divergence) + assert.Equal(t, 1, got.Divergence.TurnIndex, "the first difference, not the last") + assert.Equal(t, 1, got.TurnsMatched) + assert.Equal(t, "read_file", got.Divergence.A.ToolCalls[0].Name) + assert.Equal(t, "shell", got.Divergence.B.ToolCalls[0].Name) +} + +func TestCompare_LengthMismatchAfterMatchingPrefix(t *testing.T) { + t.Parallel() + + short := indexed(turn("", call("read_file", "{}"))) + long := indexed(turn("", call("read_file", "{}")), turn("", call("shell", "{}"))) + + t.Run("second run continued", func(t *testing.T) { + t.Parallel() + got := replay.Compare(short, long) + require.NotNil(t, got.Divergence) + assert.Equal(t, replay.KindExtraTurn, got.Divergence.Kind) + assert.Equal(t, 1, got.Divergence.TurnIndex) + assert.Nil(t, got.Divergence.A) + require.NotNil(t, got.Divergence.B) + }) + + t.Run("second run stopped early", func(t *testing.T) { + t.Parallel() + got := replay.Compare(long, short) + require.NotNil(t, got.Divergence) + assert.Equal(t, replay.KindMissingTurn, got.Divergence.Kind) + require.NotNil(t, got.Divergence.A) + assert.Nil(t, got.Divergence.B) + }) +} + +func TestCompare_EmptyRuns(t *testing.T) { + t.Parallel() + + assert.True(t, replay.Compare(nil, nil).Identical()) + + got := replay.Compare(nil, indexed(turn("", call("shell", "{}")))) + require.NotNil(t, got.Divergence) + assert.Equal(t, replay.KindExtraTurn, got.Divergence.Kind) +} + +func TestTurnsOf(t *testing.T) { + t.Parallel() + + sess := &session.Session{Messages: []session.Item{ + {Message: &session.Message{Message: chat.Message{Role: chat.MessageRoleUser, Content: "do it"}}}, + {Message: &session.Message{AgentName: "root", Message: chat.Message{ + Role: chat.MessageRoleAssistant, + Content: "reading", + ToolCalls: []tools.ToolCall{ + {Function: tools.FunctionCall{Name: "read_file", Arguments: `{"path":"a"}`}}, + }, + }}}, + {Message: &session.Message{Message: chat.Message{Role: chat.MessageRoleTool, Content: "contents"}}}, + {Message: &session.Message{AgentName: "root", Message: chat.Message{ + Role: chat.MessageRoleAssistant, Content: "done", + }}}, + {}, // a non-message item (e.g. a compaction summary) + }} + + got := replay.TurnsOf(sess) + + require.Len(t, got, 2, "only assistant messages are turns") + assert.Equal(t, 0, got[0].Index) + assert.Equal(t, "root", got[0].Agent) + assert.Equal(t, "reading", got[0].Content) + require.Len(t, got[0].ToolCalls, 1) + assert.Equal(t, "read_file", got[0].ToolCalls[0].Name) + assert.Equal(t, 1, got[1].Index) + assert.Empty(t, got[1].ToolCalls) +} + +func TestTurnsOf_NilSession(t *testing.T) { + t.Parallel() + assert.Empty(t, replay.TurnsOf(nil)) +} + +func TestCompareSessions(t *testing.T) { + t.Parallel() + + mk := func(toolName string) *session.Session { + return &session.Session{Messages: []session.Item{ + {Message: &session.Message{Message: chat.Message{ + Role: chat.MessageRoleAssistant, + ToolCalls: []tools.ToolCall{ + {Function: tools.FunctionCall{Name: toolName, Arguments: "{}"}}, + }, + }}}, + }} + } + + assert.True(t, replay.CompareSessions(mk("read_file"), mk("read_file")).Identical()) + assert.False(t, replay.CompareSessions(mk("read_file"), mk("shell")).Identical()) +} + +func TestPrintResult(t *testing.T) { + t.Parallel() + + t.Run("identical", func(t *testing.T) { + t.Parallel() + var buf bytes.Buffer + replay.PrintResult(&buf, replay.Compare( + indexed(turn("", call("read_file", "{}"))), + indexed(turn("", call("read_file", "{}"))), + ), "old", "new") + assert.Contains(t, buf.String(), "Identical behaviour") + }) + + t.Run("divergence names both sides", func(t *testing.T) { + t.Parallel() + var buf bytes.Buffer + replay.PrintResult(&buf, replay.Compare( + indexed(turn("", call("read_file", `{"path":"a"}`))), + indexed(turn("", call("shell", `{"cmd":"x"}`))), + ), "old", "new") + + out := buf.String() + assert.Contains(t, out, "First divergence at turn 0") + assert.Contains(t, out, "read_file") + assert.Contains(t, out, "shell") + assert.Contains(t, out, "old") + assert.Contains(t, out, "new") + assert.Contains(t, out, "downstream of the divergence") + }) + + t.Run("a final answer is labelled", func(t *testing.T) { + t.Parallel() + var buf bytes.Buffer + replay.PrintResult(&buf, replay.Compare( + indexed(turn("all done")), + indexed(turn("", call("shell", "{}"))), + ), "old", "new") + assert.Contains(t, buf.String(), "no tool calls") + }) + + t.Run("huge arguments are truncated", func(t *testing.T) { + t.Parallel() + var buf bytes.Buffer + huge := strings.Repeat("x", 5000) + replay.PrintResult(&buf, replay.Compare( + indexed(turn("", call("write_file", huge))), + indexed(turn("", call("shell", "{}"))), + ), "old", "new") + assert.Less(t, buf.Len(), 1000, "a large payload must not flood the report") + assert.Contains(t, buf.String(), "…") + }) +} + +func TestResult_IsJSONSerializable(t *testing.T) { + t.Parallel() + + r := replay.Compare( + indexed(turn("", call("read_file", "{}"))), + indexed(turn("", call("shell", "{}"))), + ) + data, err := json.Marshal(r) + require.NoError(t, err) + + var round replay.Result + require.NoError(t, json.Unmarshal(data, &round)) + require.NotNil(t, round.Divergence) + assert.Equal(t, replay.KindToolCalls, round.Divergence.Kind) +}