From 3f3f7ab109685183e1562a7a2495cb005f9b390e Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 28 Jul 2026 18:03:56 +0900 Subject: [PATCH 1/5] Bound and redact detailed database diffs (#4859) --- USER_GUIDE.md | 56 +- changelog.d/unreleased/4859.security.md | 24 + src/CodeIndex/Cli/ConsoleUi.cs | 2 +- src/CodeIndex/Cli/DiffCommandOptionsParser.cs | 55 ++ src/CodeIndex/Cli/DiffCommandRunner.cs | 653 ++++++++++++++---- src/CodeIndex/Cli/DiffCursorCodec.cs | 107 +++ src/CodeIndex/Cli/DiffResultWriter.cs | 229 +++++- src/CodeIndex/Cli/JsonOutputContracts.cs | 57 ++ .../DiffCommandHelpersTests.cs | 27 + .../CodeIndex.Tests/DiffCommandRunnerTests.cs | 382 ++++++++-- .../ExportImportCommandRunnerTests.cs | 34 +- 11 files changed, 1386 insertions(+), 240 deletions(-) create mode 100644 changelog.d/unreleased/4859.security.md create mode 100644 src/CodeIndex/Cli/DiffCursorCodec.cs diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 752c8cf70..43c335662 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -507,18 +507,32 @@ use the sibling project directory; other database paths fall back to the process current directory. `--dry-run` and its `--check` alias also compare an existing destination DB with the validated archive without replacing it. JSON `destination_delta.comparison` reports schema and count deltas plus bounded -file, symbol, reference-edge, chunk, and metadata samples. Use -`--limit ` and `--offset ` to page those samples. If the destination -does not exist or cannot be read, `destination_delta` reports that state instead -of claiming a comparison. +file, symbol, reference-edge, chunk, and metadata records. Text fields in those +records are represented by named SHA-256 and UTF-8 byte-length metadata rather +than source content or paths. Use `--limit ` and `--offset ` to +page those records. If the destination does not exist or cannot be read, +`destination_delta` reports that state instead of claiming a comparison. Use `cdidx diff --detailed --json` to verify the restored index. Database identity is based on semantic index content: reference-line links are compared by their indexed path, line, and context rather than SQLite surrogate row IDs, so equivalent databases remain identical after rows are rehydrated. -Detailed JSON also includes `references_only_in_left/right` and -`chunks_only_in_left/right`. `--limit` bounds each difference family and -`--offset` selects the page; `has_more` and `next_offset` indicate continuation. +Detailed JSON returns one deterministic `records` sequence. Each record names +its `area` and `side`, carries a stable `identity_sha256`, and exposes named +`fields` instead of an opaque encoded row. By default, text fields and database +paths are redacted to SHA-256 and UTF-8 byte-length metadata; source text is +returned only when `--include-content` is explicitly combined with +`--detailed --json`. + +Detailed JSON is capped at 1 MiB by default. Every JSON mode accepts +`--max-json-bytes ` (4096 through 16777216) for a caller-controlled +whole-output UTF-8 budget. In detailed mode, CodeIndex stops only at complete +record boundaries, so the result remains valid JSON. `total_count`, `returned_count`, +`omitted_count`, `truncated`, and `truncation_reason` describe the page. +`--limit` bounds the unified record page, `--offset` remains available for +direct paging, and `next_cursor` plus `replay.next_page_arguments` provide the +preferred deterministic continuation contract. Reuse the same database +arguments and emitted replay flags to resume. ## Flag compatibility and migrations @@ -3718,17 +3732,31 @@ SQLite file が CodeIndex DB であることを検証してから destination da それ以外の database path では process current directory に fallback します。 `--dry-run` と alias の `--check` は置換せず、既存 destination DB と検証済み archive を 比較します。JSON の `destination_delta.comparison` には schema / count delta と、 -file、symbol、reference edge、chunk、metadata の bounded sample が含まれます。 -sample の paging には `--limit ` と `--offset ` を使います。destination が -存在しない、または読み取れない場合は、比較済みとせずその状態を `destination_delta` に返します。 +file、symbol、reference edge、chunk、metadata の bounded record が含まれます。 +これらの record の text field は source content や path そのものではなく、名前付きの +SHA-256 と UTF-8 byte length metadata として表現されます。record の paging には +`--limit ` と `--offset ` を使います。destination が存在しない、または +読み取れない場合は、比較済みとせずその状態を `destination_delta` に返します。 復元した index の確認には `cdidx diff --detailed --json` を使います。 database の同一性は semantic index content に基づきます。reference-line link は SQLite の surrogate row ID ではなく indexed path、line、context で比較されるため、row が再構築されても -意味的に同等な database は identical のままです。詳細 JSON には -`references_only_in_left/right` と `chunks_only_in_left/right` も含まれます。 -`--limit` は difference family ごとの件数を制限し、`--offset` は page を選択します。 -続きがある場合は `has_more` と `next_offset` を返します。 +意味的に同等な database は identical のままです。詳細 JSON は deterministic な単一の +`records` sequence を返します。各 record は `area` と `side` を明示し、stable な +`identity_sha256` と、opaque な encoded row ではなく名前付きの `fields` を持ちます。 +既定では text field と database path を SHA-256 と UTF-8 byte length metadata に +redact します。source text を返すには `--detailed --json` とともに +`--include-content` を明示的に指定してください。 + +詳細 JSON は既定で 1 MiB に制限されます。すべての JSON mode で、caller が output +全体の UTF-8 budget を指定するための `--max-json-bytes `（4096 以上 16777216 以下） +を使えます。詳細 mode では CodeIndex が complete record の境界でのみ停止するため、 +結果は常に valid JSON です。 +page の状態は `total_count`、`returned_count`、`omitted_count`、`truncated`、 +`truncation_reason` で確認できます。`--limit` は unified record page を制限し、 +direct paging 用の `--offset` も引き続き使えます。deterministic な続きの取得には +`next_cursor` と `replay.next_page_arguments` を使うことを推奨します。同じ database +arguments と、出力された replay flags を再利用して再開してください。 ## フラグ互換性と移行 diff --git a/changelog.d/unreleased/4859.security.md b/changelog.d/unreleased/4859.security.md new file mode 100644 index 000000000..ec609630c --- /dev/null +++ b/changelog.d/unreleased/4859.security.md @@ -0,0 +1,24 @@ +--- +category: security +issues: + - 4859 +affected: + - src/CodeIndex/Cli/DiffCommandRunner.cs + - src/CodeIndex/Cli/DiffCommandOptionsParser.cs + - src/CodeIndex/Cli/DiffCursorCodec.cs + - src/CodeIndex/Cli/DiffResultWriter.cs + - src/CodeIndex/Cli/JsonOutputContracts.cs + - src/CodeIndex/Cli/ConsoleUi.cs + - tests/CodeIndex.Tests/DiffCommandRunnerTests.cs + - tests/CodeIndex.Tests/DiffCommandHelpersTests.cs + - tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs + - USER_GUIDE.md +--- + +## English + +- **Detailed database diffs are now structured, redacted, and byte-bounded by default (#4859)** — `diff --detailed --json` emits named fields with hashes and byte lengths instead of opaque source-bearing rows, caps the complete UTF-8 JSON output at 1 MiB by default, and returns count, truncation, cursor, and replay metadata. Callers can set `--max-json-bytes`, resume with `--cursor`, or explicitly opt into source and path values with `--include-content`. + +## 日本語 + +- **詳細 database diff を既定で構造化・秘匿化し、byte 上限を適用するようになりました (#4859)** — `diff --detailed --json` は source を含み得る opaque row の代わりに hash と byte length を持つ名前付き field を出力し、UTF-8 JSON 全体を既定で 1 MiB に制限して、件数・truncation・cursor・replay metadata を返します。caller は `--max-json-bytes` で上限を指定し、`--cursor` で再開できます。source と path の値が必要な場合だけ `--include-content` で明示的に opt-in できます。 diff --git a/src/CodeIndex/Cli/ConsoleUi.cs b/src/CodeIndex/Cli/ConsoleUi.cs index 3555ddc6d..5a7e2042e 100644 --- a/src/CodeIndex/Cli/ConsoleUi.cs +++ b/src/CodeIndex/Cli/ConsoleUi.cs @@ -126,7 +126,7 @@ private static readonly (string Command, string Usage)[] CommandUsageLines = ("db-checkpoints", "cdidx db checkpoints --list|--delete |--prune [--keep ] [--dry-run] [--db ] [--json]"), ("db-restore", "cdidx db restore [--dry-run] [--db ] [--json]"), ("db-restore-backups", "cdidx db restore-backups --list|--prune [--keep ] [--dry-run] [--db ] [--json]"), - ("diff", "cdidx diff [--json] [--summary-only] [--detailed] [--limit ] [--offset ]"), + ("diff", $"cdidx diff [--json] [--summary-only] [--detailed] [--include-content] [--max-json-bytes ] [--limit ] [--offset |--cursor ]"), ("report", "cdidx report --output [--overwrite] [--db ] [--json] [--redact-paths] [--log-lines ] [--no-log] [--include-args]"), ("validate", "cdidx validate [--db ] [--json[=array]] [--format ] [--verbose] [--limit |--top ] [--kind ] [--severity ] [--path ]"), ("impact", "cdidx impact |--query |-- [--db ] [--json] [--format ] [--compact] [--fields ] [--cursor ] [--max-json-bytes ] [--verbose] [--limit |--top ] [--lang ] [--path ] [--exclude-path ] [--exclude-tests] [--body] [--snippet-lines ] [--max-line-width ] [--max-hops ] [--exact-name] [--count] [--with-paths]"), diff --git a/src/CodeIndex/Cli/DiffCommandOptionsParser.cs b/src/CodeIndex/Cli/DiffCommandOptionsParser.cs index 0dd07c6f5..3efcd07e8 100644 --- a/src/CodeIndex/Cli/DiffCommandOptionsParser.cs +++ b/src/CodeIndex/Cli/DiffCommandOptionsParser.cs @@ -10,8 +10,12 @@ internal static DiffCommandOptions Parse(string[] args, int maxLimit) var json = false; var detailed = false; var summaryOnly = false; + var includeContent = false; var limit = DefaultLimit; var offset = 0; + var offsetExplicit = false; + int? maxJsonBytes = null; + string? cursor = null; string? parseError = null; for (var i = 0; i < args.Length; i++) @@ -30,6 +34,35 @@ internal static DiffCommandOptions Parse(string[] args, int maxLimit) case "--summary-only": summaryOnly = true; break; + case "--include-content": + includeContent = true; + break; + case "--max-json-bytes" when i + 1 < args.Length: + if (!int.TryParse(args[++i], out var parsedMaxJsonBytes) + || parsedMaxJsonBytes < DiffCommandRunner.MinDiffJsonBytes) + { + parseError = $"--max-json-bytes must be at least {DiffCommandRunner.MinDiffJsonBytes}"; + } + else if (parsedMaxJsonBytes > DiffCommandRunner.MaxDiffJsonBytes) + { + parseError = $"--max-json-bytes must be less than or equal to {DiffCommandRunner.MaxDiffJsonBytes}"; + } + else + { + maxJsonBytes = parsedMaxJsonBytes; + } + break; + case "--max-json-bytes": + parseError = "--max-json-bytes requires a value"; + break; + case "--cursor" when i + 1 < args.Length: + cursor = args[++i]; + if (cursor.Length > DiffCursorCodec.MaxCursorLength) + parseError = $"--cursor must not exceed {DiffCursorCodec.MaxCursorLength} characters"; + break; + case "--cursor": + parseError = "--cursor requires a value"; + break; case "--limit" when i + 1 < args.Length: if (!int.TryParse(args[++i], out limit) || limit < 0) parseError = "--limit requires a non-negative integer"; @@ -43,6 +76,7 @@ internal static DiffCommandOptions Parse(string[] args, int maxLimit) parseError = "--limit requires a value"; break; case "--offset" when i + 1 < args.Length: + offsetExplicit = true; if (!int.TryParse(args[++i], out offset) || offset < 0) parseError = "--offset requires a non-negative integer"; break; @@ -65,6 +99,23 @@ internal static DiffCommandOptions Parse(string[] args, int maxLimit) if (parseError is null && dbs.Count != 2) parseError = "diff requires exactly two database paths"; + if (parseError is null && includeContent && (!detailed || !json || summaryOnly)) + parseError = "--include-content requires --detailed --json and cannot be combined with --summary-only"; + if (parseError is null && maxJsonBytes.HasValue && !(json || summaryOnly)) + parseError = "--max-json-bytes is only supported with JSON diff output"; + if (parseError is null && cursor is not null && (!detailed || !json || summaryOnly)) + parseError = "--cursor requires --detailed --json and cannot be combined with --summary-only"; + if (parseError is null && cursor is not null && offsetExplicit) + parseError = "--cursor cannot be combined with --offset"; + if (parseError is null && cursor is not null) + { + var selectionFingerprint = DiffCursorCodec.CreateSelectionFingerprint( + dbs[0], + dbs[1], + includeContent); + if (!DiffCursorCodec.TryDecode(cursor, selectionFingerprint, out offset, out var cursorError)) + parseError = cursorError; + } if (parseError is null && offset > int.MaxValue - limit) parseError = "--offset is too large for the requested --limit"; @@ -75,8 +126,12 @@ internal static DiffCommandOptions Parse(string[] args, int maxLimit) Json = json, Detailed = detailed, SummaryOnly = summaryOnly, + IncludeContent = includeContent, Limit = limit, Offset = offset, + OffsetExplicit = offsetExplicit, + MaxJsonBytes = maxJsonBytes, + Cursor = cursor, ParseError = parseError, }; } diff --git a/src/CodeIndex/Cli/DiffCommandRunner.cs b/src/CodeIndex/Cli/DiffCommandRunner.cs index 291a88130..ed85f0d3e 100644 --- a/src/CodeIndex/Cli/DiffCommandRunner.cs +++ b/src/CodeIndex/Cli/DiffCommandRunner.cs @@ -10,9 +10,11 @@ namespace CodeIndex.Cli; public static class DiffCommandRunner { internal const int DefaultDiffLimit = DiffCommandOptionsParser.DefaultLimit; - internal const int MaxDiffEncodedFieldSampleLength = 1024; internal const int MaxDiffComparedRowsPerSide = 1_000_000; internal const int MaxDiffComparedRowBytes = 4 * 1024 * 1024; + internal const int MinDiffJsonBytes = 4 * 1024; + internal const int DefaultDiffJsonBytes = 1024 * 1024; + internal const int MaxDiffJsonBytes = 16 * 1024 * 1024; internal static int MaxDiffLimit => QueryCommandRunner.NumericFlagUpperBounds["--limit"]; internal static int? MaxDiffComparedRowsPerSideForTesting { get; set; } internal static int? MaxDiffComparedRowBytesForTesting { get; set; } @@ -39,14 +41,23 @@ public static int Run(string[] args, JsonSerializerOptions jsonOptions, Cancella options.ParseError, CommandExitCodes.UsageError, "Run `cdidx diff --help` to see the supported command shape.", - CommandErrorCodes.UsageError); + CommandErrorCodes.UsageError, + options.MaxJsonBytes); var json = options.Json || options.SummaryOnly; - var leftUriValidationExitCode = ValidateReadableDbFileUri(options.LeftDb!, json, jsonOptions); + var leftUriValidationExitCode = ValidateReadableDbFileUri( + options.LeftDb!, + json, + jsonOptions, + options.MaxJsonBytes); if (leftUriValidationExitCode != null) return leftUriValidationExitCode.Value; - var rightUriValidationExitCode = ValidateReadableDbFileUri(options.RightDb!, json, jsonOptions); + var rightUriValidationExitCode = ValidateReadableDbFileUri( + options.RightDb!, + json, + jsonOptions, + options.MaxJsonBytes); if (rightUriValidationExitCode != null) return rightUriValidationExitCode.Value; @@ -54,7 +65,9 @@ public static int Run(string[] args, JsonSerializerOptions jsonOptions, Cancella { var result = CompareDatabases(options, cancellationToken); - DiffResultWriter.WriteResult(result, options, jsonOptions); + var outputExitCode = DiffResultWriter.WriteResult(result, options, jsonOptions); + if (outputExitCode.HasValue) + return outputExitCode.Value; if (result.Status == "schema_mismatch") return SchemaMismatchExitCode; @@ -68,7 +81,8 @@ public static int Run(string[] args, JsonSerializerOptions jsonOptions, Cancella "diff comparison cancelled before it could complete", CommandExitCodes.CancelledBySignal, "Retry the diff with a smaller database pair or after the cancelling operation completes.", - CommandErrorCodes.Interrupted); + CommandErrorCodes.Interrupted, + options.MaxJsonBytes); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or SqliteException or InvalidOperationException) { @@ -78,11 +92,16 @@ public static int Run(string[] args, JsonSerializerOptions jsonOptions, Cancella $"failed to compare databases: {CommandErrorWriter.FormatSanitizedExceptionMessage(ex)}", UnreadableExitCode, "Pass two readable CodeIndex SQLite database paths.", - CommandErrorCodes.DbError); + CommandErrorCodes.DbError, + options.MaxJsonBytes); } } - private static int? ValidateReadableDbFileUri(string dbPath, bool json, JsonSerializerOptions jsonOptions) + private static int? ValidateReadableDbFileUri( + string dbPath, + bool json, + JsonSerializerOptions jsonOptions, + int? maxJsonBytes) { if (SqliteFileUri.TryValidateBounds(dbPath, out var parseError)) return null; @@ -93,7 +112,8 @@ public static int Run(string[] args, JsonSerializerOptions jsonOptions, Cancella $"invalid database file URI: {SqliteFileUri.FormatParseError(parseError)}", UnreadableExitCode, "Pass two readable CodeIndex SQLite database paths or valid SQLite file URIs.", - CommandErrorCodes.DbError); + CommandErrorCodes.DbError, + maxJsonBytes); } internal static DiffCommandOptions ParseArgs(string[] args) @@ -107,7 +127,8 @@ internal static DiffJsonResult CompareDatabases( bool detailed, CancellationToken cancellationToken, string? leftDisplayPath = null, - string? rightDisplayPath = null) + string? rightDisplayPath = null, + bool includeContent = false) { var options = new DiffCommandOptions { @@ -116,12 +137,17 @@ internal static DiffJsonResult CompareDatabases( Limit = limit, Offset = offset, Detailed = detailed, + IncludeContent = includeContent, }; var result = CompareDatabases(options, cancellationToken); return result with { - LeftDb = leftDisplayPath ?? result.LeftDb, - RightDb = rightDisplayPath ?? result.RightDb, + LeftDb = leftDisplayPath is null + ? result.LeftDb + : FormatSensitiveText(leftDisplayPath, options), + RightDb = rightDisplayPath is null + ? result.RightDb + : FormatSensitiveText(rightDisplayPath, options), }; } @@ -138,6 +164,105 @@ private static DiffJsonResult CompareDatabases(DiffCommandOptions options, Cance private const string FilePathRowsSql = "SELECT path FROM files ORDER BY path"; + private static readonly DiffRowSchema FileRowSchema = new( + "file", + ["path", "language", "size", "lines", "checksum"]); + private static readonly DiffRowSchema ChunkRowSchema = new( + "chunk", + ["path", "chunk_index", "start_line", "end_line", "content"]); + private static readonly DiffRowSchema ReferenceLineRowSchema = new( + "reference_line", + ["path", "line", "context"]); + private static readonly DiffRowSchema FileIssueRowSchema = new( + "file_issue", + ["path", "kind", "line", "message"]); + private static readonly DiffRowSchema SymbolRowSchema = new( + "symbol", + [ + "path", + "kind", + "sub_kind", + "name", + "name_folded", + "line", + "start_line", + "start_column", + "end_line", + "body_start_line", + "body_end_line", + "signature", + "container_kind", + "container_name", + "container_qualified_name", + "family_key", + "visibility", + "return_type", + "is_metadata_target", + "metadata_target_source", + ]); + private static readonly DiffRowSchema LegacyReferenceRowSchema = new( + "reference", + [ + "reference_path", + "symbol_name", + "symbol_name_folded", + "reference_kind", + "line", + "column", + "context", + "has_reference_line", + "reference_line_path", + "reference_line_line", + "reference_line_context", + "container_kind", + "container_name", + "container_name_folded", + ]); + private static readonly DiffRowSchema ReferenceRowSchema = new( + "reference", + [ + "reference_path", + "symbol_name", + "symbol_name_folded", + "reference_kind", + "line", + "column", + "context", + "has_reference_line", + "reference_line_path", + "reference_line_line", + "reference_line_context", + "container_kind", + "container_name", + "container_name_folded", + "source_language", + "source_path", + "source_kind", + "source_container", + "source_name", + "source_line", + "target_qualifier", + "resolution_state", + "resolution_candidate_count", + "is_self_reference", + "is_mutual_recursion", + "target_symbol_key", + "target_language", + "target_path", + "target_kind", + "target_container", + "target_name", + "target_line", + "candidate_scope_rank", + "candidate_language", + "candidate_path", + "candidate_kind", + "candidate_container", + "candidate_name", + "candidate_line", + "candidate_signature", + ]); + private const string FileRowsSql = """ SELECT path, @@ -302,13 +427,6 @@ private static DiffJsonResult BuildDiff(DiffDbHeader left, DiffDbHeader right, D var filesOnlyInLeft = new List(); var filesOnlyInRight = new List(); - var symbolsOnlyInLeft = new List(); - var symbolsOnlyInRight = new List(); - var referencesOnlyInLeft = new List(); - var referencesOnlyInRight = new List(); - var chunksOnlyInLeft = new List(); - var chunksOnlyInRight = new List(); - List? metadataDrift = null; var diagnostics = new List(); var hasMore = false; var identical = @@ -323,8 +441,10 @@ private static DiffJsonResult BuildDiff(DiffDbHeader left, DiffDbHeader right, D var rightSymbolRowsSql = BuildSymbolRowsSql(rightConnection); var leftReferenceRowsSql = BuildReferenceRowsSql(leftConnection); var rightReferenceRowsSql = BuildReferenceRowsSql(rightConnection); + var leftReferenceRowSchema = GetReferenceRowSchema(leftConnection); + var rightReferenceRowSchema = GetReferenceRowSchema(rightConnection); - if (!options.SummaryOnly) + if (!options.SummaryOnly && !options.Detailed) { cancellationToken.ThrowIfCancellationRequested(); var fileDiff = DiffOrderedStrings(leftConnection, rightConnection, FilePathRowsSql, options.Limit, options.Offset, cancellationToken); @@ -335,75 +455,185 @@ private static DiffJsonResult BuildDiff(DiffDbHeader left, DiffDbHeader right, D AddPagingDiagnostic(diagnostics, fileDiff.Omitted, fileDiff.HasMore, "file differences", options); } - if (options.Detailed) + DiffRecordPageCollector? collector = null; + if (options.Detailed && !options.SummaryOnly) { + collector = new DiffRecordPageCollector(options.Offset, options.Limit, options.IncludeContent); + + cancellationToken.ThrowIfCancellationRequested(); + identical &= CollectOrderedRows( + leftConnection, + rightConnection, + FileRowsSql, + FileRowsSql, + FileRowSchema, + FileRowSchema, + collector, + cancellationToken); + + cancellationToken.ThrowIfCancellationRequested(); + identical &= CollectOrderedRows( + leftConnection, + rightConnection, + leftSymbolRowsSql, + rightSymbolRowsSql, + SymbolRowSchema, + SymbolRowSchema, + collector, + cancellationToken); + + cancellationToken.ThrowIfCancellationRequested(); + identical &= CollectOrderedRows( + leftConnection, + rightConnection, + leftReferenceRowsSql, + rightReferenceRowsSql, + leftReferenceRowSchema, + rightReferenceRowSchema, + collector, + cancellationToken); + + cancellationToken.ThrowIfCancellationRequested(); + identical &= CollectOrderedRows( + leftConnection, + rightConnection, + ChunkRowsSql, + ChunkRowsSql, + ChunkRowSchema, + ChunkRowSchema, + collector, + cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); - var symbolDiff = DiffOrderedRows(leftConnection, rightConnection, leftSymbolRowsSql, rightSymbolRowsSql, options.Limit, options.Offset, cancellationToken); - symbolsOnlyInLeft = symbolDiff.OnlyInLeft; - symbolsOnlyInRight = symbolDiff.OnlyInRight; - identical = identical && symbolDiff.Equal; - hasMore |= symbolDiff.HasMore; - AddPagingDiagnostic(diagnostics, symbolDiff.Omitted, symbolDiff.HasMore, "symbol differences", options); + identical &= CollectOrderedRows( + leftConnection, + rightConnection, + ReferenceLineRowsSql, + ReferenceLineRowsSql, + ReferenceLineRowSchema, + ReferenceLineRowSchema, + collector, + cancellationToken); cancellationToken.ThrowIfCancellationRequested(); - var referenceDiff = DiffOrderedRows(leftConnection, rightConnection, leftReferenceRowsSql, rightReferenceRowsSql, options.Limit, options.Offset, cancellationToken); - referencesOnlyInLeft = referenceDiff.OnlyInLeft; - referencesOnlyInRight = referenceDiff.OnlyInRight; - identical = identical && referenceDiff.Equal; - hasMore |= referenceDiff.HasMore; - AddPagingDiagnostic(diagnostics, referenceDiff.Omitted, referenceDiff.HasMore, "reference-edge differences", options); + identical &= CollectOrderedRows( + leftConnection, + rightConnection, + FileIssueRowsSql, + FileIssueRowsSql, + FileIssueRowSchema, + FileIssueRowSchema, + collector, + cancellationToken); cancellationToken.ThrowIfCancellationRequested(); - var chunkDiff = DiffOrderedRows(leftConnection, rightConnection, ChunkRowsSql, ChunkRowsSql, options.Limit, options.Offset, cancellationToken); - chunksOnlyInLeft = chunkDiff.OnlyInLeft; - chunksOnlyInRight = chunkDiff.OnlyInRight; - identical = identical && chunkDiff.Equal; - hasMore |= chunkDiff.HasMore; - AddPagingDiagnostic(diagnostics, chunkDiff.Omitted, chunkDiff.HasMore, "chunk differences", options); + identical &= CollectMetadataRows( + leftConnection, + rightConnection, + MetaRowsSql, + "contract_metadata", + collector, + cancellationToken); cancellationToken.ThrowIfCancellationRequested(); - var metadataDiff = DiffMetadataRows(leftConnection, rightConnection, options.Limit, options.Offset, cancellationToken); - metadataDrift = metadataDiff.Drift; - hasMore |= metadataDiff.HasMore; - AddPagingDiagnostic(diagnostics, metadataDiff.Omitted, metadataDiff.HasMore, "metadata differences", options); + _ = CollectMetadataRows( + leftConnection, + rightConnection, + OperationalMetaRowsSql, + "operational_metadata", + collector, + cancellationToken); } - if (identical) + if (identical && collector is null) { cancellationToken.ThrowIfCancellationRequested(); identical = RowsEqual(leftConnection, rightConnection, FileRowsSql, cancellationToken) && - (options.Detailed || RowsEqual(leftConnection, rightConnection, ChunkRowsSql, cancellationToken)) && + RowsEqual(leftConnection, rightConnection, ChunkRowsSql, cancellationToken) && RowsEqual(leftConnection, rightConnection, ReferenceLineRowsSql, cancellationToken) && RowsEqual(leftConnection, rightConnection, FileIssueRowsSql, cancellationToken) && RowsEqual(leftConnection, rightConnection, MetaRowsSql, cancellationToken) && - (options.Detailed || RowsEqual(leftConnection, rightConnection, leftSymbolRowsSql, rightSymbolRowsSql, cancellationToken)) && - (options.Detailed || RowsEqual(leftConnection, rightConnection, leftReferenceRowsSql, rightReferenceRowsSql, cancellationToken)); + RowsEqual(leftConnection, rightConnection, leftSymbolRowsSql, rightSymbolRowsSql, cancellationToken) && + RowsEqual(leftConnection, rightConnection, leftReferenceRowsSql, rightReferenceRowsSql, cancellationToken); } + List? records = null; + long? totalCount = null; + int? returnedCount = null; + long? omittedCount = null; + string? selectionFingerprint = null; + string? currentCursor = null; + string? nextCursor = null; + DiffReplayJsonResult? replay = null; + string? truncationReason = null; + int? nextOffset = null; var truncated = diagnostics.Count > 0; + if (collector is not null) + { + records = collector.Records; + totalCount = collector.TotalCount; + returnedCount = records.Count; + omittedCount = collector.TotalCount - records.Count; + hasMore = collector.TotalCount > (long)options.Offset + records.Count; + truncated = omittedCount > 0; + truncationReason = truncated ? "limit_or_offset" : null; + selectionFingerprint = DiffCursorCodec.CreateSelectionFingerprint( + options.LeftDb!, + options.RightDb!, + options.IncludeContent); + currentCursor = DiffCursorCodec.Encode(options.Offset, selectionFingerprint); + if (hasMore) + { + nextOffset = checked(options.Offset + records.Count); + nextCursor = DiffCursorCodec.Encode(nextOffset.Value, selectionFingerprint); + } + + replay = BuildReplayMetadata(options, selectionFingerprint, currentCursor, nextCursor); + if (truncated) + { + diagnostics.Add(new DiffDiagnosticJsonResult( + "diff_records_truncated", + hasMore + ? "Detailed diff records were omitted; use replay.next_page_arguments to continue from the next whole record." + : "Detailed diff records before the requested offset were omitted from this page.")); + } + } + return new DiffJsonResult( identical ? "identical" : "different", identical, - left.Path, - right.Path, + FormatSensitiveText(left.Path, options), + FormatSensitiveText(right.Path, options), summary, filesOnlyInLeft, filesOnlyInRight, - options.Detailed ? symbolsOnlyInLeft : null, - options.Detailed ? symbolsOnlyInRight : null, - options.Detailed ? referencesOnlyInLeft : null, - options.Detailed ? referencesOnlyInRight : null, - options.Detailed ? chunksOnlyInLeft : null, - options.Detailed ? chunksOnlyInRight : null, - options.Detailed ? metadataDrift : null, + options.Detailed ? [] : null, + options.Detailed ? [] : null, + options.Detailed ? [] : null, + options.Detailed ? [] : null, + options.Detailed ? [] : null, + options.Detailed ? [] : null, + options.Detailed ? [] : null, options.Limit, options.Offset, options.Detailed, hasMore, - hasMore && options.Limit > 0 ? checked(options.Offset + options.Limit) : null, + nextOffset, truncated, - truncated ? diagnostics : null); + diagnostics.Count > 0 ? diagnostics : null, + Records: records, + TotalCount: totalCount, + ReturnedCount: returnedCount, + OmittedCount: omittedCount, + ContentIncluded: collector is null ? null : options.IncludeContent, + ContentPolicy: collector is null ? null : options.IncludeContent ? "included" : "redacted_hashes", + MaxJsonBytes: options.MaxJsonBytes, + SelectionFingerprint: selectionFingerprint, + CurrentCursor: currentCursor, + NextCursor: nextCursor, + Replay: replay, + TruncationReason: truncationReason); } private static void AddPagingDiagnostic( @@ -439,12 +669,18 @@ private static DiffJsonResult BuildSchemaMismatchDiff(DiffDbHeader left, DiffDbH left.SchemaVersion, right.SchemaVersion, false); + var selectionFingerprint = options.Detailed + ? DiffCursorCodec.CreateSelectionFingerprint(options.LeftDb!, options.RightDb!, options.IncludeContent) + : null; + var currentCursor = selectionFingerprint is null + ? null + : DiffCursorCodec.Encode(options.Offset, selectionFingerprint); return new DiffJsonResult( "schema_mismatch", false, - left.Path, - right.Path, + FormatSensitiveText(left.Path, options), + FormatSensitiveText(right.Path, options), summary, [], [], @@ -457,30 +693,41 @@ private static DiffJsonResult BuildSchemaMismatchDiff(DiffDbHeader left, DiffDbH options.Detailed ? [] : null, options.Limit, options.Offset, - options.Detailed); + options.Detailed, + Records: options.Detailed ? [] : null, + TotalCount: options.Detailed ? 0 : null, + ReturnedCount: options.Detailed ? 0 : null, + OmittedCount: options.Detailed ? 0 : null, + ContentIncluded: options.Detailed ? options.IncludeContent : null, + ContentPolicy: options.Detailed ? options.IncludeContent ? "included" : "redacted_hashes" : null, + MaxJsonBytes: options.MaxJsonBytes, + SelectionFingerprint: selectionFingerprint, + CurrentCursor: currentCursor, + Replay: selectionFingerprint is null || currentCursor is null + ? null + : BuildReplayMetadata(options, selectionFingerprint, currentCursor, null)); } - private static MetadataRowsDiff DiffMetadataRows( + private static bool CollectMetadataRows( SqliteConnection leftConnection, SqliteConnection rightConnection, - int limit, - int offset, + string sql, + string area, + DiffRecordPageCollector collector, CancellationToken cancellationToken) { using var leftCommand = leftConnection.CreateCommand(); - leftCommand.CommandText = OperationalMetaRowsSql; + leftCommand.CommandText = sql; using var rightCommand = rightConnection.CreateCommand(); - rightCommand.CommandText = OperationalMetaRowsSql; + rightCommand.CommandText = sql; using var leftReader = leftCommand.ExecuteReader(); using var rightReader = rightCommand.ExecuteReader(); - var drift = new List(limit); var leftRowsRead = 0; var rightRowsRead = 0; var leftHasValue = TryReadMetadataRow(leftReader, out var leftValue, ref leftRowsRead, "left", cancellationToken); var rightHasValue = TryReadMetadataRow(rightReader, out var rightValue, ref rightRowsRead, "right", cancellationToken); var equal = true; - var differenceCount = 0; while (leftHasValue || rightHasValue) { @@ -494,9 +741,7 @@ private static MetadataRowsDiff DiffMetadataRows( if (!string.Equals(leftValue.Value, rightValue.Value, StringComparison.Ordinal)) { equal = false; - if (differenceCount >= offset && drift.Count < limit) - drift.Add(new DiffMetadataDriftJsonResult(leftValue.Key, leftValue.Value, rightValue.Value)); - differenceCount++; + collector.AddMetadata(area, leftValue.Key, leftValue.Value, rightValue.Value); } leftHasValue = TryReadMetadataRow(leftReader, out leftValue, ref leftRowsRead, "left", cancellationToken); @@ -507,32 +752,27 @@ private static MetadataRowsDiff DiffMetadataRows( equal = false; if (comparison < 0) { - if (differenceCount >= offset && drift.Count < limit) - drift.Add(new DiffMetadataDriftJsonResult(leftValue.Key, leftValue.Value, null)); - differenceCount++; + collector.AddMetadata(area, leftValue.Key, leftValue.Value, null); leftHasValue = TryReadMetadataRow(leftReader, out leftValue, ref leftRowsRead, "left", cancellationToken); } else { - if (differenceCount >= offset && drift.Count < limit) - drift.Add(new DiffMetadataDriftJsonResult(rightValue.Key, null, rightValue.Value)); - differenceCount++; + collector.AddMetadata(area, rightValue.Key, null, rightValue.Value); rightHasValue = TryReadMetadataRow(rightReader, out rightValue, ref rightRowsRead, "right", cancellationToken); } - } - var hasMore = differenceCount > (long)offset + drift.Count; - return new MetadataRowsDiff(equal, drift, hasMore, differenceCount != drift.Count); + return equal; } - private static OrderedRowsDiff DiffOrderedRows( + private static bool CollectOrderedRows( SqliteConnection leftConnection, SqliteConnection rightConnection, string leftSql, string rightSql, - int limit, - int offset, + DiffRowSchema leftSchema, + DiffRowSchema rightSchema, + DiffRecordPageCollector collector, CancellationToken cancellationToken) { using var leftCommand = leftConnection.CreateCommand(); @@ -542,15 +782,11 @@ private static OrderedRowsDiff DiffOrderedRows( using var leftReader = leftCommand.ExecuteReader(); using var rightReader = rightCommand.ExecuteReader(); - var onlyInLeft = new List(limit); - var onlyInRight = new List(limit); var leftRowsRead = 0; var rightRowsRead = 0; var leftHasValue = TryReadRow(leftReader, out var leftValue, ref leftRowsRead, "left", cancellationToken); var rightHasValue = TryReadRow(rightReader, out var rightValue, ref rightRowsRead, "right", cancellationToken); var equal = true; - var leftDifferenceCount = 0; - var rightDifferenceCount = 0; while (leftHasValue || rightHasValue) { @@ -569,27 +805,17 @@ private static OrderedRowsDiff DiffOrderedRows( equal = false; if (comparison < 0) { - if (leftDifferenceCount >= offset && onlyInLeft.Count < limit) - onlyInLeft.Add(EncodeRow(leftValue.SortValues)); - leftDifferenceCount++; + collector.Add(leftSchema, "left", leftValue); leftHasValue = TryReadRow(leftReader, out leftValue, ref leftRowsRead, "left", cancellationToken); } else { - if (rightDifferenceCount >= offset && onlyInRight.Count < limit) - onlyInRight.Add(EncodeRow(rightValue.SortValues)); - rightDifferenceCount++; + collector.Add(rightSchema, "right", rightValue); rightHasValue = TryReadRow(rightReader, out rightValue, ref rightRowsRead, "right", cancellationToken); } } - var hasMore = - leftDifferenceCount > (long)offset + onlyInLeft.Count || - rightDifferenceCount > (long)offset + onlyInRight.Count; - var omitted = - leftDifferenceCount != onlyInLeft.Count || - rightDifferenceCount != onlyInRight.Count; - return new OrderedRowsDiff(equal, onlyInLeft, onlyInRight, hasMore, omitted); + return equal; } private static OrderedRowsDiff DiffOrderedStrings( @@ -924,17 +1150,8 @@ private static bool TableExists(SqliteConnection connection, string table) private static string BuildReferenceRowsSql(SqliteConnection connection) { - if (!TableExists(connection, "symbol_reference_candidates") - || !ColumnExists(connection, "symbol_references", "source_symbol_id") - || !ColumnExists(connection, "symbol_references", "target_symbol_id") - || !ColumnExists(connection, "symbol_references", "target_symbol_key") - || !ColumnExists(connection, "symbol_references", "target_qualifier") - || !ColumnExists(connection, "symbol_references", "resolution_state") - || !ColumnExists(connection, "symbol_references", "resolution_candidate_count") - || !ColumnExists(connection, "symbols", "container_qualified_name")) - { + if (!HasResolvedReferenceRowSchema(connection)) return LegacyReferenceRowsSql; - } return """ SELECT @@ -997,6 +1214,21 @@ ORDER BY """; } + private static DiffRowSchema GetReferenceRowSchema(SqliteConnection connection) + => HasResolvedReferenceRowSchema(connection) + ? ReferenceRowSchema + : LegacyReferenceRowSchema; + + private static bool HasResolvedReferenceRowSchema(SqliteConnection connection) + => TableExists(connection, "symbol_reference_candidates") + && ColumnExists(connection, "symbol_references", "source_symbol_id") + && ColumnExists(connection, "symbol_references", "target_symbol_id") + && ColumnExists(connection, "symbol_references", "target_symbol_key") + && ColumnExists(connection, "symbol_references", "target_qualifier") + && ColumnExists(connection, "symbol_references", "resolution_state") + && ColumnExists(connection, "symbol_references", "resolution_candidate_count") + && ColumnExists(connection, "symbols", "container_qualified_name"); + private static string BuildSymbolRowsSql(SqliteConnection connection) { var metadataTargetExpr = ColumnExists(connection, "symbols", "is_metadata_target") @@ -1054,39 +1286,154 @@ ORDER BY """; } - private static string EncodeRow(object?[] values) + internal static DiffReplayJsonResult BuildReplayMetadata( + DiffCommandOptions options, + string selectionFingerprint, + string currentCursor, + string? nextCursor, + int? effectiveMaxJsonBytes = null) { - var fields = new string[values.Length]; - for (var i = 0; i < values.Length; i++) + List? nextPageArguments = null; + if (nextCursor is not null) { - var rawValue = values[i]; - if (rawValue is null or DBNull) + nextPageArguments = + [ + "--detailed", + "--json", + "--limit", + options.Limit.ToString(System.Globalization.CultureInfo.InvariantCulture), + "--cursor", + nextCursor, + ]; + if (options.IncludeContent) + nextPageArguments.Add("--include-content"); + var replayMaxJsonBytes = effectiveMaxJsonBytes ?? options.MaxJsonBytes; + if (replayMaxJsonBytes.HasValue) { - fields[i] = "-1:"; - continue; + nextPageArguments.Add("--max-json-bytes"); + nextPageArguments.Add(replayMaxJsonBytes.Value.ToString(System.Globalization.CultureInfo.InvariantCulture)); } - - var value = Convert.ToString(rawValue, System.Globalization.CultureInfo.InvariantCulture) ?? string.Empty; - var encodedValue = EncodeFieldValue(value); - fields[i] = encodedValue.Length.ToString(System.Globalization.CultureInfo.InvariantCulture) + ":" + encodedValue; } - return string.Join("|", fields); + return new DiffReplayJsonResult( + DiffCursorCodec.Prefix.TrimEnd(':'), + selectionFingerprint, + currentCursor, + nextCursor, + nextPageArguments); } - private static string EncodeFieldValue(string value) + private static string FormatSensitiveText(string value, DiffCommandOptions options) { - if (value.Length <= MaxDiffEncodedFieldSampleLength) + if (!options.Detailed || options.IncludeContent) return value; - var sample = value[..MaxDiffEncodedFieldSampleLength]; - var hash = HexEncoding.ToLowerHexString(SHA256.HashData(Encoding.UTF8.GetBytes(value))); - return sample - + "...[truncated original_length=" - + value.Length.ToString(System.Globalization.CultureInfo.InvariantCulture) - + " sha256=" - + hash - + "]"; + var bytes = Encoding.UTF8.GetBytes(value); + var hash = HexEncoding.ToLowerHexString(SHA256.HashData(bytes)); + return string.Create( + System.Globalization.CultureInfo.InvariantCulture, + $"sha256:{hash} ({bytes.LongLength} UTF-8 bytes)"); + } + + private static DiffRecordJsonResult CreateDiffRecord( + DiffRowSchema schema, + string side, + DiffRow row, + bool includeContent) + { + if (schema.FieldNames.Length != row.SortValues.Length) + { + throw new InvalidOperationException( + $"diff {schema.Area} row schema expected {schema.FieldNames.Length} fields but read {row.SortValues.Length}."); + } + + using var identityHash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + AppendIdentityPart(identityHash, schema.Area); + var fields = new List(row.SortValues.Length); + for (var i = 0; i < row.SortValues.Length; i++) + { + var name = schema.FieldNames[i]; + var value = row.SortValues[i]; + AppendIdentityPart(identityHash, name); + AppendIdentityValue(identityHash, value); + fields.Add(CreateDiffField(name, value, includeContent)); + } + + return new DiffRecordJsonResult( + schema.Area, + side, + HexEncoding.ToLowerHexString(identityHash.GetHashAndReset()), + fields); + } + + private static DiffFieldJsonResult CreateDiffField(string name, object? rawValue, bool includeContent) + { + if (rawValue is null or DBNull) + return new DiffFieldJsonResult(name, "null", null, null, null, 0, false); + + if (rawValue is byte[] binary) + { + return new DiffFieldJsonResult( + name, + "blob", + includeContent ? Convert.ToBase64String(binary) : null, + includeContent ? "base64" : null, + HexEncoding.ToLowerHexString(SHA256.HashData(binary)), + binary.LongLength, + !includeContent); + } + + var value = Convert.ToString(rawValue, System.Globalization.CultureInfo.InvariantCulture) ?? string.Empty; + var bytes = Encoding.UTF8.GetBytes(value); + var valueType = rawValue is byte or sbyte or short or ushort or int or uint or long or ulong + ? "integer" + : rawValue is float or double or decimal + ? "real" + : "text"; + var redact = valueType == "text" && !includeContent; + return new DiffFieldJsonResult( + name, + valueType, + redact ? null : value, + redact || valueType != "text" ? null : "utf-8", + valueType == "text" + ? HexEncoding.ToLowerHexString(SHA256.HashData(bytes)) + : null, + bytes.LongLength, + redact); + } + + private static void AppendIdentityPart(IncrementalHash hash, string value) + { + var bytes = Encoding.UTF8.GetBytes(value); + Span length = stackalloc byte[sizeof(int)]; + System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(length, bytes.Length); + hash.AppendData(length); + hash.AppendData(bytes); + } + + private static void AppendIdentityValue(IncrementalHash hash, object? rawValue) + { + if (rawValue is null or DBNull) + { + AppendIdentityPart(hash, "null"); + return; + } + + if (rawValue is byte[] binary) + { + AppendIdentityPart(hash, "blob"); + Span length = stackalloc byte[sizeof(int)]; + System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(length, binary.Length); + hash.AppendData(length); + hash.AppendData(binary); + return; + } + + AppendIdentityPart(hash, rawValue.GetType().FullName ?? rawValue.GetType().Name); + AppendIdentityPart( + hash, + Convert.ToString(rawValue, System.Globalization.CultureInfo.InvariantCulture) ?? string.Empty); } private sealed record OrderedRowsDiff( @@ -1096,11 +1443,39 @@ private sealed record OrderedRowsDiff( bool HasMore, bool Omitted); - private sealed record MetadataRowsDiff( - bool Equal, - List Drift, - bool HasMore, - bool Omitted); + private sealed record DiffRowSchema( + string Area, + string[] FieldNames); + + private sealed class DiffRecordPageCollector + { + private readonly int _offset; + private readonly int _limit; + private readonly bool _includeContent; + + internal DiffRecordPageCollector(int offset, int limit, bool includeContent) + { + _offset = offset; + _limit = limit; + _includeContent = includeContent; + } + + internal List Records { get; } = []; + internal long TotalCount { get; private set; } + + internal void Add(DiffRowSchema schema, string side, DiffRow row) + { + if (TotalCount >= _offset && Records.Count < _limit) + Records.Add(CreateDiffRecord(schema, side, row, _includeContent)); + TotalCount++; + } + + internal void AddMetadata(string area, string key, string? leftValue, string? rightValue) + => Add( + new DiffRowSchema(area, ["key", "left_value", "right_value"]), + "changed", + new DiffRow([key, leftValue, rightValue])); + } private sealed record DiffRow( object?[] SortValues) @@ -1130,8 +1505,12 @@ internal sealed class DiffCommandOptions public bool Json { get; init; } public bool Detailed { get; init; } public bool SummaryOnly { get; init; } + public bool IncludeContent { get; init; } public bool ShowHelp { get; init; } public int Limit { get; init; } = 20; public int Offset { get; init; } + public bool OffsetExplicit { get; init; } + public int? MaxJsonBytes { get; init; } + public string? Cursor { get; init; } public string? ParseError { get; init; } } diff --git a/src/CodeIndex/Cli/DiffCursorCodec.cs b/src/CodeIndex/Cli/DiffCursorCodec.cs new file mode 100644 index 000000000..eb1a10f3d --- /dev/null +++ b/src/CodeIndex/Cli/DiffCursorCodec.cs @@ -0,0 +1,107 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; + +namespace CodeIndex.Cli; + +internal static class DiffCursorCodec +{ + internal const string Prefix = "diff:v1:"; + internal const int MaxCursorLength = 512; + + internal static string CreateSelectionFingerprint(string leftDb, string rightDb, bool includeContent) + { + var material = string.Join( + "\n", + "diff-record-contract:v1", + leftDb, + rightDb, + includeContent ? "include-content" : "redacted"); + return HexEncoding.ToLowerHexString(SHA256.HashData(Encoding.UTF8.GetBytes(material))); + } + + internal static string Encode(int offset, string selectionFingerprint) + { + var payload = string.Create( + CultureInfo.InvariantCulture, + $"{offset}\n{selectionFingerprint}"); + return Prefix + ToBase64Url(Encoding.UTF8.GetBytes(payload)); + } + + internal static bool TryDecode( + string cursor, + string expectedSelectionFingerprint, + out int offset, + out string error) + { + offset = 0; + if (cursor.Length > MaxCursorLength + || !cursor.StartsWith(Prefix, StringComparison.Ordinal) + || !TryFromBase64Url(cursor[Prefix.Length..], out var payloadBytes)) + { + error = $"--cursor must be an opaque {Prefix} cursor returned by a prior detailed diff response"; + return false; + } + + var payload = Encoding.UTF8.GetString(payloadBytes); + var separator = payload.IndexOf('\n'); + if (separator <= 0 + || separator == payload.Length - 1 + || !int.TryParse(payload.AsSpan(0, separator), NumberStyles.None, CultureInfo.InvariantCulture, out offset) + || offset < 0) + { + offset = 0; + error = $"--cursor must be an opaque {Prefix} cursor returned by a prior detailed diff response"; + return false; + } + + var actualFingerprint = payload[(separator + 1)..]; + if (!FixedTimeEquals(actualFingerprint, expectedSelectionFingerprint)) + { + offset = 0; + error = "--cursor does not match the selected database pair or content policy; restart without --cursor"; + return false; + } + + error = string.Empty; + return true; + } + + private static bool FixedTimeEquals(string left, string right) + { + var leftBytes = Encoding.UTF8.GetBytes(left); + var rightBytes = Encoding.UTF8.GetBytes(right); + return leftBytes.Length == rightBytes.Length + && CryptographicOperations.FixedTimeEquals(leftBytes, rightBytes); + } + + private static string ToBase64Url(byte[] bytes) + => Convert.ToBase64String(bytes) + .TrimEnd('=') + .Replace('+', '-') + .Replace('/', '_'); + + private static bool TryFromBase64Url(string value, out byte[] bytes) + { + bytes = []; + if (value.Length == 0) + return false; + + var normalized = value.Replace('-', '+').Replace('_', '/'); + var padding = normalized.Length % 4; + if (padding == 1) + return false; + if (padding > 0) + normalized = normalized.PadRight(normalized.Length + (4 - padding), '='); + + try + { + bytes = Convert.FromBase64String(normalized); + return true; + } + catch (FormatException) + { + return false; + } + } +} diff --git a/src/CodeIndex/Cli/DiffResultWriter.cs b/src/CodeIndex/Cli/DiffResultWriter.cs index f4199cc67..df1be0343 100644 --- a/src/CodeIndex/Cli/DiffResultWriter.cs +++ b/src/CodeIndex/Cli/DiffResultWriter.cs @@ -5,22 +5,45 @@ namespace CodeIndex.Cli; internal static class DiffResultWriter { - internal static void WriteResult(DiffJsonResult result, DiffCommandOptions options, JsonSerializerOptions jsonOptions) + internal static int? WriteResult(DiffJsonResult result, DiffCommandOptions options, JsonSerializerOptions jsonOptions) { if (options.SummaryOnly) - WriteSummaryJson(result, jsonOptions); + return WriteSummaryJson(result, options, jsonOptions); else if (options.Json) - WriteJson(result, jsonOptions); + return WriteJson(result, options, jsonOptions); else WriteText(result, options); + return null; } - internal static int WriteCommandError(bool json, JsonSerializerOptions jsonOptions, string message, int exitCode, string? hint = null, string? errorCode = null) + internal static int WriteCommandError( + bool json, + JsonSerializerOptions jsonOptions, + string message, + int exitCode, + string? hint = null, + string? errorCode = null, + int? maxJsonBytes = null) { if (json) - Console.WriteLine(JsonSerializer.Serialize( + { + var serialized = JsonSerializer.Serialize( new CommandErrorJsonResult("error", message, hint, errorCode), - CliJsonSerializerContextFactory.Create(jsonOptions).CommandErrorJsonResult)); + CliJsonSerializerContextFactory.Create(jsonOptions).CommandErrorJsonResult); + if (maxJsonBytes.HasValue && !FitsJsonLine(serialized, maxJsonBytes.Value)) + { + serialized = JsonSerializer.Serialize( + new CommandErrorJsonResult( + "error", + "diff error details were omitted to honor --max-json-bytes", + "Increase --max-json-bytes and retry to inspect the full error.", + errorCode), + CliJsonSerializerContextFactory.Create(jsonOptions).CommandErrorJsonResult); + } + + if (!maxJsonBytes.HasValue || FitsJsonLine(serialized, maxJsonBytes.Value)) + Console.WriteLine(serialized); + } else { CommandErrorWriter.WriteStderr($"Error [{errorCode ?? CommandErrorCodes.UsageError}]: {message}"); @@ -33,18 +56,94 @@ internal static int WriteCommandError(bool json, JsonSerializerOptions jsonOptio internal static string FormatDelta(long delta) => delta >= 0 ? $"+{delta}" : delta.ToString(CultureInfo.InvariantCulture); - private static void WriteJson(DiffJsonResult result, JsonSerializerOptions jsonOptions) + private static int? WriteJson( + DiffJsonResult result, + DiffCommandOptions options, + JsonSerializerOptions jsonOptions) { - Console.WriteLine(JsonSerializer.Serialize( - result, - CliJsonSerializerContextFactory.Create(jsonOptions).DiffJsonResult)); + if (!options.Detailed) + { + var serialized = JsonSerializer.Serialize( + result, + CliJsonSerializerContextFactory.Create(jsonOptions).DiffJsonResult); + if (options.MaxJsonBytes.HasValue && !FitsJsonLine(serialized, options.MaxJsonBytes.Value)) + { + return WriteCommandError( + json: true, + jsonOptions, + "--max-json-bytes is too small for the requested diff samples", + CommandExitCodes.UsageError, + "Increase --max-json-bytes, lower --limit, or use --summary-only.", + CommandErrorCodes.UsageError, + options.MaxJsonBytes); + } + + Console.WriteLine(serialized); + return null; + } + + var budget = options.MaxJsonBytes ?? DiffCommandRunner.DefaultDiffJsonBytes; + var context = CliJsonSerializerContextFactory.Create(jsonOptions); + var sourceRecords = result.Records ?? []; + var low = 0; + var high = sourceRecords.Count; + DiffJsonResult? bestResult = null; + string? bestJson = null; + while (low <= high) + { + var candidateCount = low + ((high - low) / 2); + var candidate = BuildBoundedCandidate(result, options, sourceRecords, candidateCount, budget, context); + var candidateJson = JsonSerializer.Serialize(candidate, context.DiffJsonResult); + if (FitsJsonLine(candidateJson, budget)) + { + bestResult = candidate; + bestJson = candidateJson; + low = candidateCount + 1; + } + else + { + high = candidateCount - 1; + } + } + + if (bestResult is null || bestJson is null) + { + return WriteCommandError( + json: true, + jsonOptions, + $"--max-json-bytes is too small for detailed diff metadata; use at least {DiffCommandRunner.MinDiffJsonBytes}", + CommandExitCodes.UsageError, + "Increase --max-json-bytes and rerun the same database comparison.", + CommandErrorCodes.UsageError, + budget); + } + + Console.WriteLine(bestJson); + return null; } - private static void WriteSummaryJson(DiffJsonResult result, JsonSerializerOptions jsonOptions) + private static int? WriteSummaryJson( + DiffJsonResult result, + DiffCommandOptions options, + JsonSerializerOptions jsonOptions) { - Console.WriteLine(JsonSerializer.Serialize( + var serialized = JsonSerializer.Serialize( new DiffSummaryOnlyJsonResult(result.Status, result.Identical, result.LeftDb, result.RightDb, result.Summary), - CliJsonSerializerContextFactory.Create(jsonOptions).DiffSummaryOnlyJsonResult)); + CliJsonSerializerContextFactory.Create(jsonOptions).DiffSummaryOnlyJsonResult); + if (options.MaxJsonBytes.HasValue && !FitsJsonLine(serialized, options.MaxJsonBytes.Value)) + { + return WriteCommandError( + json: true, + jsonOptions, + "--max-json-bytes is too small for the diff summary", + CommandExitCodes.UsageError, + "Increase --max-json-bytes and rerun the same database comparison.", + CommandErrorCodes.UsageError, + options.MaxJsonBytes); + } + + Console.WriteLine(serialized); + return null; } private static void WriteText(DiffJsonResult result, DiffCommandOptions options) @@ -63,15 +162,10 @@ private static void WriteText(DiffJsonResult result, DiffCommandOptions options) WriteList("files only in left", result.FilesOnlyInLeft); WriteList("files only in right", result.FilesOnlyInRight); if (options.Detailed) - { - WriteList("symbols only in left", result.SymbolsOnlyInLeft ?? []); - WriteList("symbols only in right", result.SymbolsOnlyInRight ?? []); - WriteList("reference edges only in left", result.ReferencesOnlyInLeft ?? []); - WriteList("reference edges only in right", result.ReferencesOnlyInRight ?? []); - WriteList("chunks only in left", result.ChunksOnlyInLeft ?? []); - WriteList("chunks only in right", result.ChunksOnlyInRight ?? []); - } - if (result.HasMore && result.NextOffset is int nextOffset) + WriteRecords(result.Records ?? []); + if (result.HasMore && result.NextCursor is not null) + Console.WriteLine($" more : rerun with --cursor {result.NextCursor}"); + else if (result.HasMore && result.NextOffset is int nextOffset) Console.WriteLine($" more : rerun with --offset {nextOffset}"); } @@ -83,4 +177,95 @@ private static void WriteList(string label, List values) foreach (var value in values) Console.WriteLine($" - {value}"); } + + private static void WriteRecords(List records) + { + if (records.Count == 0) + return; + + Console.WriteLine(" detailed records:"); + foreach (var record in records) + { + Console.WriteLine($" - {record.Area} {record.Side} identity_sha256={record.IdentitySha256}"); + foreach (var field in record.Fields) + { + var value = field.Redacted + ? $"[redacted byte_length={field.ByteLength} sha256={field.Sha256}]" + : field.Value ?? "null"; + Console.WriteLine($" {field.Name}: {value}"); + } + } + } + + private static DiffJsonResult BuildBoundedCandidate( + DiffJsonResult source, + DiffCommandOptions options, + List sourceRecords, + int returnedCount, + int budget, + CliJsonSerializerContext context) + { + var records = sourceRecords.Take(returnedCount).ToList(); + var totalCount = source.TotalCount ?? sourceRecords.Count; + var omittedCount = totalCount - returnedCount; + var hasMore = totalCount > (long)source.Offset + returnedCount; + var nextOffset = hasMore ? checked(source.Offset + returnedCount) : default(int?); + var selectionFingerprint = source.SelectionFingerprint + ?? throw new InvalidOperationException("detailed diff selection fingerprint is missing"); + var currentCursor = source.CurrentCursor + ?? DiffCursorCodec.Encode(source.Offset, selectionFingerprint); + var nextCursor = nextOffset.HasValue + ? DiffCursorCodec.Encode(nextOffset.Value, selectionFingerprint) + : null; + var byteTruncated = returnedCount < sourceRecords.Count; + var diagnostics = (source.Diagnostics ?? []) + .Where(item => item.Code is not "diff_records_truncated" and not "diff_json_bytes_truncated") + .ToList(); + if (omittedCount > 0) + { + diagnostics.Add(new DiffDiagnosticJsonResult( + byteTruncated ? "diff_json_bytes_truncated" : "diff_records_truncated", + byteTruncated + ? "Detailed diff output stopped at a whole-record boundary to honor --max-json-bytes; use replay.next_page_arguments to continue." + : hasMore + ? "Detailed diff records were omitted; use replay.next_page_arguments to continue from the next whole record." + : "Detailed diff records before the requested offset were omitted from this page.")); + } + + int? firstOmittedRecordBytes = null; + if (byteTruncated) + { + firstOmittedRecordBytes = JsonSerializer.SerializeToUtf8Bytes( + sourceRecords[returnedCount], + context.DiffRecordJsonResult).Length; + } + + return source with + { + Records = records, + ReturnedCount = returnedCount, + OmittedCount = omittedCount, + HasMore = hasMore, + NextOffset = nextOffset, + NextCursor = nextCursor, + Replay = DiffCommandRunner.BuildReplayMetadata( + options, + selectionFingerprint, + currentCursor, + nextCursor, + budget), + Truncated = omittedCount > 0, + TruncationReason = omittedCount > 0 + ? byteTruncated ? "max_json_bytes" : source.TruncationReason ?? "limit_or_offset" + : null, + Diagnostics = diagnostics.Count > 0 ? diagnostics : null, + MaxJsonBytes = budget, + FirstOmittedRecordBytes = firstOmittedRecordBytes, + }; + } + + private static bool FitsJsonLine(string json, int maxJsonBytes) + => System.Text.Encoding.UTF8.GetByteCount(json) + + System.Text.Encoding.UTF8.GetByteCount(Console.Out.NewLine) + <= maxJsonBytes; } diff --git a/src/CodeIndex/Cli/JsonOutputContracts.cs b/src/CodeIndex/Cli/JsonOutputContracts.cs index 577ab877a..9265fc656 100644 --- a/src/CodeIndex/Cli/JsonOutputContracts.cs +++ b/src/CodeIndex/Cli/JsonOutputContracts.cs @@ -444,12 +444,66 @@ internal sealed record DiffJsonResult( [property: JsonPropertyName("next_offset")] int? NextOffset = null, [property: JsonPropertyName("truncated")] bool Truncated = false, [property: JsonPropertyName("diagnostics")] List? Diagnostics = null, + [property: JsonPropertyName("records")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] List? Records = null, + [property: JsonPropertyName("total_count")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] long? TotalCount = null, + [property: JsonPropertyName("returned_count")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] int? ReturnedCount = null, + [property: JsonPropertyName("omitted_count")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] long? OmittedCount = null, + [property: JsonPropertyName("content_included")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] bool? ContentIncluded = null, + [property: JsonPropertyName("content_policy")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? ContentPolicy = null, + [property: JsonPropertyName("max_json_bytes")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] int? MaxJsonBytes = null, + [property: JsonPropertyName("selection_fingerprint")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? SelectionFingerprint = null, + [property: JsonPropertyName("current_cursor")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? CurrentCursor = null, + [property: JsonPropertyName("next_cursor")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? NextCursor = null, + [property: JsonPropertyName("replay")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] DiffReplayJsonResult? Replay = null, + [property: JsonPropertyName("truncation_reason")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? TruncationReason = null, + [property: JsonPropertyName("first_omitted_record_bytes")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] int? FirstOmittedRecordBytes = null, [property: JsonPropertyName("api_version")] string ApiVersion = JsonOutputContract.ApiVersion) : IVersionedJsonResult; internal sealed record DiffDiagnosticJsonResult( [property: JsonPropertyName("code")] string Code, [property: JsonPropertyName("message")] string Message); +internal sealed record DiffRecordJsonResult( + [property: JsonPropertyName("area")] string Area, + [property: JsonPropertyName("side")] string Side, + [property: JsonPropertyName("identity_sha256")] string IdentitySha256, + [property: JsonPropertyName("fields")] List Fields); + +internal sealed record DiffFieldJsonResult( + [property: JsonPropertyName("name")] string Name, + [property: JsonPropertyName("value_type")] string ValueType, + [property: JsonPropertyName("value")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? Value, + [property: JsonPropertyName("encoding")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? Encoding, + [property: JsonPropertyName("sha256")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? Sha256, + [property: JsonPropertyName("byte_length")] long ByteLength, + [property: JsonPropertyName("redacted")] bool Redacted); + +internal sealed record DiffReplayJsonResult( + [property: JsonPropertyName("cursor_version")] string CursorVersion, + [property: JsonPropertyName("selection_fingerprint")] string SelectionFingerprint, + [property: JsonPropertyName("current_cursor")] string CurrentCursor, + [property: JsonPropertyName("next_cursor")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? NextCursor, + [property: JsonPropertyName("next_page_arguments")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] List? NextPageArguments, + [property: JsonPropertyName("database_arguments_required")] bool DatabaseArgumentsRequired = true); + internal sealed record DiffMetadataDriftJsonResult( [property: JsonPropertyName("key")] string Key, [property: JsonPropertyName("left_value")] string? LeftValue, @@ -1018,8 +1072,11 @@ internal sealed record ValidateConfigJsonResult( [JsonSerializable(typeof(DefinitionResult))] [JsonSerializable(typeof(Dictionary))] [JsonSerializable(typeof(Dictionary))] +[JsonSerializable(typeof(DiffFieldJsonResult))] [JsonSerializable(typeof(DiffJsonResult))] [JsonSerializable(typeof(DiffMetadataDriftJsonResult))] +[JsonSerializable(typeof(DiffRecordJsonResult))] +[JsonSerializable(typeof(DiffReplayJsonResult))] [JsonSerializable(typeof(DiffSummaryOnlyJsonResult))] [JsonSerializable(typeof(DiffSummaryJsonResult))] [JsonSerializable(typeof(ExactZeroHintResult))] diff --git a/tests/CodeIndex.Tests/DiffCommandHelpersTests.cs b/tests/CodeIndex.Tests/DiffCommandHelpersTests.cs index 6c435a0c2..0706807cd 100644 --- a/tests/CodeIndex.Tests/DiffCommandHelpersTests.cs +++ b/tests/CodeIndex.Tests/DiffCommandHelpersTests.cs @@ -1,3 +1,4 @@ +using System.Text; using System.Text.Json; using CodeIndex.Cli; @@ -43,6 +44,32 @@ public void WriteResult_SummaryOnlyOmitsDiffSamples_Issue4181() Assert.False(root.TryGetProperty("files_only_in_right", out _)); } + [Fact] + public void WriteCommandError_BoundsOversizedJsonDetails_Issue4859() + { + var jsonOptions = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower }; + + var output = CaptureStdout(() => DiffResultWriter.WriteCommandError( + json: true, + jsonOptions, + new string('x', DiffCommandRunner.MinDiffJsonBytes * 2), + CommandExitCodes.UsageError, + errorCode: CommandErrorCodes.UsageError, + maxJsonBytes: DiffCommandRunner.MinDiffJsonBytes)); + + Assert.InRange( + Encoding.UTF8.GetByteCount(output), + 1, + DiffCommandRunner.MinDiffJsonBytes); + using var document = JsonDocument.Parse(output); + Assert.Equal( + "diff error details were omitted to honor --max-json-bytes", + document.RootElement.GetProperty("message").GetString()); + Assert.Equal( + CommandErrorCodes.UsageError, + document.RootElement.GetProperty("error_code").GetString()); + } + [Theory] [InlineData(0, "+0")] [InlineData(3, "+3")] diff --git a/tests/CodeIndex.Tests/DiffCommandRunnerTests.cs b/tests/CodeIndex.Tests/DiffCommandRunnerTests.cs index 5ec0501de..af563ec72 100644 --- a/tests/CodeIndex.Tests/DiffCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/DiffCommandRunnerTests.cs @@ -11,6 +11,8 @@ namespace CodeIndex.Tests; [Collection("Console sensitive")] public class DiffCommandRunnerTests { + private const int LargeDiffFieldLength = 4096; + private readonly JsonSerializerOptions _jsonOptions = new() { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, @@ -161,6 +163,212 @@ public void ParseArgs_OffsetAcceptsNonNegativePagingValue_Issue4714() Assert.Null(options.ParseError); } + [Fact] + public void ParseArgs_JsonBudgetAndContentFlagsEnforceDetailedContract_Issue4859() + { + var minimum = DiffCommandRunner.ParseArgs( + ["left.db", "right.db", "--json", "--detailed", "--max-json-bytes", $"{DiffCommandRunner.MinDiffJsonBytes}"]); + var belowMinimum = DiffCommandRunner.ParseArgs( + ["left.db", "right.db", "--json", "--detailed", "--max-json-bytes", $"{DiffCommandRunner.MinDiffJsonBytes - 1}"]); + var contentWithoutDetailedJson = DiffCommandRunner.ParseArgs( + ["left.db", "right.db", "--include-content"]); + + Assert.Equal(DiffCommandRunner.MinDiffJsonBytes, minimum.MaxJsonBytes); + Assert.Null(minimum.ParseError); + Assert.Equal( + $"--max-json-bytes must be at least {DiffCommandRunner.MinDiffJsonBytes}", + belowMinimum.ParseError); + Assert.Equal( + "--include-content requires --detailed --json and cannot be combined with --summary-only", + contentWithoutDetailedJson.ParseError); + } + + [Fact] + public void Run_DetailedJsonRedactsSensitiveTextUntilContentIsExplicitlyIncluded_Issue4859() + { + var leftRoot = TestProjectHelper.CreateTempProject("cdidx_diff_private_left"); + var rightRoot = TestProjectHelper.CreateTempProject("cdidx_diff_private_right"); + try + { + const string privatePath = "src/customer-acme-api-key.cs"; + const string secret = "sk-test-super-secret-value"; + var leftDb = TestProjectHelper.CreateProjectDb(leftRoot); + var rightDb = TestProjectHelper.CreateProjectDb(rightRoot); + TestProjectHelper.InsertIndexedFile( + leftDb, + privatePath, + "csharp", + $"public static class Secret {{ public const string Value = \"{secret}\"; }}"); + + var (redactedExitCode, redactedOutput) = RunWithCapturedOut( + [leftDb, rightDb, "--json", "--detailed", "--limit", "100"]); + + Assert.Equal(1, redactedExitCode); + using var redactedDocument = JsonDocument.Parse(redactedOutput); + var redactedRoot = redactedDocument.RootElement; + Assert.False(redactedRoot.GetProperty("content_included").GetBoolean()); + Assert.Equal("redacted_hashes", redactedRoot.GetProperty("content_policy").GetString()); + Assert.DoesNotContain(privatePath, redactedOutput, StringComparison.Ordinal); + Assert.DoesNotContain(secret, redactedOutput, StringComparison.Ordinal); + Assert.DoesNotContain(leftDb, redactedOutput, StringComparison.Ordinal); + Assert.DoesNotContain(rightDb, redactedOutput, StringComparison.Ordinal); + var fileRecord = Assert.Single(GetRecords(redactedRoot, "file", "left")); + var pathField = GetField(fileRecord, "path"); + Assert.True(pathField.GetProperty("redacted").GetBoolean()); + Assert.False(pathField.TryGetProperty("value", out _)); + Assert.Equal(64, pathField.GetProperty("sha256").GetString()?.Length); + Assert.True(pathField.GetProperty("byte_length").GetInt64() > 0); + + const int byteBudget = 65_536; + var (includedExitCode, includedOutput) = RunWithCapturedOut( + [ + leftDb, + rightDb, + "--json", + "--detailed", + "--include-content", + "--limit", + "100", + "--max-json-bytes", + $"{byteBudget}", + ]); + + Assert.Equal(1, includedExitCode); + Assert.InRange(Encoding.UTF8.GetByteCount(includedOutput), 1, byteBudget); + using var includedDocument = JsonDocument.Parse(includedOutput); + var includedRoot = includedDocument.RootElement; + Assert.True(includedRoot.GetProperty("content_included").GetBoolean()); + Assert.Equal("included", includedRoot.GetProperty("content_policy").GetString()); + Assert.Contains(privatePath, includedOutput, StringComparison.Ordinal); + Assert.Contains(secret, includedOutput, StringComparison.Ordinal); + var includedPath = GetField( + Assert.Single(GetRecords(includedRoot, "file", "left")), + "path"); + Assert.False(includedPath.GetProperty("redacted").GetBoolean()); + Assert.Equal(privatePath, includedPath.GetProperty("value").GetString()); + } + finally + { + TestProjectHelper.DeleteDirectory(leftRoot); + TestProjectHelper.DeleteDirectory(rightRoot); + } + } + + [Fact] + public void Run_DetailedJsonStopsAtRecordBoundaryAndCursorResumes_Issue4859() + { + var leftRoot = TestProjectHelper.CreateTempProject("cdidx_diff_bounded_left"); + var rightRoot = TestProjectHelper.CreateTempProject("cdidx_diff_bounded_right"); + try + { + var leftDb = TestProjectHelper.CreateProjectDb(leftRoot); + var rightDb = TestProjectHelper.CreateProjectDb(rightRoot); + for (var i = 0; i < 30; i++) + { + TestProjectHelper.InsertIndexedFile( + leftDb, + $"src/PrivateFile{i:00}.cs", + "csharp", + $"public static class PrivateFile{i:00} {{ public static string Value => \"{new string('x', 256)}\"; }}"); + } + + const int byteBudget = 8_192; + var (firstExitCode, firstOutput) = RunWithCapturedOut( + [ + leftDb, + rightDb, + "--json", + "--detailed", + "--limit", + "100", + "--max-json-bytes", + $"{byteBudget}", + ]); + + Assert.Equal(1, firstExitCode); + Assert.InRange(Encoding.UTF8.GetByteCount(firstOutput), 1, byteBudget); + using var firstDocument = JsonDocument.Parse(firstOutput); + var first = firstDocument.RootElement; + Assert.Equal("max_json_bytes", first.GetProperty("truncation_reason").GetString()); + Assert.True(first.GetProperty("truncated").GetBoolean()); + Assert.True(first.GetProperty("has_more").GetBoolean()); + Assert.True(first.GetProperty("returned_count").GetInt32() > 0); + Assert.True(first.GetProperty("first_omitted_record_bytes").GetInt32() > 0); + Assert.Equal( + first.GetProperty("total_count").GetInt64(), + first.GetProperty("returned_count").GetInt32() + first.GetProperty("omitted_count").GetInt64()); + var replay = first.GetProperty("replay"); + Assert.True(replay.GetProperty("database_arguments_required").GetBoolean()); + var nextCursor = replay.GetProperty("next_cursor").GetString(); + Assert.False(string.IsNullOrWhiteSpace(nextCursor)); + var replayArguments = replay.GetProperty("next_page_arguments") + .EnumerateArray() + .Select(item => item.GetString()!) + .ToArray(); + + var (secondExitCode, secondOutput) = RunWithCapturedOut([leftDb, rightDb, .. replayArguments]); + + Assert.Equal(1, secondExitCode); + Assert.InRange(Encoding.UTF8.GetByteCount(secondOutput), 1, byteBudget); + using var secondDocument = JsonDocument.Parse(secondOutput); + var second = secondDocument.RootElement; + Assert.Equal(nextCursor, second.GetProperty("current_cursor").GetString()); + var firstIdentities = first.GetProperty("records") + .EnumerateArray() + .Select(record => record.GetProperty("identity_sha256").GetString()) + .ToHashSet(StringComparer.Ordinal); + Assert.DoesNotContain( + second.GetProperty("records").EnumerateArray(), + record => firstIdentities.Contains(record.GetProperty("identity_sha256").GetString())); + + var (mismatchExitCode, mismatchOutput) = RunWithCapturedOut( + [leftDb, rightDb, "--json", "--detailed", "--include-content", "--cursor", nextCursor!]); + Assert.Equal(CommandExitCodes.UsageError, mismatchExitCode); + using var mismatchDocument = JsonDocument.Parse(mismatchOutput); + Assert.Equal("error", mismatchDocument.RootElement.GetProperty("status").GetString()); + } + finally + { + TestProjectHelper.DeleteDirectory(leftRoot); + TestProjectHelper.DeleteDirectory(rightRoot); + } + } + + [Fact] + public void Run_JsonModesHonorMinimumExplicitBudget_Issue4859() + { + var root = TestProjectHelper.CreateTempProject("cdidx_diff_minimum_budget"); + try + { + var db = TestProjectHelper.CreateProjectDb(root); + const int byteBudget = DiffCommandRunner.MinDiffJsonBytes; + + var (detailedExitCode, detailedOutput) = RunWithCapturedOut( + [db, db, "--json", "--detailed", "--max-json-bytes", $"{byteBudget}"]); + var (sampleExitCode, sampleOutput) = RunWithCapturedOut( + [db, db, "--json", "--max-json-bytes", $"{byteBudget}"]); + var (summaryExitCode, summaryOutput) = RunWithCapturedOut( + [db, db, "--summary-only", "--max-json-bytes", $"{byteBudget}"]); + + Assert.Equal(0, detailedExitCode); + Assert.Equal(0, sampleExitCode); + Assert.Equal(0, summaryExitCode); + Assert.InRange(Encoding.UTF8.GetByteCount(detailedOutput), 1, byteBudget); + Assert.InRange(Encoding.UTF8.GetByteCount(sampleOutput), 1, byteBudget); + Assert.InRange(Encoding.UTF8.GetByteCount(summaryOutput), 1, byteBudget); + using var detailedDocument = JsonDocument.Parse(detailedOutput); + using var sampleDocument = JsonDocument.Parse(sampleOutput); + using var summaryDocument = JsonDocument.Parse(summaryOutput); + Assert.Equal("identical", detailedDocument.RootElement.GetProperty("status").GetString()); + Assert.Equal("identical", sampleDocument.RootElement.GetProperty("status").GetString()); + Assert.Equal("identical", summaryDocument.RootElement.GetProperty("status").GetString()); + } + finally + { + TestProjectHelper.DeleteDirectory(root); + } + } + [Fact] public void Run_DetailedJsonReportsLimitedSymbolRows_Issue2885() { @@ -170,20 +378,32 @@ public void Run_DetailedJsonReportsLimitedSymbolRows_Issue2885() { var leftDb = TestProjectHelper.CreateProjectDb(leftRoot); var rightDb = TestProjectHelper.CreateProjectDb(rightRoot); - TestProjectHelper.InsertIndexedFile(leftDb, "src/Same.cs", "csharp", "public class LeftOnly { public void Run() { } }"); - TestProjectHelper.InsertIndexedFile(rightDb, "src/Same.cs", "csharp", "public class RightOnly { public void Run() { } }"); + const string content = "public class Same { public void Run() { } }"; + TestProjectHelper.InsertIndexedFile(leftDb, "src/Same.cs", "csharp", content); + TestProjectHelper.InsertIndexedFile(rightDb, "src/Same.cs", "csharp", content); + InsertSyntheticMethodSymbol(leftDb, "src/Same.cs", "LeftOnly", "void LeftOnly()"); + InsertSyntheticMethodSymbol(rightDb, "src/Same.cs", "RightOnly", "void RightOnly()"); - var (exitCode, output) = RunWithCapturedOut([leftDb, rightDb, "--json", "--detailed", "--limit", "1"]); + var (exitCode, output) = RunWithCapturedOut( + [leftDb, rightDb, "--json", "--detailed", "--include-content", "--limit", "2"]); Assert.Equal(1, exitCode); using var document = JsonDocument.Parse(output); var root = document.RootElement; - var symbolsOnlyInLeft = root.GetProperty("symbols_only_in_left").EnumerateArray().ToList(); - var symbolsOnlyInRight = root.GetProperty("symbols_only_in_right").EnumerateArray().ToList(); - Assert.Single(symbolsOnlyInLeft); - Assert.Single(symbolsOnlyInRight); - Assert.Contains("LeftOnly", symbolsOnlyInLeft[0].GetString(), StringComparison.Ordinal); - Assert.Contains("RightOnly", symbolsOnlyInRight[0].GetString(), StringComparison.Ordinal); + var records = GetRecords(root, "symbol"); + Assert.Collection( + records, + record => + { + Assert.Equal("left", record.GetProperty("side").GetString()); + Assert.Equal("LeftOnly", GetField(record, "name").GetProperty("value").GetString()); + }, + record => + { + Assert.Equal("right", record.GetProperty("side").GetString()); + Assert.Equal("RightOnly", GetField(record, "name").GetProperty("value").GetString()); + }); + Assert.All(records, record => Assert.Equal(64, record.GetProperty("identity_sha256").GetString()?.Length)); } finally { @@ -211,29 +431,33 @@ public void Run_DetailedJsonPagesReferenceAndChunkDrift_Issue4714() ExecuteNonQuery(rightDb, "UPDATE symbol_references SET context = COALESCE(context, '') || ' // right'"); var (firstExitCode, firstOutput) = RunWithCapturedOut( - [leftDb, rightDb, "--json", "--detailed", "--limit", "1", "--offset", "0"]); - var (secondExitCode, secondOutput) = RunWithCapturedOut( - [leftDb, rightDb, "--json", "--detailed", "--limit", "1", "--offset", "1"]); + [leftDb, rightDb, "--json", "--detailed", "--limit", "1"]); Assert.Equal(1, firstExitCode); - Assert.Equal(1, secondExitCode); using var firstDocument = JsonDocument.Parse(firstOutput); - using var secondDocument = JsonDocument.Parse(secondOutput); var first = firstDocument.RootElement; - var second = secondDocument.RootElement; - Assert.Single(first.GetProperty("references_only_in_left").EnumerateArray()); - Assert.Single(first.GetProperty("references_only_in_right").EnumerateArray()); - Assert.Single(first.GetProperty("chunks_only_in_left").EnumerateArray()); - Assert.Single(first.GetProperty("chunks_only_in_right").EnumerateArray()); + var firstRecord = Assert.Single(first.GetProperty("records").EnumerateArray()); + Assert.Equal("reference", firstRecord.GetProperty("area").GetString()); Assert.True(first.GetProperty("has_more").GetBoolean()); Assert.Equal(1, first.GetProperty("next_offset").GetInt32()); + var nextCursor = first.GetProperty("next_cursor").GetString(); + Assert.False(string.IsNullOrWhiteSpace(nextCursor)); + Assert.Equal( + nextCursor, + first.GetProperty("replay").GetProperty("next_cursor").GetString()); + + var (secondExitCode, secondOutput) = RunWithCapturedOut( + [leftDb, rightDb, "--json", "--detailed", "--limit", "1", "--cursor", nextCursor!]); + + Assert.Equal(1, secondExitCode); + using var secondDocument = JsonDocument.Parse(secondOutput); + var second = secondDocument.RootElement; Assert.Equal(1, second.GetProperty("offset").GetInt32()); + var secondRecord = Assert.Single(second.GetProperty("records").EnumerateArray()); + Assert.Equal("reference", secondRecord.GetProperty("area").GetString()); Assert.NotEqual( - first.GetProperty("chunks_only_in_left")[0].GetString(), - second.GetProperty("chunks_only_in_left")[0].GetString()); - Assert.NotEqual( - first.GetProperty("references_only_in_left")[0].GetString(), - second.GetProperty("references_only_in_left")[0].GetString()); + firstRecord.GetProperty("identity_sha256").GetString(), + secondRecord.GetProperty("identity_sha256").GetString()); } finally { @@ -290,8 +514,9 @@ UPDATE symbol_references var root = document.RootElement; Assert.Equal("different", root.GetProperty("status").GetString()); Assert.False(root.GetProperty("identical").GetBoolean()); - Assert.NotEmpty(root.GetProperty("references_only_in_left").EnumerateArray()); - Assert.NotEmpty(root.GetProperty("references_only_in_right").EnumerateArray()); + var referenceRecords = GetRecords(root, "reference"); + Assert.Contains(referenceRecords, record => record.GetProperty("side").GetString() == "left"); + Assert.Contains(referenceRecords, record => record.GetProperty("side").GetString() == "right"); } finally { @@ -319,14 +544,13 @@ public void Run_DetailedJsonReportsOperationalMetadataDrift_Issue4357() var root = document.RootElement; Assert.Equal("identical", root.GetProperty("status").GetString()); Assert.True(root.GetProperty("identical").GetBoolean()); - Assert.Empty(root.GetProperty("files_only_in_left").EnumerateArray()); - Assert.Empty(root.GetProperty("files_only_in_right").EnumerateArray()); - Assert.Empty(root.GetProperty("symbols_only_in_left").EnumerateArray()); - Assert.Empty(root.GetProperty("symbols_only_in_right").EnumerateArray()); - var drift = Assert.Single(root.GetProperty("metadata_drift").EnumerateArray()); - Assert.Equal("indexed_project_root", drift.GetProperty("key").GetString()); - Assert.Equal(Path.GetFullPath(leftRoot), drift.GetProperty("left_value").GetString()); - Assert.Equal(Path.GetFullPath(rightRoot), drift.GetProperty("right_value").GetString()); + var drift = Assert.Single(GetRecords(root, "operational_metadata")); + Assert.Equal("changed", drift.GetProperty("side").GetString()); + Assert.True(GetField(drift, "key").GetProperty("redacted").GetBoolean()); + Assert.True(GetField(drift, "left_value").GetProperty("redacted").GetBoolean()); + Assert.True(GetField(drift, "right_value").GetProperty("redacted").GetBoolean()); + Assert.DoesNotContain(Path.GetFullPath(leftRoot), output, StringComparison.Ordinal); + Assert.DoesNotContain(Path.GetFullPath(rightRoot), output, StringComparison.Ordinal); } finally { @@ -350,17 +574,18 @@ public void Run_DetailedJsonTreatsWriterVersionAsOperationalMetadataDrift_Issue4 SetMeta(leftDb, DbContext.CdidxWriterVersionMetaKey, "writer-left"); SetMeta(rightDb, DbContext.CdidxWriterVersionMetaKey, "writer-right"); - var (exitCode, output) = RunWithCapturedOut([leftDb, rightDb, "--json", "--detailed", "--limit", "5"]); + var (exitCode, output) = RunWithCapturedOut( + [leftDb, rightDb, "--json", "--detailed", "--include-content", "--limit", "5"]); Assert.Equal(0, exitCode); using var document = JsonDocument.Parse(output); var root = document.RootElement; Assert.Equal("identical", root.GetProperty("status").GetString()); Assert.True(root.GetProperty("identical").GetBoolean()); - var drift = Assert.Single(root.GetProperty("metadata_drift").EnumerateArray()); - Assert.Equal("cdidx_writer_version", drift.GetProperty("key").GetString()); - Assert.Equal("writer-left", drift.GetProperty("left_value").GetString()); - Assert.Equal("writer-right", drift.GetProperty("right_value").GetString()); + var drift = Assert.Single(GetRecords(root, "operational_metadata")); + Assert.Equal("cdidx_writer_version", GetField(drift, "key").GetProperty("value").GetString()); + Assert.Equal("writer-left", GetField(drift, "left_value").GetProperty("value").GetString()); + Assert.Equal("writer-right", GetField(drift, "right_value").GetProperty("value").GetString()); } finally { @@ -476,7 +701,7 @@ public void ColumnExists_QuotesTableIdentifiersForPragmaInfo_Issue3834() } [Fact] - public void Run_DetailedJsonTruncatesLargeEncodedSymbolFields_Issue3163() + public void Run_DetailedJsonHashesLargeSymbolFieldsByDefault_Issue3163() { var leftRoot = TestProjectHelper.CreateTempProject("cdidx_diff_large_field_left"); var rightRoot = TestProjectHelper.CreateTempProject("cdidx_diff_large_field_right"); @@ -487,21 +712,44 @@ public void Run_DetailedJsonTruncatesLargeEncodedSymbolFields_Issue3163() TestProjectHelper.InsertIndexedFile(leftDb, "src/Same.cs", "csharp", "public class Same { }"); TestProjectHelper.InsertIndexedFile(rightDb, "src/Same.cs", "csharp", "public class Same { }"); - var longSignature = new string('a', DiffCommandRunner.MaxDiffEncodedFieldSampleLength * 4); + var longSignature = new string('a', LargeDiffFieldLength); InsertSyntheticMethodSymbol(leftDb, "src/Same.cs", "Drifted", longSignature); var (exitCode, output) = RunWithCapturedOut([leftDb, rightDb, "--json", "--detailed", "--limit", "1"]); Assert.Equal(1, exitCode); using var document = JsonDocument.Parse(output); - var row = Assert.Single(document.RootElement.GetProperty("symbols_only_in_left").EnumerateArray()).GetString(); - Assert.NotNull(row); + var row = Assert.Single(GetRecords(document.RootElement, "symbol")); + var signature = GetField(row, "signature"); var expectedHash = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(longSignature))).ToLowerInvariant(); - Assert.Contains(new string('a', DiffCommandRunner.MaxDiffEncodedFieldSampleLength), row, StringComparison.Ordinal); - Assert.Contains($"[truncated original_length={longSignature.Length}", row, StringComparison.Ordinal); - Assert.Contains($"sha256={expectedHash}", row, StringComparison.Ordinal); - Assert.DoesNotContain(new string('a', DiffCommandRunner.MaxDiffEncodedFieldSampleLength + 1), row, StringComparison.Ordinal); - Assert.True(row!.Length < longSignature.Length); + Assert.True(signature.GetProperty("redacted").GetBoolean()); + Assert.Equal(expectedHash, signature.GetProperty("sha256").GetString()); + Assert.Equal(longSignature.Length, signature.GetProperty("byte_length").GetInt64()); + Assert.False(signature.TryGetProperty("value", out _)); + Assert.DoesNotContain(longSignature, output, StringComparison.Ordinal); + + var (boundedExitCode, boundedOutput) = RunWithCapturedOut( + [ + leftDb, + rightDb, + "--json", + "--detailed", + "--include-content", + "--limit", + "1", + "--max-json-bytes", + $"{DiffCommandRunner.MinDiffJsonBytes}", + ]); + Assert.Equal(1, boundedExitCode); + Assert.InRange( + Encoding.UTF8.GetByteCount(boundedOutput), + 1, + DiffCommandRunner.MinDiffJsonBytes); + using var boundedDocument = JsonDocument.Parse(boundedOutput); + var boundedRoot = boundedDocument.RootElement; + Assert.Empty(boundedRoot.GetProperty("records").EnumerateArray()); + Assert.Equal("max_json_bytes", boundedRoot.GetProperty("truncation_reason").GetString()); + Assert.True(boundedRoot.GetProperty("first_omitted_record_bytes").GetInt32() > 0); } finally { @@ -522,7 +770,7 @@ public void Run_SummaryOnlyDetectsLargeRowDriftAfterSharedDisplayPrefix_Issue316 TestProjectHelper.InsertIndexedFile(leftDb, "src/Same.cs", "csharp", "public class Same { public void Run() { } }"); TestProjectHelper.InsertIndexedFile(rightDb, "src/Same.cs", "csharp", "public class Same { public void Run() { } }"); - var sharedPrefix = new string('x', DiffCommandRunner.MaxDiffEncodedFieldSampleLength); + var sharedPrefix = new string('x', LargeDiffFieldLength); UpdateFirstChunkContent(leftDb, sharedPrefix + "left"); UpdateFirstChunkContent(rightDb, sharedPrefix + "right"); @@ -553,16 +801,17 @@ public void Run_DetailedJsonUsesSqlOrderForStreamingSymbolDiff_Issue2885() TestProjectHelper.InsertIndexedFile(leftDb, "src/b.cs", "csharp", "public class b { }"); TestProjectHelper.InsertIndexedFile(rightDb, "src/b.cs", "csharp", "public class b { }"); - var (exitCode, output) = RunWithCapturedOut([leftDb, rightDb, "--json", "--detailed", "--limit", "5"]); + var (exitCode, output) = RunWithCapturedOut( + [leftDb, rightDb, "--json", "--detailed", "--include-content", "--limit", "5"]); Assert.Equal(1, exitCode); using var document = JsonDocument.Parse(output); var root = document.RootElement; - var symbolsOnlyInLeft = root.GetProperty("symbols_only_in_left").EnumerateArray().ToList(); - var symbolsOnlyInRight = root.GetProperty("symbols_only_in_right").EnumerateArray().ToList(); - Assert.Contains(symbolsOnlyInLeft, item => item.GetString()?.Contains("src/aa.cs", StringComparison.Ordinal) == true); - Assert.DoesNotContain(symbolsOnlyInLeft, item => item.GetString()?.Contains("src/b.cs", StringComparison.Ordinal) == true); - Assert.Empty(symbolsOnlyInRight); + var leftRecords = GetRecords(root, "symbol", "left"); + var rightRecords = GetRecords(root, "symbol", "right"); + Assert.Contains(leftRecords, record => GetField(record, "path").GetProperty("value").GetString() == "src/aa.cs"); + Assert.DoesNotContain(leftRecords, record => GetField(record, "path").GetProperty("value").GetString() == "src/b.cs"); + Assert.Empty(rightRecords); } finally { @@ -860,11 +1109,10 @@ public void Run_DetailedJsonIgnoresReferenceLineSurrogateIds_Issue4478() var root = document.RootElement; Assert.Equal("identical", root.GetProperty("status").GetString()); Assert.True(root.GetProperty("identical").GetBoolean()); - Assert.Empty(root.GetProperty("files_only_in_left").EnumerateArray()); - Assert.Empty(root.GetProperty("files_only_in_right").EnumerateArray()); - Assert.Empty(root.GetProperty("symbols_only_in_left").EnumerateArray()); - Assert.Empty(root.GetProperty("symbols_only_in_right").EnumerateArray()); - Assert.Empty(root.GetProperty("metadata_drift").EnumerateArray()); + Assert.Empty(root.GetProperty("records").EnumerateArray()); + Assert.Equal(0, root.GetProperty("total_count").GetInt64()); + Assert.Equal(0, root.GetProperty("returned_count").GetInt32()); + Assert.Equal(0, root.GetProperty("omitted_count").GetInt64()); } finally { @@ -1153,6 +1401,20 @@ private static void ExecuteNonQuery(string dbPath, string sql, Action GetRecords(JsonElement root, string area, string? side = null) + => root.GetProperty("records") + .EnumerateArray() + .Where(record => + record.GetProperty("area").GetString() == area + && (side is null || record.GetProperty("side").GetString() == side)) + .ToList(); + + private static JsonElement GetField(JsonElement record, string name) + => Assert.Single( + record.GetProperty("fields") + .EnumerateArray() + .Where(field => field.GetProperty("name").GetString() == name)); + private (int ExitCode, string Output) RunWithCapturedOut(string[] args, CancellationToken cancellationToken = default) { var originalOut = Console.Out; diff --git a/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs b/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs index c4350aaf3..29d4edb76 100644 --- a/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs @@ -1,6 +1,7 @@ using System.IO.Compression; using System.Globalization; using System.Security.Cryptography; +using System.Text; using System.Text.Json; using CodeIndex.Cli; using CodeIndex.Database; @@ -1230,10 +1231,20 @@ public void RunImport_DryRunReportsBoundedDestinationDeltaWithoutReplacingDb_Iss var comparison = delta.GetProperty("comparison"); Assert.Equal(1, comparison.GetProperty("limit").GetInt32()); Assert.Equal(0, comparison.GetProperty("offset").GetInt32()); - Assert.NotEmpty(comparison.GetProperty("symbols_only_in_left").EnumerateArray()); - Assert.NotEmpty(comparison.GetProperty("symbols_only_in_right").EnumerateArray()); - Assert.NotEmpty(comparison.GetProperty("chunks_only_in_left").EnumerateArray()); - Assert.NotEmpty(comparison.GetProperty("chunks_only_in_right").EnumerateArray()); + var record = Assert.Single(comparison.GetProperty("records").EnumerateArray()); + Assert.Equal("file", record.GetProperty("area").GetString()); + Assert.True(record.GetProperty("side").GetString() is "left" or "right"); + Assert.False(string.IsNullOrEmpty(record.GetProperty("identity_sha256").GetString())); + Assert.Contains( + record.GetProperty("fields").EnumerateArray(), + field => + field.GetProperty("name").GetString() == "path" + && field.GetProperty("redacted").GetBoolean()); + Assert.True(comparison.GetProperty("has_more").GetBoolean()); + Assert.Equal(1, comparison.GetProperty("returned_count").GetInt32()); + Assert.True(comparison.GetProperty("omitted_count").GetInt64() > 0); + Assert.DoesNotContain("public class Imported", stdout, StringComparison.Ordinal); + Assert.DoesNotContain("public class Existing", stdout, StringComparison.Ordinal); } finally { @@ -1302,9 +1313,20 @@ INSERT INTO files(path, lang, size, lines, checksum) .GetProperty("destination_delta") .GetProperty("comparison"); Assert.False(comparison.GetProperty("identical").GetBoolean()); + var expectedPathHash = Convert.ToHexString( + SHA256.HashData(Encoding.UTF8.GetBytes("src/WalOnly.cs"))) + .ToLowerInvariant(); Assert.Contains( - comparison.GetProperty("files_only_in_left").EnumerateArray(), - item => item.GetString() == "src/WalOnly.cs"); + comparison.GetProperty("records").EnumerateArray(), + record => + record.GetProperty("area").GetString() == "file" + && record.GetProperty("side").GetString() == "left" + && record.GetProperty("fields").EnumerateArray().Any( + field => + field.GetProperty("name").GetString() == "path" + && field.GetProperty("sha256").GetString() == expectedPathHash + && field.GetProperty("redacted").GetBoolean())); + Assert.DoesNotContain("src/WalOnly.cs", stdout, StringComparison.Ordinal); } finally { From 4f707285dfb790a4b0e9e5c87dc04325b9dbf71f Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 28 Jul 2026 18:36:11 +0900 Subject: [PATCH 2/5] Fix detailed diff continuation bounds (#4859) --- TESTING_GUIDE.md | 2 + src/CodeIndex/Cli/DiffCommandRunner.cs | 52 +++++++++-- src/CodeIndex/Cli/DiffResultWriter.cs | 41 +++++++-- .../CodeIndex.Tests/DiffCommandRunnerTests.cs | 89 +++++++++++++++++++ 4 files changed, 170 insertions(+), 14 deletions(-) diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 2841accfb..479be60ee 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -844,6 +844,7 @@ For background log or metrics sinks, use explicit writer-entry/release signals a - When command, subcommand, or flag metadata changes, assert that every accepted primary flag appears in shell completions and in command help when the shared parser is authoritative. Dedicated or nested parsers must instead prove their exact usage syntax is complete and does not emit a misleading partial parent-command option list; nested command families still use the shared catalog, and optional-subcommand families retain parent flag candidates alongside nested verbs. - Immediate help-routing tests must assert both rendered usage and exit code, use a nested command whose normal execution would require additional input so the test proves help does not dispatch it, accept supported public aliases, and reject internal usage keys. - For JSON output, parse it with `JsonDocument` instead of asserting raw strings. +- Detailed database-diff coverage must keep record materialization within the requested JSON budget, stop only at complete UTF-8 record boundaries, and withhold continuations that cannot advance. Assert recovery guidance for zero-limit and first-record-too-large pages, plus valid `--offset` continuation for non-detailed JSON and detailed text output. - Keep the `files` / `map` `--exclude-tests` cross-command invariant in one seeded fixture: compare `map.file_count` with the parsed `files` row count for both the implicit production-source preset and an explicit `--path`. - JSON failure-contract tests must assert that stdout contains one parseable versioned error object, stderr is empty, the documented exit code is preserved, and `status` / `error_code` identify the failure. - Auth-token recipe ranking coverage must pair high-signal credential fixtures with comment, regex-definition, and structural `CancellationToken` / syntax-token decoys, then assert explicit top-N precision for the bearer/authorization, GitHub/API/access-token, and token-secret child queries. @@ -1738,6 +1739,7 @@ background の log / metrics sink は、sleep や狭い stopwatch 閾値では - command、subcommand、flag metadata を変更した場合は、受理される全 primary flag が shell completion に現れ、共有 parser が正本の場合は command help にも現れることを検証する。専用 parser / nested parser では代わりに、正確な usage 構文が完全で、誤解を招く不完全な親 command の option 一覧を出さないことを証明し、nested command family が共有 catalog を使い、optional-subcommand family では nested verb と親 command の flag 候補を併せて維持することも確認する。 - immediate help routing の test は rendered usage と終了コードを併せて検証し、通常実行なら追加入力を必要とする nested command を含めて help が dispatch しないことを証明し、対応する公開 alias の受理と内部 usage key の拒否も確認する。 - JSON 出力は生文字列比較ではなく `JsonDocument` で解析して検証する。 +- 詳細 database diff の coverage では、record の materialize を指定 JSON budget 内に抑え、完全な UTF-8 record 境界でのみ停止し、前進できない continuation を返さないことを維持する。limit 0 と最初の record が大きすぎる page の recovery 案内に加え、非詳細 JSON と詳細 text 出力で有効な `--offset` continuation を検証する。 - `files` / `map` の `--exclude-tests` に関する cross-command invariant は、1 つの seed 済み fixture で維持する。暗黙の本番ソース preset と明示的な `--path` の両方について、`map.file_count` と parse 済み `files` row 数を比較する。 - JSON failure contract のテストでは、stdout に parse 可能な version 付き error object が 1 件だけあること、stderr が空であること、documented exit code が維持されること、`status` / `error_code` が失敗を識別することを検証する。 - auth-token recipe の ranking coverage では、高 signal な credential fixture と comment、regex 定義、構造的な `CancellationToken` / syntax-token decoy を対にし、bearer / authorization、GitHub / API / access-token、token-secret の各 child query について明示的な top-N precision を検証する。 diff --git a/src/CodeIndex/Cli/DiffCommandRunner.cs b/src/CodeIndex/Cli/DiffCommandRunner.cs index ed85f0d3e..fae7b94f2 100644 --- a/src/CodeIndex/Cli/DiffCommandRunner.cs +++ b/src/CodeIndex/Cli/DiffCommandRunner.cs @@ -458,7 +458,13 @@ private static DiffJsonResult BuildDiff(DiffDbHeader left, DiffDbHeader right, D DiffRecordPageCollector? collector = null; if (options.Detailed && !options.SummaryOnly) { - collector = new DiffRecordPageCollector(options.Offset, options.Limit, options.IncludeContent); + collector = new DiffRecordPageCollector( + options.Offset, + options.Limit, + options.IncludeContent, + options.Json + ? options.MaxJsonBytes ?? DefaultDiffJsonBytes + : null); cancellationToken.ThrowIfCancellationRequested(); identical &= CollectOrderedRows( @@ -583,7 +589,7 @@ private static DiffJsonResult BuildDiff(DiffDbHeader left, DiffDbHeader right, D options.RightDb!, options.IncludeContent); currentCursor = DiffCursorCodec.Encode(options.Offset, selectionFingerprint); - if (hasMore) + if (hasMore && records.Count > 0) { nextOffset = checked(options.Offset + records.Count); nextCursor = DiffCursorCodec.Encode(nextOffset.Value, selectionFingerprint); @@ -594,11 +600,17 @@ private static DiffJsonResult BuildDiff(DiffDbHeader left, DiffDbHeader right, D { diagnostics.Add(new DiffDiagnosticJsonResult( "diff_records_truncated", - hasMore + hasMore && records.Count > 0 ? "Detailed diff records were omitted; use replay.next_page_arguments to continue from the next whole record." + : hasMore && options.Limit == 0 + ? "Detailed diff records were omitted because --limit is 0; rerun with a positive --limit." : "Detailed diff records before the requested offset were omitted from this page.")); } } + else if (hasMore && options.Limit > 0) + { + nextOffset = checked(options.Offset + options.Limit); + } return new DiffJsonResult( identical ? "identical" : "different", @@ -1452,12 +1464,20 @@ private sealed class DiffRecordPageCollector private readonly int _offset; private readonly int _limit; private readonly bool _includeContent; - - internal DiffRecordPageCollector(int offset, int limit, bool includeContent) + private readonly int? _materializationByteBudget; + private long _materializedRecordBytes; + private bool _materializationBudgetReached; + + internal DiffRecordPageCollector( + int offset, + int limit, + bool includeContent, + int? materializationByteBudget) { _offset = offset; _limit = limit; _includeContent = includeContent; + _materializationByteBudget = materializationByteBudget; } internal List Records { get; } = []; @@ -1465,8 +1485,26 @@ internal DiffRecordPageCollector(int offset, int limit, bool includeContent) internal void Add(DiffRowSchema schema, string side, DiffRow row) { - if (TotalCount >= _offset && Records.Count < _limit) - Records.Add(CreateDiffRecord(schema, side, row, _includeContent)); + if (TotalCount >= _offset && + Records.Count < _limit && + !_materializationBudgetReached) + { + var record = CreateDiffRecord(schema, side, row, _includeContent); + if (_materializationByteBudget is null) + { + Records.Add(record); + } + else + { + var recordBytes = JsonSerializer.SerializeToUtf8Bytes( + record, + CliJsonSerializerContext.Default.DiffRecordJsonResult).LongLength; + Records.Add(record); + _materializedRecordBytes = checked(_materializedRecordBytes + recordBytes); + _materializationBudgetReached = + _materializedRecordBytes > _materializationByteBudget.Value; + } + } TotalCount++; } diff --git a/src/CodeIndex/Cli/DiffResultWriter.cs b/src/CodeIndex/Cli/DiffResultWriter.cs index df1be0343..9be3caeae 100644 --- a/src/CodeIndex/Cli/DiffResultWriter.cs +++ b/src/CodeIndex/Cli/DiffResultWriter.cs @@ -86,7 +86,7 @@ internal static string FormatDelta(long delta) var context = CliJsonSerializerContextFactory.Create(jsonOptions); var sourceRecords = result.Records ?? []; var low = 0; - var high = sourceRecords.Count; + var high = GetSerializedRecordCeiling(sourceRecords, budget, context); DiffJsonResult? bestResult = null; string? bestJson = null; while (low <= high) @@ -163,10 +163,10 @@ private static void WriteText(DiffJsonResult result, DiffCommandOptions options) WriteList("files only in right", result.FilesOnlyInRight); if (options.Detailed) WriteRecords(result.Records ?? []); - if (result.HasMore && result.NextCursor is not null) - Console.WriteLine($" more : rerun with --cursor {result.NextCursor}"); - else if (result.HasMore && result.NextOffset is int nextOffset) + if (result.HasMore && result.NextOffset is int nextOffset) Console.WriteLine($" more : rerun with --offset {nextOffset}"); + else if (result.HasMore && result.Limit == 0) + Console.WriteLine(" more : rerun with a positive --limit"); } private static void WriteList(string label, List values) @@ -209,7 +209,10 @@ private static DiffJsonResult BuildBoundedCandidate( var totalCount = source.TotalCount ?? sourceRecords.Count; var omittedCount = totalCount - returnedCount; var hasMore = totalCount > (long)source.Offset + returnedCount; - var nextOffset = hasMore ? checked(source.Offset + returnedCount) : default(int?); + var canAdvance = hasMore && returnedCount > 0; + var nextOffset = canAdvance + ? checked(source.Offset + returnedCount) + : default(int?); var selectionFingerprint = source.SelectionFingerprint ?? throw new InvalidOperationException("detailed diff selection fingerprint is missing"); var currentCursor = source.CurrentCursor @@ -225,10 +228,14 @@ private static DiffJsonResult BuildBoundedCandidate( { diagnostics.Add(new DiffDiagnosticJsonResult( byteTruncated ? "diff_json_bytes_truncated" : "diff_records_truncated", - byteTruncated + byteTruncated && returnedCount == 0 + ? "No whole detailed diff record fits within --max-json-bytes; increase the byte budget or rerun without --include-content." + : byteTruncated ? "Detailed diff output stopped at a whole-record boundary to honor --max-json-bytes; use replay.next_page_arguments to continue." - : hasMore + : hasMore && returnedCount > 0 ? "Detailed diff records were omitted; use replay.next_page_arguments to continue from the next whole record." + : hasMore && options.Limit == 0 + ? "Detailed diff records were omitted because --limit is 0; rerun with a positive --limit." : "Detailed diff records before the requested offset were omitted from this page.")); } @@ -264,6 +271,26 @@ private static DiffJsonResult BuildBoundedCandidate( }; } + private static int GetSerializedRecordCeiling( + List records, + int budget, + CliJsonSerializerContext context) + { + long measuredBytes = 0; + for (var i = 0; i < records.Count; i++) + { + var recordBytes = JsonSerializer.SerializeToUtf8Bytes( + records[i], + context.DiffRecordJsonResult).LongLength; + var separatorBytes = i == 0 ? 0 : 1; + if (recordBytes + separatorBytes > budget - measuredBytes) + return i + 1; + measuredBytes += recordBytes + separatorBytes; + } + + return records.Count; + } + private static bool FitsJsonLine(string json, int maxJsonBytes) => System.Text.Encoding.UTF8.GetByteCount(json) + System.Text.Encoding.UTF8.GetByteCount(Console.Out.NewLine) diff --git a/tests/CodeIndex.Tests/DiffCommandRunnerTests.cs b/tests/CodeIndex.Tests/DiffCommandRunnerTests.cs index af563ec72..7e2b337cd 100644 --- a/tests/CodeIndex.Tests/DiffCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/DiffCommandRunnerTests.cs @@ -183,6 +183,44 @@ public void ParseArgs_JsonBudgetAndContentFlagsEnforceDetailedContract_Issue4859 contentWithoutDetailedJson.ParseError); } + [Fact] + public void Run_PagedModesOnlyAdvertiseValidAdvancingContinuations_Issue4859() + { + var leftRoot = TestProjectHelper.CreateTempProject("cdidx_diff_continuation_left"); + var rightRoot = TestProjectHelper.CreateTempProject("cdidx_diff_continuation_right"); + try + { + var leftDb = TestProjectHelper.CreateProjectDb(leftRoot); + var rightDb = TestProjectHelper.CreateProjectDb(rightRoot); + for (var i = 0; i < 3; i++) + { + TestProjectHelper.InsertIndexedFile( + leftDb, + $"src/OnlyInLeft{i}.cs", + "csharp", + $"public class OnlyInLeft{i} {{ }}"); + } + + var (sampleExitCode, sampleOutput) = RunWithCapturedOut( + [leftDb, rightDb, "--json", "--limit", "1"]); + var (detailedTextExitCode, detailedTextOutput) = RunWithCapturedOut( + [leftDb, rightDb, "--detailed", "--limit", "1"]); + + Assert.Equal(1, sampleExitCode); + using var sampleDocument = JsonDocument.Parse(sampleOutput); + Assert.True(sampleDocument.RootElement.GetProperty("has_more").GetBoolean()); + Assert.Equal(1, sampleDocument.RootElement.GetProperty("next_offset").GetInt32()); + Assert.Equal(1, detailedTextExitCode); + Assert.Contains("rerun with --offset 1", detailedTextOutput, StringComparison.Ordinal); + Assert.DoesNotContain("rerun with --cursor", detailedTextOutput, StringComparison.Ordinal); + } + finally + { + TestProjectHelper.DeleteDirectory(leftRoot); + TestProjectHelper.DeleteDirectory(rightRoot); + } + } + [Fact] public void Run_DetailedJsonRedactsSensitiveTextUntilContentIsExplicitlyIncluded_Issue4859() { @@ -334,6 +372,47 @@ record => firstIdentities.Contains(record.GetProperty("identity_sha256").GetStri } } + [Fact] + public void Run_DetailedJsonLimitZeroWithholdsNonAdvancingReplay_Issue4859() + { + var leftRoot = TestProjectHelper.CreateTempProject("cdidx_diff_zero_limit_left"); + var rightRoot = TestProjectHelper.CreateTempProject("cdidx_diff_zero_limit_right"); + try + { + var leftDb = TestProjectHelper.CreateProjectDb(leftRoot); + var rightDb = TestProjectHelper.CreateProjectDb(rightRoot); + TestProjectHelper.InsertIndexedFile( + leftDb, + "src/OnlyInLeft.cs", + "csharp", + "public class OnlyInLeft { }"); + + var (exitCode, output) = RunWithCapturedOut( + [leftDb, rightDb, "--json", "--detailed", "--limit", "0"]); + + Assert.Equal(1, exitCode); + using var document = JsonDocument.Parse(output); + var root = document.RootElement; + Assert.True(root.GetProperty("has_more").GetBoolean()); + Assert.Empty(root.GetProperty("records").EnumerateArray()); + Assert.Equal(JsonValueKind.Null, root.GetProperty("next_offset").ValueKind); + Assert.False(root.TryGetProperty("next_cursor", out _)); + var replay = root.GetProperty("replay"); + Assert.False(replay.TryGetProperty("next_cursor", out _)); + Assert.False(replay.TryGetProperty("next_page_arguments", out _)); + Assert.Contains( + root.GetProperty("diagnostics").EnumerateArray(), + diagnostic => diagnostic.GetProperty("message").GetString()?.Contains( + "positive --limit", + StringComparison.Ordinal) == true); + } + finally + { + TestProjectHelper.DeleteDirectory(leftRoot); + TestProjectHelper.DeleteDirectory(rightRoot); + } + } + [Fact] public void Run_JsonModesHonorMinimumExplicitBudget_Issue4859() { @@ -750,6 +829,16 @@ public void Run_DetailedJsonHashesLargeSymbolFieldsByDefault_Issue3163() Assert.Empty(boundedRoot.GetProperty("records").EnumerateArray()); Assert.Equal("max_json_bytes", boundedRoot.GetProperty("truncation_reason").GetString()); Assert.True(boundedRoot.GetProperty("first_omitted_record_bytes").GetInt32() > 0); + Assert.Equal(JsonValueKind.Null, boundedRoot.GetProperty("next_offset").ValueKind); + Assert.False(boundedRoot.TryGetProperty("next_cursor", out _)); + var replay = boundedRoot.GetProperty("replay"); + Assert.False(replay.TryGetProperty("next_cursor", out _)); + Assert.False(replay.TryGetProperty("next_page_arguments", out _)); + Assert.Contains( + boundedRoot.GetProperty("diagnostics").EnumerateArray(), + diagnostic => diagnostic.GetProperty("message").GetString()?.Contains( + "increase the byte budget", + StringComparison.OrdinalIgnoreCase) == true); } finally { From 3aa503566442205d49f3dfa4cb85f119c06a8ca6 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 28 Jul 2026 19:21:14 +0900 Subject: [PATCH 3/5] Bind diff cursors to compared content (#4859) --- TESTING_GUIDE.md | 4 +- USER_GUIDE.md | 8 +- changelog.d/unreleased/4859.security.md | 5 +- src/CodeIndex/Cli/DiffCommandOptionsParser.cs | 12 +- src/CodeIndex/Cli/DiffCommandRunner.cs | 119 ++++++++++++++---- src/CodeIndex/Cli/DiffCursorCodec.cs | 55 ++++++-- .../ExportImportCommandRunner.ArchiveScope.cs | 3 +- .../CodeIndex.Tests/DiffCommandRunnerTests.cs | 16 +++ .../ExportImportCommandRunnerTests.cs | 4 + 9 files changed, 175 insertions(+), 51 deletions(-) diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 169a809cf..0d1525a1a 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -845,7 +845,7 @@ For background log or metrics sinks, use explicit writer-entry/release signals a - When command, subcommand, or flag metadata changes, assert that every accepted primary flag appears in shell completions and in command help when the shared parser is authoritative. Dedicated or nested parsers must instead prove their exact usage syntax is complete and does not emit a misleading partial parent-command option list; nested command families still use the shared catalog, and optional-subcommand families retain parent flag candidates alongside nested verbs. - Immediate help-routing tests must assert both rendered usage and exit code, use a nested command whose normal execution would require additional input so the test proves help does not dispatch it, accept supported public aliases, and reject internal usage keys. - For JSON output, parse it with `JsonDocument` instead of asserting raw strings. -- Detailed database-diff coverage must keep record materialization within the requested JSON budget, stop only at complete UTF-8 record boundaries, and withhold continuations that cannot advance. Assert recovery guidance for zero-limit and first-record-too-large pages, plus valid `--offset` continuation for non-detailed JSON and detailed text output. +- Detailed database-diff coverage must keep record materialization within the requested JSON budget plus at most one candidate record, stop only at complete UTF-8 record boundaries, reject cursors after the deterministic difference sequence changes, and withhold continuations that cannot advance. Assert recovery guidance for zero-limit and first-record-too-large pages, suppress unusable cursor/replay metadata in embedded import dry-run comparisons, and preserve valid `--offset` continuation for non-detailed JSON and detailed text output. - Keep the `files` / `map` `--exclude-tests` cross-command invariant in one seeded fixture: compare `map.file_count` with the parsed `files` row count for both the implicit production-source preset and an explicit `--path`. - JSON failure-contract tests must assert that stdout contains one parseable versioned error object, stderr is empty, the documented exit code is preserved, and `status` / `error_code` identify the failure. - Auth-token recipe ranking coverage must pair high-signal credential fixtures with comment, regex-definition, and structural `CancellationToken` / syntax-token decoys, then assert explicit top-N precision for the bearer/authorization, GitHub/API/access-token, and token-secret child queries. @@ -1741,7 +1741,7 @@ background の log / metrics sink は、sleep や狭い stopwatch 閾値では - command、subcommand、flag metadata を変更した場合は、受理される全 primary flag が shell completion に現れ、共有 parser が正本の場合は command help にも現れることを検証する。専用 parser / nested parser では代わりに、正確な usage 構文が完全で、誤解を招く不完全な親 command の option 一覧を出さないことを証明し、nested command family が共有 catalog を使い、optional-subcommand family では nested verb と親 command の flag 候補を併せて維持することも確認する。 - immediate help routing の test は rendered usage と終了コードを併せて検証し、通常実行なら追加入力を必要とする nested command を含めて help が dispatch しないことを証明し、対応する公開 alias の受理と内部 usage key の拒否も確認する。 - JSON 出力は生文字列比較ではなく `JsonDocument` で解析して検証する。 -- 詳細 database diff の coverage では、record の materialize を指定 JSON budget 内に抑え、完全な UTF-8 record 境界でのみ停止し、前進できない continuation を返さないことを維持する。limit 0 と最初の record が大きすぎる page の recovery 案内に加え、非詳細 JSON と詳細 text 出力で有効な `--offset` continuation を検証する。 +- 詳細 database diff の coverage では、record の materialize を指定 JSON budget と最大 1 件の candidate record 以内に抑え、完全な UTF-8 record 境界でのみ停止し、deterministic な差分 sequence が変化した後の cursor を拒否し、前進できない continuation を返さないことを維持する。limit 0 と最初の record が大きすぎる page の recovery 案内、埋め込み import dry-run 比較で利用不能な cursor / replay metadata を抑止することに加え、非詳細 JSON と詳細 text 出力で有効な `--offset` continuation を検証する。 - `files` / `map` の `--exclude-tests` に関する cross-command invariant は、1 つの seed 済み fixture で維持する。暗黙の本番ソース preset と明示的な `--path` の両方について、`map.file_count` と parse 済み `files` row 数を比較する。 - JSON failure contract のテストでは、stdout に parse 可能な version 付き error object が 1 件だけあること、stderr が空であること、documented exit code が維持されること、`status` / `error_code` が失敗を識別することを検証する。 - auth-token recipe の ranking coverage では、高 signal な credential fixture と comment、regex 定義、構造的な `CancellationToken` / syntax-token decoy を対にし、bearer / authorization、GitHub / API / access-token、token-secret の各 child query について明示的な top-N precision を検証する。 diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 30198ba85..9f00c6bea 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -532,7 +532,9 @@ record boundaries, so the result remains valid JSON. `total_count`, `returned_co `--limit` bounds the unified record page, `--offset` remains available for direct paging, and `next_cursor` plus `replay.next_page_arguments` provide the preferred deterministic continuation contract. Reuse the same database -arguments and emitted replay flags to resume. +arguments and emitted replay flags to resume. Cursors bind the complete +deterministic difference sequence as well as the database arguments and content +policy; if either database changes, restart without `--cursor`. ## Flag compatibility and migrations @@ -3756,7 +3758,9 @@ page の状態は `total_count`、`returned_count`、`omitted_count`、`truncate `truncation_reason` で確認できます。`--limit` は unified record page を制限し、 direct paging 用の `--offset` も引き続き使えます。deterministic な続きの取得には `next_cursor` と `replay.next_page_arguments` を使うことを推奨します。同じ database -arguments と、出力された replay flags を再利用して再開してください。 +arguments と、出力された replay flags を再利用して再開してください。cursor は database +arguments と content policy に加えて deterministic な差分 sequence 全体に束縛されます。 +いずれかの database が変更された場合は `--cursor` なしで最初からやり直してください。 ## フラグ互換性と移行 diff --git a/changelog.d/unreleased/4859.security.md b/changelog.d/unreleased/4859.security.md index ec609630c..aa42f5184 100644 --- a/changelog.d/unreleased/4859.security.md +++ b/changelog.d/unreleased/4859.security.md @@ -12,13 +12,14 @@ affected: - tests/CodeIndex.Tests/DiffCommandRunnerTests.cs - tests/CodeIndex.Tests/DiffCommandHelpersTests.cs - tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs + - TESTING_GUIDE.md - USER_GUIDE.md --- ## English -- **Detailed database diffs are now structured, redacted, and byte-bounded by default (#4859)** — `diff --detailed --json` emits named fields with hashes and byte lengths instead of opaque source-bearing rows, caps the complete UTF-8 JSON output at 1 MiB by default, and returns count, truncation, cursor, and replay metadata. Callers can set `--max-json-bytes`, resume with `--cursor`, or explicitly opt into source and path values with `--include-content`. +- **Detailed database diffs are now structured, redacted, and byte-bounded by default (#4859)** — `diff --detailed --json` emits named fields with hashes and byte lengths instead of opaque source-bearing rows, caps the complete UTF-8 JSON output at 1 MiB by default, and returns count, truncation, cursor, and replay metadata. Cursors bind the deterministic difference sequence and reject changed database contents. Callers can set `--max-json-bytes`, resume with `--cursor`, or explicitly opt into source and path values with `--include-content`. ## 日本語 -- **詳細 database diff を既定で構造化・秘匿化し、byte 上限を適用するようになりました (#4859)** — `diff --detailed --json` は source を含み得る opaque row の代わりに hash と byte length を持つ名前付き field を出力し、UTF-8 JSON 全体を既定で 1 MiB に制限して、件数・truncation・cursor・replay metadata を返します。caller は `--max-json-bytes` で上限を指定し、`--cursor` で再開できます。source と path の値が必要な場合だけ `--include-content` で明示的に opt-in できます。 +- **詳細 database diff を既定で構造化・秘匿化し、byte 上限を適用するようになりました (#4859)** — `diff --detailed --json` は source を含み得る opaque row の代わりに hash と byte length を持つ名前付き field を出力し、UTF-8 JSON 全体を既定で 1 MiB に制限して、件数・truncation・cursor・replay metadata を返します。cursor は deterministic な差分 sequence に束縛され、database 内容が変わると拒否されます。caller は `--max-json-bytes` で上限を指定し、`--cursor` で再開できます。source と path の値が必要な場合だけ `--include-content` で明示的に opt-in できます。 diff --git a/src/CodeIndex/Cli/DiffCommandOptionsParser.cs b/src/CodeIndex/Cli/DiffCommandOptionsParser.cs index 3efcd07e8..033a05bab 100644 --- a/src/CodeIndex/Cli/DiffCommandOptionsParser.cs +++ b/src/CodeIndex/Cli/DiffCommandOptionsParser.cs @@ -16,6 +16,7 @@ internal static DiffCommandOptions Parse(string[] args, int maxLimit) var offsetExplicit = false; int? maxJsonBytes = null; string? cursor = null; + string? cursorSelectionFingerprint = null; string? parseError = null; for (var i = 0; i < args.Length; i++) @@ -109,11 +110,11 @@ internal static DiffCommandOptions Parse(string[] args, int maxLimit) parseError = "--cursor cannot be combined with --offset"; if (parseError is null && cursor is not null) { - var selectionFingerprint = DiffCursorCodec.CreateSelectionFingerprint( - dbs[0], - dbs[1], - includeContent); - if (!DiffCursorCodec.TryDecode(cursor, selectionFingerprint, out offset, out var cursorError)) + if (!DiffCursorCodec.TryDecode( + cursor, + out offset, + out cursorSelectionFingerprint, + out var cursorError)) parseError = cursorError; } if (parseError is null && offset > int.MaxValue - limit) @@ -132,6 +133,7 @@ internal static DiffCommandOptions Parse(string[] args, int maxLimit) OffsetExplicit = offsetExplicit, MaxJsonBytes = maxJsonBytes, Cursor = cursor, + CursorSelectionFingerprint = cursorSelectionFingerprint, ParseError = parseError, }; } diff --git a/src/CodeIndex/Cli/DiffCommandRunner.cs b/src/CodeIndex/Cli/DiffCommandRunner.cs index fae7b94f2..60a56e5d0 100644 --- a/src/CodeIndex/Cli/DiffCommandRunner.cs +++ b/src/CodeIndex/Cli/DiffCommandRunner.cs @@ -64,6 +64,21 @@ public static int Run(string[] args, JsonSerializerOptions jsonOptions, Cancella try { var result = CompareDatabases(options, cancellationToken); + if (options.CursorSelectionFingerprint is { } cursorSelectionFingerprint + && (result.SelectionFingerprint is not { } currentSelectionFingerprint + || !DiffCursorCodec.FixedTimeEquals( + cursorSelectionFingerprint, + currentSelectionFingerprint))) + { + return DiffResultWriter.WriteCommandError( + options.Json || options.SummaryOnly, + jsonOptions, + "--cursor no longer matches the selected database contents; restart without --cursor", + CommandExitCodes.UsageError, + "Rerun the detailed diff without --cursor to start from the current deterministic record sequence.", + CommandErrorCodes.UsageError, + options.MaxJsonBytes); + } var outputExitCode = DiffResultWriter.WriteResult(result, options, jsonOptions); if (outputExitCode.HasValue) @@ -128,7 +143,8 @@ internal static DiffJsonResult CompareDatabases( CancellationToken cancellationToken, string? leftDisplayPath = null, string? rightDisplayPath = null, - bool includeContent = false) + bool includeContent = false, + bool emitCursorMetadata = true) { var options = new DiffCommandOptions { @@ -138,6 +154,7 @@ internal static DiffJsonResult CompareDatabases( Offset = offset, Detailed = detailed, IncludeContent = includeContent, + EmitCursorMetadata = emitCursorMetadata, }; var result = CompareDatabases(options, cancellationToken); return result with @@ -455,17 +472,19 @@ private static DiffJsonResult BuildDiff(DiffDbHeader left, DiffDbHeader right, D AddPagingDiagnostic(diagnostics, fileDiff.Omitted, fileDiff.HasMore, "file differences", options); } - DiffRecordPageCollector? collector = null; - if (options.Detailed && !options.SummaryOnly) - { - collector = new DiffRecordPageCollector( + using DiffRecordPageCollector? collector = options.Detailed && !options.SummaryOnly + ? new DiffRecordPageCollector( + options.LeftDb!, + options.RightDb!, options.Offset, options.Limit, options.IncludeContent, options.Json ? options.MaxJsonBytes ?? DefaultDiffJsonBytes - : null); - + : null) + : null; + if (collector is not null) + { cancellationToken.ThrowIfCancellationRequested(); identical &= CollectOrderedRows( leftConnection, @@ -584,24 +603,30 @@ private static DiffJsonResult BuildDiff(DiffDbHeader left, DiffDbHeader right, D hasMore = collector.TotalCount > (long)options.Offset + records.Count; truncated = omittedCount > 0; truncationReason = truncated ? "limit_or_offset" : null; - selectionFingerprint = DiffCursorCodec.CreateSelectionFingerprint( - options.LeftDb!, - options.RightDb!, - options.IncludeContent); - currentCursor = DiffCursorCodec.Encode(options.Offset, selectionFingerprint); - if (hasMore && records.Count > 0) + selectionFingerprint = collector.CompleteSelectionFingerprint(); + if (options.EmitCursorMetadata) + { + currentCursor = DiffCursorCodec.Encode(options.Offset, selectionFingerprint); + if (hasMore && records.Count > 0) + { + nextOffset = checked(options.Offset + records.Count); + nextCursor = DiffCursorCodec.Encode(nextOffset.Value, selectionFingerprint); + } + + replay = BuildReplayMetadata(options, selectionFingerprint, currentCursor, nextCursor); + } + else if (hasMore && records.Count > 0) { nextOffset = checked(options.Offset + records.Count); - nextCursor = DiffCursorCodec.Encode(nextOffset.Value, selectionFingerprint); } - - replay = BuildReplayMetadata(options, selectionFingerprint, currentCursor, nextCursor); if (truncated) { diagnostics.Add(new DiffDiagnosticJsonResult( "diff_records_truncated", - hasMore && records.Count > 0 + hasMore && records.Count > 0 && options.EmitCursorMetadata ? "Detailed diff records were omitted; use replay.next_page_arguments to continue from the next whole record." + : hasMore && records.Count > 0 + ? $"Detailed diff records were omitted; continue from the next whole record with --offset {nextOffset}." : hasMore && options.Limit == 0 ? "Detailed diff records were omitted because --limit is 0; rerun with a positive --limit." : "Detailed diff records before the requested offset were omitted from this page.")); @@ -684,7 +709,7 @@ private static DiffJsonResult BuildSchemaMismatchDiff(DiffDbHeader left, DiffDbH var selectionFingerprint = options.Detailed ? DiffCursorCodec.CreateSelectionFingerprint(options.LeftDb!, options.RightDb!, options.IncludeContent) : null; - var currentCursor = selectionFingerprint is null + var currentCursor = selectionFingerprint is null || !options.EmitCursorMetadata ? null : DiffCursorCodec.Encode(options.Offset, selectionFingerprint); @@ -1347,11 +1372,9 @@ private static string FormatSensitiveText(string value, DiffCommandOptions optio $"sha256:{hash} ({bytes.LongLength} UTF-8 bytes)"); } - private static DiffRecordJsonResult CreateDiffRecord( + private static string CreateDiffRecordIdentity( DiffRowSchema schema, - string side, - DiffRow row, - bool includeContent) + DiffRow row) { if (schema.FieldNames.Length != row.SortValues.Length) { @@ -1361,20 +1384,32 @@ private static DiffRecordJsonResult CreateDiffRecord( using var identityHash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); AppendIdentityPart(identityHash, schema.Area); - var fields = new List(row.SortValues.Length); for (var i = 0; i < row.SortValues.Length; i++) { var name = schema.FieldNames[i]; var value = row.SortValues[i]; AppendIdentityPart(identityHash, name); AppendIdentityValue(identityHash, value); - fields.Add(CreateDiffField(name, value, includeContent)); } + return HexEncoding.ToLowerHexString(identityHash.GetHashAndReset()); + } + + private static DiffRecordJsonResult CreateDiffRecord( + DiffRowSchema schema, + string side, + DiffRow row, + string identitySha256, + bool includeContent) + { + var fields = new List(row.SortValues.Length); + for (var i = 0; i < row.SortValues.Length; i++) + fields.Add(CreateDiffField(schema.FieldNames[i], row.SortValues[i], includeContent)); + return new DiffRecordJsonResult( schema.Area, side, - HexEncoding.ToLowerHexString(identityHash.GetHashAndReset()), + identitySha256, fields); } @@ -1459,16 +1494,20 @@ private sealed record DiffRowSchema( string Area, string[] FieldNames); - private sealed class DiffRecordPageCollector + private sealed class DiffRecordPageCollector : IDisposable { private readonly int _offset; private readonly int _limit; private readonly bool _includeContent; private readonly int? _materializationByteBudget; + private readonly IncrementalHash _selectionHash; private long _materializedRecordBytes; private bool _materializationBudgetReached; + private bool _selectionFingerprintCompleted; internal DiffRecordPageCollector( + string leftDb, + string rightDb, int offset, int limit, bool includeContent, @@ -1478,6 +1517,7 @@ internal DiffRecordPageCollector( _limit = limit; _includeContent = includeContent; _materializationByteBudget = materializationByteBudget; + _selectionHash = DiffCursorCodec.CreateSelectionHash(leftDb, rightDb, includeContent); } internal List Records { get; } = []; @@ -1485,11 +1525,22 @@ internal DiffRecordPageCollector( internal void Add(DiffRowSchema schema, string side, DiffRow row) { + var identitySha256 = CreateDiffRecordIdentity(schema, row); + DiffCursorCodec.AppendSelectionRecord( + _selectionHash, + schema.Area, + side, + identitySha256); if (TotalCount >= _offset && Records.Count < _limit && !_materializationBudgetReached) { - var record = CreateDiffRecord(schema, side, row, _includeContent); + var record = CreateDiffRecord( + schema, + side, + row, + identitySha256, + _includeContent); if (_materializationByteBudget is null) { Records.Add(record); @@ -1508,11 +1559,23 @@ internal void Add(DiffRowSchema schema, string side, DiffRow row) TotalCount++; } + internal string CompleteSelectionFingerprint() + { + if (_selectionFingerprintCompleted) + throw new InvalidOperationException("diff selection fingerprint was already completed"); + + _selectionFingerprintCompleted = true; + return DiffCursorCodec.CompleteSelectionFingerprint(_selectionHash); + } + internal void AddMetadata(string area, string key, string? leftValue, string? rightValue) => Add( new DiffRowSchema(area, ["key", "left_value", "right_value"]), "changed", new DiffRow([key, leftValue, rightValue])); + + public void Dispose() + => _selectionHash.Dispose(); } private sealed record DiffRow( @@ -1550,5 +1613,7 @@ internal sealed class DiffCommandOptions public bool OffsetExplicit { get; init; } public int? MaxJsonBytes { get; init; } public string? Cursor { get; init; } + public string? CursorSelectionFingerprint { get; init; } + public bool EmitCursorMetadata { get; init; } = true; public string? ParseError { get; init; } } diff --git a/src/CodeIndex/Cli/DiffCursorCodec.cs b/src/CodeIndex/Cli/DiffCursorCodec.cs index eb1a10f3d..e0a3be1b6 100644 --- a/src/CodeIndex/Cli/DiffCursorCodec.cs +++ b/src/CodeIndex/Cli/DiffCursorCodec.cs @@ -11,15 +11,34 @@ internal static class DiffCursorCodec internal static string CreateSelectionFingerprint(string leftDb, string rightDb, bool includeContent) { - var material = string.Join( - "\n", - "diff-record-contract:v1", - leftDb, - rightDb, - includeContent ? "include-content" : "redacted"); - return HexEncoding.ToLowerHexString(SHA256.HashData(Encoding.UTF8.GetBytes(material))); + using var hash = CreateSelectionHash(leftDb, rightDb, includeContent); + return CompleteSelectionFingerprint(hash); } + internal static IncrementalHash CreateSelectionHash(string leftDb, string rightDb, bool includeContent) + { + var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + AppendSelectionPart(hash, "diff-record-selection:v1"); + AppendSelectionPart(hash, leftDb); + AppendSelectionPart(hash, rightDb); + AppendSelectionPart(hash, includeContent ? "include-content" : "redacted"); + return hash; + } + + internal static void AppendSelectionRecord( + IncrementalHash hash, + string area, + string side, + string identitySha256) + { + AppendSelectionPart(hash, area); + AppendSelectionPart(hash, side); + AppendSelectionPart(hash, identitySha256); + } + + internal static string CompleteSelectionFingerprint(IncrementalHash hash) + => HexEncoding.ToLowerHexString(hash.GetHashAndReset()); + internal static string Encode(int offset, string selectionFingerprint) { var payload = string.Create( @@ -30,11 +49,12 @@ internal static string Encode(int offset, string selectionFingerprint) internal static bool TryDecode( string cursor, - string expectedSelectionFingerprint, out int offset, + out string selectionFingerprint, out string error) { offset = 0; + selectionFingerprint = string.Empty; if (cursor.Length > MaxCursorLength || !cursor.StartsWith(Prefix, StringComparison.Ordinal) || !TryFromBase64Url(cursor[Prefix.Length..], out var payloadBytes)) @@ -55,11 +75,13 @@ internal static bool TryDecode( return false; } - var actualFingerprint = payload[(separator + 1)..]; - if (!FixedTimeEquals(actualFingerprint, expectedSelectionFingerprint)) + selectionFingerprint = payload[(separator + 1)..]; + if (selectionFingerprint.Length != SHA256.HashSizeInBytes * 2 + || selectionFingerprint.Any(character => !Uri.IsHexDigit(character))) { offset = 0; - error = "--cursor does not match the selected database pair or content policy; restart without --cursor"; + selectionFingerprint = string.Empty; + error = $"--cursor must be an opaque {Prefix} cursor returned by a prior detailed diff response"; return false; } @@ -67,7 +89,7 @@ internal static bool TryDecode( return true; } - private static bool FixedTimeEquals(string left, string right) + internal static bool FixedTimeEquals(string left, string right) { var leftBytes = Encoding.UTF8.GetBytes(left); var rightBytes = Encoding.UTF8.GetBytes(right); @@ -75,6 +97,15 @@ private static bool FixedTimeEquals(string left, string right) && CryptographicOperations.FixedTimeEquals(leftBytes, rightBytes); } + private static void AppendSelectionPart(IncrementalHash hash, string value) + { + var bytes = Encoding.UTF8.GetBytes(value); + Span length = stackalloc byte[sizeof(int)]; + System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(length, bytes.Length); + hash.AppendData(length); + hash.AppendData(bytes); + } + private static string ToBase64Url(byte[] bytes) => Convert.ToBase64String(bytes) .TrimEnd('=') diff --git a/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs b/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs index 9ff1e81e9..319452ecc 100644 --- a/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs +++ b/src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs @@ -291,7 +291,8 @@ private static ImportDestinationDeltaResult BuildImportDestinationDelta( detailed: true, cancellationToken, destinationDbPath, - archivePath); + archivePath, + emitCursorMetadata: false); var comparable = comparison.Status != "schema_mismatch"; return new ImportDestinationDeltaResult( DestinationExists: true, diff --git a/tests/CodeIndex.Tests/DiffCommandRunnerTests.cs b/tests/CodeIndex.Tests/DiffCommandRunnerTests.cs index 7e2b337cd..23939bc04 100644 --- a/tests/CodeIndex.Tests/DiffCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/DiffCommandRunnerTests.cs @@ -364,6 +364,22 @@ record => firstIdentities.Contains(record.GetProperty("identity_sha256").GetStri Assert.Equal(CommandExitCodes.UsageError, mismatchExitCode); using var mismatchDocument = JsonDocument.Parse(mismatchOutput); Assert.Equal("error", mismatchDocument.RootElement.GetProperty("status").GetString()); + + TestProjectHelper.InsertIndexedFile( + leftDb, + "src/AfterCursorMutation.cs", + "csharp", + "public static class AfterCursorMutation { }"); + var (staleExitCode, staleOutput) = RunWithCapturedOut( + [leftDb, rightDb, "--json", "--detailed", "--cursor", nextCursor!]); + + Assert.Equal(CommandExitCodes.UsageError, staleExitCode); + using var staleDocument = JsonDocument.Parse(staleOutput); + Assert.Equal("error", staleDocument.RootElement.GetProperty("status").GetString()); + Assert.Contains( + "no longer matches the selected database contents", + staleDocument.RootElement.GetProperty("message").GetString(), + StringComparison.Ordinal); } finally { diff --git a/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs b/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs index 29d4edb76..b02692143 100644 --- a/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs @@ -1243,6 +1243,10 @@ public void RunImport_DryRunReportsBoundedDestinationDeltaWithoutReplacingDb_Iss Assert.True(comparison.GetProperty("has_more").GetBoolean()); Assert.Equal(1, comparison.GetProperty("returned_count").GetInt32()); Assert.True(comparison.GetProperty("omitted_count").GetInt64() > 0); + Assert.False(comparison.TryGetProperty("current_cursor", out _)); + Assert.False(comparison.TryGetProperty("next_cursor", out _)); + Assert.False(comparison.TryGetProperty("replay", out _)); + Assert.Equal(1, comparison.GetProperty("next_offset").GetInt32()); Assert.DoesNotContain("public class Imported", stdout, StringComparison.Ordinal); Assert.DoesNotContain("public class Existing", stdout, StringComparison.Ordinal); } From 776d6e81bf7154c892dbacb609043294fbe77c1f Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 28 Jul 2026 21:35:58 +0900 Subject: [PATCH 4/5] Honor embedded and pretty diff bounds (#4859) --- src/CodeIndex/Cli/DiffCommandRunner.cs | 47 +++++-- .../CodeIndex.Tests/DiffCommandRunnerTests.cs | 121 ++++++++++++++++++ .../ExportImportCommandRunnerTests.cs | 1 + 3 files changed, 159 insertions(+), 10 deletions(-) diff --git a/src/CodeIndex/Cli/DiffCommandRunner.cs b/src/CodeIndex/Cli/DiffCommandRunner.cs index 60a56e5d0..017a6053e 100644 --- a/src/CodeIndex/Cli/DiffCommandRunner.cs +++ b/src/CodeIndex/Cli/DiffCommandRunner.cs @@ -63,7 +63,7 @@ public static int Run(string[] args, JsonSerializerOptions jsonOptions, Cancella try { - var result = CompareDatabases(options, cancellationToken); + var result = CompareDatabases(options, jsonOptions, cancellationToken); if (options.CursorSelectionFingerprint is { } cursorSelectionFingerprint && (result.SelectionFingerprint is not { } currentSelectionFingerprint || !DiffCursorCodec.FixedTimeEquals( @@ -156,7 +156,7 @@ internal static DiffJsonResult CompareDatabases( IncludeContent = includeContent, EmitCursorMetadata = emitCursorMetadata, }; - var result = CompareDatabases(options, cancellationToken); + var result = CompareDatabasesCore(options, cancellationToken, materializationJsonContext: null); return result with { LeftDb = leftDisplayPath is null @@ -168,7 +168,19 @@ internal static DiffJsonResult CompareDatabases( }; } - private static DiffJsonResult CompareDatabases(DiffCommandOptions options, CancellationToken cancellationToken) + internal static DiffJsonResult CompareDatabases( + DiffCommandOptions options, + JsonSerializerOptions jsonOptions, + CancellationToken cancellationToken) + => CompareDatabasesCore( + options, + cancellationToken, + CliJsonSerializerContextFactory.Create(jsonOptions)); + + private static DiffJsonResult CompareDatabasesCore( + DiffCommandOptions options, + CancellationToken cancellationToken, + CliJsonSerializerContext? materializationJsonContext) { cancellationToken.ThrowIfCancellationRequested(); var leftHeader = ReadHeader(options.LeftDb!); @@ -176,7 +188,12 @@ private static DiffJsonResult CompareDatabases(DiffCommandOptions options, Cance var rightHeader = ReadHeader(options.RightDb!); return leftHeader.SchemaVersion != rightHeader.SchemaVersion ? BuildSchemaMismatchDiff(leftHeader, rightHeader, options) - : BuildDiff(leftHeader, rightHeader, options, cancellationToken); + : BuildDiff( + leftHeader, + rightHeader, + options, + materializationJsonContext, + cancellationToken); } private const string FilePathRowsSql = "SELECT path FROM files ORDER BY path"; @@ -426,7 +443,12 @@ ORDER BY symbol_references.container_name_folded """; - private static DiffJsonResult BuildDiff(DiffDbHeader left, DiffDbHeader right, DiffCommandOptions options, CancellationToken cancellationToken) + private static DiffJsonResult BuildDiff( + DiffDbHeader left, + DiffDbHeader right, + DiffCommandOptions options, + CliJsonSerializerContext? materializationJsonContext, + CancellationToken cancellationToken) { var summary = new DiffSummaryJsonResult( left.FileCount, @@ -481,7 +503,8 @@ private static DiffJsonResult BuildDiff(DiffDbHeader left, DiffDbHeader right, D options.IncludeContent, options.Json ? options.MaxJsonBytes ?? DefaultDiffJsonBytes - : null) + : null, + materializationJsonContext) : null; if (collector is not null) { @@ -603,9 +626,9 @@ private static DiffJsonResult BuildDiff(DiffDbHeader left, DiffDbHeader right, D hasMore = collector.TotalCount > (long)options.Offset + records.Count; truncated = omittedCount > 0; truncationReason = truncated ? "limit_or_offset" : null; - selectionFingerprint = collector.CompleteSelectionFingerprint(); if (options.EmitCursorMetadata) { + selectionFingerprint = collector.CompleteSelectionFingerprint(); currentCursor = DiffCursorCodec.Encode(options.Offset, selectionFingerprint); if (hasMore && records.Count > 0) { @@ -706,7 +729,7 @@ private static DiffJsonResult BuildSchemaMismatchDiff(DiffDbHeader left, DiffDbH left.SchemaVersion, right.SchemaVersion, false); - var selectionFingerprint = options.Detailed + var selectionFingerprint = options.Detailed && options.EmitCursorMetadata ? DiffCursorCodec.CreateSelectionFingerprint(options.LeftDb!, options.RightDb!, options.IncludeContent) : null; var currentCursor = selectionFingerprint is null || !options.EmitCursorMetadata @@ -1500,6 +1523,7 @@ private sealed class DiffRecordPageCollector : IDisposable private readonly int _limit; private readonly bool _includeContent; private readonly int? _materializationByteBudget; + private readonly CliJsonSerializerContext _materializationJsonContext; private readonly IncrementalHash _selectionHash; private long _materializedRecordBytes; private bool _materializationBudgetReached; @@ -1511,12 +1535,15 @@ internal DiffRecordPageCollector( int offset, int limit, bool includeContent, - int? materializationByteBudget) + int? materializationByteBudget, + CliJsonSerializerContext? materializationJsonContext) { _offset = offset; _limit = limit; _includeContent = includeContent; _materializationByteBudget = materializationByteBudget; + _materializationJsonContext = materializationJsonContext + ?? CliJsonSerializerContext.Default; _selectionHash = DiffCursorCodec.CreateSelectionHash(leftDb, rightDb, includeContent); } @@ -1549,7 +1576,7 @@ internal void Add(DiffRowSchema schema, string side, DiffRow row) { var recordBytes = JsonSerializer.SerializeToUtf8Bytes( record, - CliJsonSerializerContext.Default.DiffRecordJsonResult).LongLength; + _materializationJsonContext.DiffRecordJsonResult).LongLength; Records.Add(record); _materializedRecordBytes = checked(_materializedRecordBytes + recordBytes); _materializationBudgetReached = diff --git a/tests/CodeIndex.Tests/DiffCommandRunnerTests.cs b/tests/CodeIndex.Tests/DiffCommandRunnerTests.cs index 23939bc04..b8f8e5e8d 100644 --- a/tests/CodeIndex.Tests/DiffCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/DiffCommandRunnerTests.cs @@ -388,6 +388,109 @@ record => firstIdentities.Contains(record.GetProperty("identity_sha256").GetStri } } + [Fact] + public void CompareDatabases_PrettyMaterializationUsesActiveJsonSize_Issue4859() + { + var leftRoot = TestProjectHelper.CreateTempProject("cdidx_diff_pretty_bound_left"); + var rightRoot = TestProjectHelper.CreateTempProject("cdidx_diff_pretty_bound_right"); + try + { + var leftDb = TestProjectHelper.CreateProjectDb(leftRoot); + var rightDb = TestProjectHelper.CreateProjectDb(rightRoot); + for (var i = 0; i < 30; i++) + { + TestProjectHelper.InsertIndexedFile( + leftDb, + $"src/PrettyBound{i:00}.cs", + "csharp", + $"public class PrettyBound{i:00} {{ public string Value => \"{new string('x', 128)}\"; }}"); + } + + var prettyOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + WriteIndented = true, + }; + const int byteBudget = DiffCommandRunner.MinDiffJsonBytes; + var complete = DiffCommandRunner.CompareDatabases( + new DiffCommandOptions + { + LeftDb = leftDb, + RightDb = rightDb, + Json = true, + Detailed = true, + Limit = 100, + MaxJsonBytes = DiffCommandRunner.MaxDiffJsonBytes, + }, + prettyOptions, + CancellationToken.None); + var expectedRetained = GetMaterializedRecordCount( + complete.Records ?? [], + byteBudget, + CliJsonSerializerContextFactory.Create(prettyOptions)); + var compactRetained = GetMaterializedRecordCount( + complete.Records ?? [], + byteBudget, + CliJsonSerializerContext.Default); + + var bounded = DiffCommandRunner.CompareDatabases( + new DiffCommandOptions + { + LeftDb = leftDb, + RightDb = rightDb, + Json = true, + Detailed = true, + Limit = 100, + MaxJsonBytes = byteBudget, + }, + prettyOptions, + CancellationToken.None); + + Assert.True(expectedRetained < complete.Records!.Count); + Assert.True(expectedRetained < compactRetained); + Assert.Equal(expectedRetained, bounded.Records!.Count); + } + finally + { + TestProjectHelper.DeleteDirectory(leftRoot); + TestProjectHelper.DeleteDirectory(rightRoot); + } + } + + [Fact] + public void CompareDatabases_EmbeddedSchemaMismatchOmitsCursorSelectionMetadata_Issue4859() + { + var leftRoot = TestProjectHelper.CreateTempProject("cdidx_diff_embedded_schema_left"); + var rightRoot = TestProjectHelper.CreateTempProject("cdidx_diff_embedded_schema_right"); + try + { + var leftDb = TestProjectHelper.CreateProjectDb(leftRoot); + var rightDb = TestProjectHelper.CreateProjectDb(rightRoot); + ExecuteNonQuery(rightDb, "PRAGMA user_version = 999"); + + var result = DiffCommandRunner.CompareDatabases( + leftDb, + rightDb, + limit: 10, + offset: 0, + detailed: true, + CancellationToken.None, + emitCursorMetadata: false); + + Assert.Equal("schema_mismatch", result.Status); + Assert.Null(result.SelectionFingerprint); + Assert.Null(result.CurrentCursor); + Assert.Null(result.NextCursor); + Assert.Null(result.Replay); + } + finally + { + TestProjectHelper.DeleteDirectory(leftRoot); + TestProjectHelper.DeleteDirectory(rightRoot); + } + } + [Fact] public void Run_DetailedJsonLimitZeroWithholdsNonAdvancingReplay_Issue4859() { @@ -1520,6 +1623,24 @@ private static JsonElement GetField(JsonElement record, string name) .EnumerateArray() .Where(field => field.GetProperty("name").GetString() == name)); + private static int GetMaterializedRecordCount( + List records, + int byteBudget, + CliJsonSerializerContext context) + { + long materializedBytes = 0; + for (var i = 0; i < records.Count; i++) + { + materializedBytes += JsonSerializer.SerializeToUtf8Bytes( + records[i], + context.DiffRecordJsonResult).LongLength; + if (materializedBytes > byteBudget) + return i + 1; + } + + return records.Count; + } + private (int ExitCode, string Output) RunWithCapturedOut(string[] args, CancellationToken cancellationToken = default) { var originalOut = Console.Out; diff --git a/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs b/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs index b02692143..58eb7135e 100644 --- a/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs @@ -1246,6 +1246,7 @@ public void RunImport_DryRunReportsBoundedDestinationDeltaWithoutReplacingDb_Iss Assert.False(comparison.TryGetProperty("current_cursor", out _)); Assert.False(comparison.TryGetProperty("next_cursor", out _)); Assert.False(comparison.TryGetProperty("replay", out _)); + Assert.False(comparison.TryGetProperty("selection_fingerprint", out _)); Assert.Equal(1, comparison.GetProperty("next_offset").GetInt32()); Assert.DoesNotContain("public class Imported", stdout, StringComparison.Ordinal); Assert.DoesNotContain("public class Existing", stdout, StringComparison.Ordinal); From 6553436f48539d31ad01e7b9004ea3872127e90e Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 28 Jul 2026 21:56:44 +0900 Subject: [PATCH 5/5] Fix diff byte budget edge cases (#4859) --- src/CodeIndex/Cli/DiffCommandOptionsParser.cs | 27 +++++----- src/CodeIndex/Cli/DiffCommandRunner.cs | 16 +++--- src/CodeIndex/Cli/DiffResultWriter.cs | 18 ++++++- .../CodeIndex.Tests/DiffCommandRunnerTests.cs | 54 +++++++++++++++++++ 4 files changed, 93 insertions(+), 22 deletions(-) diff --git a/src/CodeIndex/Cli/DiffCommandOptionsParser.cs b/src/CodeIndex/Cli/DiffCommandOptionsParser.cs index 033a05bab..750994f77 100644 --- a/src/CodeIndex/Cli/DiffCommandOptionsParser.cs +++ b/src/CodeIndex/Cli/DiffCommandOptionsParser.cs @@ -42,11 +42,11 @@ internal static DiffCommandOptions Parse(string[] args, int maxLimit) if (!int.TryParse(args[++i], out var parsedMaxJsonBytes) || parsedMaxJsonBytes < DiffCommandRunner.MinDiffJsonBytes) { - parseError = $"--max-json-bytes must be at least {DiffCommandRunner.MinDiffJsonBytes}"; + parseError ??= $"--max-json-bytes must be at least {DiffCommandRunner.MinDiffJsonBytes}"; } else if (parsedMaxJsonBytes > DiffCommandRunner.MaxDiffJsonBytes) { - parseError = $"--max-json-bytes must be less than or equal to {DiffCommandRunner.MaxDiffJsonBytes}"; + parseError ??= $"--max-json-bytes must be less than or equal to {DiffCommandRunner.MaxDiffJsonBytes}"; } else { @@ -54,48 +54,45 @@ internal static DiffCommandOptions Parse(string[] args, int maxLimit) } break; case "--max-json-bytes": - parseError = "--max-json-bytes requires a value"; + parseError ??= "--max-json-bytes requires a value"; break; case "--cursor" when i + 1 < args.Length: cursor = args[++i]; if (cursor.Length > DiffCursorCodec.MaxCursorLength) - parseError = $"--cursor must not exceed {DiffCursorCodec.MaxCursorLength} characters"; + parseError ??= $"--cursor must not exceed {DiffCursorCodec.MaxCursorLength} characters"; break; case "--cursor": - parseError = "--cursor requires a value"; + parseError ??= "--cursor requires a value"; break; case "--limit" when i + 1 < args.Length: if (!int.TryParse(args[++i], out limit) || limit < 0) - parseError = "--limit requires a non-negative integer"; + parseError ??= "--limit requires a non-negative integer"; else if (limit > maxLimit) { - parseError = $"--limit must be less than or equal to {maxLimit}"; + parseError ??= $"--limit must be less than or equal to {maxLimit}"; limit = DefaultLimit; } break; case "--limit": - parseError = "--limit requires a value"; + parseError ??= "--limit requires a value"; break; case "--offset" when i + 1 < args.Length: offsetExplicit = true; if (!int.TryParse(args[++i], out offset) || offset < 0) - parseError = "--offset requires a non-negative integer"; + parseError ??= "--offset requires a non-negative integer"; break; case "--offset": - parseError = "--offset requires a value"; + parseError ??= "--offset requires a value"; break; default: if (arg.StartsWith('-')) - parseError = $"diff does not support option: '{arg}'"; + parseError ??= $"diff does not support option: '{arg}'"; else if (dbs.Count >= 2) - parseError = $"diff accepts exactly two database paths; unexpected argument: '{arg}'"; + parseError ??= $"diff accepts exactly two database paths; unexpected argument: '{arg}'"; else dbs.Add(arg); break; } - - if (parseError is not null) - break; } if (parseError is null && dbs.Count != 2) diff --git a/src/CodeIndex/Cli/DiffCommandRunner.cs b/src/CodeIndex/Cli/DiffCommandRunner.cs index 017a6053e..79f0a7b45 100644 --- a/src/CodeIndex/Cli/DiffCommandRunner.cs +++ b/src/CodeIndex/Cli/DiffCommandRunner.cs @@ -28,6 +28,10 @@ public static int Run(string[] args, JsonSerializerOptions jsonOptions) public static int Run(string[] args, JsonSerializerOptions jsonOptions, CancellationToken cancellationToken) { var options = ParseArgs(args); + var errorJsonByteBudget = options.MaxJsonBytes + ?? (options.Json && options.Detailed && !options.SummaryOnly + ? DefaultDiffJsonBytes + : null); if (options.ShowHelp) { ConsoleUi.PrintUsage(); @@ -42,14 +46,14 @@ public static int Run(string[] args, JsonSerializerOptions jsonOptions, Cancella CommandExitCodes.UsageError, "Run `cdidx diff --help` to see the supported command shape.", CommandErrorCodes.UsageError, - options.MaxJsonBytes); + errorJsonByteBudget); var json = options.Json || options.SummaryOnly; var leftUriValidationExitCode = ValidateReadableDbFileUri( options.LeftDb!, json, jsonOptions, - options.MaxJsonBytes); + errorJsonByteBudget); if (leftUriValidationExitCode != null) return leftUriValidationExitCode.Value; @@ -57,7 +61,7 @@ public static int Run(string[] args, JsonSerializerOptions jsonOptions, Cancella options.RightDb!, json, jsonOptions, - options.MaxJsonBytes); + errorJsonByteBudget); if (rightUriValidationExitCode != null) return rightUriValidationExitCode.Value; @@ -77,7 +81,7 @@ public static int Run(string[] args, JsonSerializerOptions jsonOptions, Cancella CommandExitCodes.UsageError, "Rerun the detailed diff without --cursor to start from the current deterministic record sequence.", CommandErrorCodes.UsageError, - options.MaxJsonBytes); + errorJsonByteBudget); } var outputExitCode = DiffResultWriter.WriteResult(result, options, jsonOptions); @@ -97,7 +101,7 @@ public static int Run(string[] args, JsonSerializerOptions jsonOptions, Cancella CommandExitCodes.CancelledBySignal, "Retry the diff with a smaller database pair or after the cancelling operation completes.", CommandErrorCodes.Interrupted, - options.MaxJsonBytes); + errorJsonByteBudget); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or SqliteException or InvalidOperationException) { @@ -108,7 +112,7 @@ public static int Run(string[] args, JsonSerializerOptions jsonOptions, Cancella UnreadableExitCode, "Pass two readable CodeIndex SQLite database paths.", CommandErrorCodes.DbError, - options.MaxJsonBytes); + errorJsonByteBudget); } } diff --git a/src/CodeIndex/Cli/DiffResultWriter.cs b/src/CodeIndex/Cli/DiffResultWriter.cs index 9be3caeae..554e9a754 100644 --- a/src/CodeIndex/Cli/DiffResultWriter.cs +++ b/src/CodeIndex/Cli/DiffResultWriter.cs @@ -85,8 +85,24 @@ internal static string FormatDelta(long delta) var budget = options.MaxJsonBytes ?? DiffCommandRunner.DefaultDiffJsonBytes; var context = CliJsonSerializerContextFactory.Create(jsonOptions); var sourceRecords = result.Records ?? []; + var maximumCandidate = BuildBoundedCandidate( + result, + options, + sourceRecords, + sourceRecords.Count, + budget, + context); + var maximumJson = JsonSerializer.Serialize(maximumCandidate, context.DiffJsonResult); + if (FitsJsonLine(maximumJson, budget)) + { + Console.WriteLine(maximumJson); + return null; + } + var low = 0; - var high = GetSerializedRecordCeiling(sourceRecords, budget, context); + var high = Math.Min( + GetSerializedRecordCeiling(sourceRecords, budget, context), + sourceRecords.Count - 1); DiffJsonResult? bestResult = null; string? bestJson = null; while (low <= high) diff --git a/tests/CodeIndex.Tests/DiffCommandRunnerTests.cs b/tests/CodeIndex.Tests/DiffCommandRunnerTests.cs index b8f8e5e8d..3c18943dc 100644 --- a/tests/CodeIndex.Tests/DiffCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/DiffCommandRunnerTests.cs @@ -547,19 +547,34 @@ public void Run_JsonModesHonorMinimumExplicitBudget_Issue4859() [db, db, "--json", "--max-json-bytes", $"{byteBudget}"]); var (summaryExitCode, summaryOutput) = RunWithCapturedOut( [db, db, "--summary-only", "--max-json-bytes", $"{byteBudget}"]); + var oversizedUnsupportedOption = "--" + new string('x', byteBudget * 2); + var (parseErrorExitCode, parseErrorOutput) = RunWithCapturedOut( + [ + db, + db, + "--json", + oversizedUnsupportedOption, + "--max-json-bytes", + $"{byteBudget}", + ]); Assert.Equal(0, detailedExitCode); Assert.Equal(0, sampleExitCode); Assert.Equal(0, summaryExitCode); + Assert.Equal(CommandExitCodes.UsageError, parseErrorExitCode); Assert.InRange(Encoding.UTF8.GetByteCount(detailedOutput), 1, byteBudget); Assert.InRange(Encoding.UTF8.GetByteCount(sampleOutput), 1, byteBudget); Assert.InRange(Encoding.UTF8.GetByteCount(summaryOutput), 1, byteBudget); + Assert.InRange(Encoding.UTF8.GetByteCount(parseErrorOutput), 1, byteBudget); using var detailedDocument = JsonDocument.Parse(detailedOutput); using var sampleDocument = JsonDocument.Parse(sampleOutput); using var summaryDocument = JsonDocument.Parse(summaryOutput); + using var parseErrorDocument = JsonDocument.Parse(parseErrorOutput); Assert.Equal("identical", detailedDocument.RootElement.GetProperty("status").GetString()); Assert.Equal("identical", sampleDocument.RootElement.GetProperty("status").GetString()); Assert.Equal("identical", summaryDocument.RootElement.GetProperty("status").GetString()); + Assert.Equal("error", parseErrorDocument.RootElement.GetProperty("status").GetString()); + Assert.DoesNotContain(oversizedUnsupportedOption, parseErrorOutput, StringComparison.Ordinal); } finally { @@ -749,6 +764,45 @@ public void Run_DetailedJsonReportsOperationalMetadataDrift_Issue4357() Assert.True(GetField(drift, "right_value").GetProperty("redacted").GetBoolean()); Assert.DoesNotContain(Path.GetFullPath(leftRoot), output, StringComparison.Ordinal); Assert.DoesNotContain(Path.GetFullPath(rightRoot), output, StringComparison.Ordinal); + + var additionalOperationalKeys = new[] + { + "indexed_follow_symlinks_policy", + "indexed_head_commit", + "indexed_head_commit_branch", + "indexed_head_sha", + "indexed_head_branch", + "indexed_head_timestamp", + "commit_scoped_fresh_head_sha", + "workspace_path_case_sensitive", + DbContext.CdidxWriterVersionMetaKey, + }; + foreach (var key in additionalOperationalKeys) + { + SetMeta(leftDb, key, $"left-{key}"); + SetMeta(rightDb, key, $"right-{key}"); + } + + const int completePageBudget = 7_500; + var (boundedExitCode, boundedOutput) = RunWithCapturedOut( + [ + leftDb, + rightDb, + "--json", + "--detailed", + "--limit", + "100", + "--max-json-bytes", + $"{completePageBudget}", + ]); + + Assert.Equal(0, boundedExitCode); + Assert.InRange(Encoding.UTF8.GetByteCount(boundedOutput), 1, completePageBudget); + using var boundedDocument = JsonDocument.Parse(boundedOutput); + var boundedRoot = boundedDocument.RootElement; + Assert.Equal(10, boundedRoot.GetProperty("total_count").GetInt64()); + Assert.Equal(10, boundedRoot.GetProperty("returned_count").GetInt32()); + Assert.False(boundedRoot.GetProperty("truncated").GetBoolean()); } finally {