From 2c8a8c206dbb18553b45e7a8dc5887590a3cf3af Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 27 Jul 2026 15:55:30 +0900 Subject: [PATCH 1/2] Fix discovery JSON shape consistency (#4835) --- DEVELOPER_GUIDE.md | 4 +- changelog.d/unreleased/4835.fixed.md | 18 ++ .../Cli/JsonEnvelopeWrapper.Bounded.cs | 36 ++- .../Cli/QueryCommandRunner.Discovery.cs | 50 ++-- .../QueryCommandRunnerIssue4835Tests.cs | 220 ++++++++++++++++++ 5 files changed, 307 insertions(+), 21 deletions(-) create mode 100644 changelog.d/unreleased/4835.fixed.md create mode 100644 tests/CodeIndex.Tests/QueryCommandRunnerIssue4835Tests.cs diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 201e88b7d..afec09046 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1521,7 +1521,7 @@ For the AI agent search-rule template, see [AI Integration](USER_GUIDE.md#ai-int | Human-readable default | Query commands (`search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `excerpt`, `map`, `inspect`, `outline`, `suggestions`) default to **human-readable output**. | | `--json` | Emits JSON lines output, one JSON object per line, designed for easy parsing by AI agents. | | `definition --json` miss | A default-format definition lookup that finds no matching symbol emits the shared versioned `E018_QUERY_NOT_FOUND` command-error object and exits `2`, with or without `--body`; it never succeeds with empty stdout. Bounded-envelope controls move the object to `metadata.error` and keep `results` empty instead of projecting it as a location row. The object is preflighted against `--max-json-bytes`; an impossible cap returns a usage error without oversized stdout. `--count` still returns its structured zero-count object, and explicit location formats retain their existing format-specific empty-result output. | -| Raw discovery JSON shape | `symbols` and `files` preserve the selected flat shape for every result cardinality and under `--max-json-bytes`: zero-result NDJSON is an empty stream, `--json=array` is always an array, and byte-capped output omits whole trailing rows without changing the top-level type. Use `--format compact` or `--json-envelope` when truncation and freshness metadata must accompany the results. | +| Raw discovery JSON shape | `symbols` and `files` build each result row through the same DTO path for array, NDJSON, and envelope output. `symbols --json=array` therefore preserves `exact_index_available` just like NDJSON. Every cardinality and `--max-json-bytes` path keeps the selected flat shape: zero-result NDJSON is an empty stream, `--json=array` is always an array, and byte-capped output omits whole trailing rows without changing the top-level type. Bounded projections keep rows in `results`, pagination facts in `metadata`, and exact-query readiness in `metadata.response_context`; they never reuse a result row as response context. Use `--format compact` or `--json-envelope` when truncation and freshness metadata must accompany the results. | | Generated-code filtering metadata | DB-backed discovery `query_context` always reports `include_generated`, `generated_code_policy`, and `generated_file_filter_available`. The `files --count --json` and every JSON `map` summary (including `issue-drafts`) also report `generated_file_count_excluded` and `generated_file_count_excluded_authoritative`. The excluded count is `0` when generated files are included. For a legacy DB without `files.generated` when filtering is requested, the policy is `unavailable`, the count is `null`, and the authoritative/available flags are `false` rather than claiming that an unavailable filter ran; explicit `--include-generated` remains `include` with an authoritative excluded count of `0`. Byte-capped and uncapped raw discovery arrays retain SQLite trust diagnostics even when the query returns no result rows. | | Map scope, depth, and freshness | `map --depth ` applies path, language, test, generated-code, and exclusion filters before aggregating modules by the requested prefix depth. Scoped map output excludes the workspace-global decomposition plan. Workspace HEAD metadata is read in one query from the same SQLite snapshot as the map and remains explicit under `head_freshness`: `scope=workspace`, `indexed_head_source=latest_index` for the current successful index stamp (or `legacy_full_scan` only when it is the fallback), and `legacy_full_scan_head` for the separately labeled compatibility stamp. `issue-drafts` evaluates every scoped file for its thresholds, so `candidate_source=evaluated_scoped_candidates`, candidate counts, group totals, omitted counts, and `truncation.issue_draft_candidates` are candidate-based even though candidate details remain bounded; `truncation.largest_files` is a labeled compatibility alias only. | | `test-extractor` JSON | Machine-readable `test-extractor` success uses a versioned `{"api_version":"1","symbols":[...]}` envelope; the nested symbol objects retain their established property names. `--json` failures use the shared versioned command-error contract. | @@ -4684,7 +4684,7 @@ AI エージェント向け検索ルールのテンプレートについては | human-readable default | query command（`search`、`definition`、`references`、`callers`、`callees`、`symbols`、`files`、`excerpt`、`map`、`inspect`、`outline`、`suggestions`）は既定で**人間向け出力**です。 | | `--json` | JSON lines output（1 行 1 JSON object）に切り替えます。AI agent が容易に parse できるよう設計されています。 | | `definition --json` の未検出 | 既定 format の definition lookup で一致する symbol がない場合、`--body` の有無にかかわらず、共通の versioned `E018_QUERY_NOT_FOUND` command-error object を出力して終了コード `2` を返します。空の stdout のまま成功することはありません。bounded-envelope control の使用時は object を location row として projection せず `metadata.error` に移し、`results` は空のままにします。この object は `--max-json-bytes` に対して事前検査され、収まらない上限では oversized stdout を出さず usage error を返します。`--count` は引き続き構造化された 0 件 object を返し、明示的な location format も既存の format 固有の empty-result output を維持します。 | -| raw discovery JSON shape | `symbols` と `files` は、結果件数や `--max-json-bytes` の有無にかかわらず、選択した flat shape を維持します。0 件の NDJSON は空 stream、`--json=array` は常に array となり、byte cap 到達時は top-level type を変えずに末尾の完全な row を省略します。truncation / freshness metadata も結果と一緒に必要な場合は `--format compact` または `--json-envelope` を使用します。 | +| raw discovery JSON shape | `symbols` と `files` は、array、NDJSON、envelope の各出力で同じ DTO 経路から result row を構築します。そのため `symbols --json=array` も NDJSON と同様に `exact_index_available` を保持します。結果件数や `--max-json-bytes` の有無にかかわらず選択した flat shape を維持し、0 件の NDJSON は空 stream、`--json=array` は常に array となり、byte cap 到達時は top-level type を変えずに末尾の完全な row を省略します。bounded projection は row を `results`、pagination fact を `metadata`、exact-query readiness を `metadata.response_context` に保持し、result row を response context として再利用しません。truncation / freshness metadata も結果と一緒に必要な場合は `--format compact` または `--json-envelope` を使用します。 | | generated-code filtering metadata | DB-backed discovery の `query_context` は常に `include_generated`、`generated_code_policy`、`generated_file_filter_available` を返します。`files --count --json` と `issue-drafts` を含むすべての JSON `map` summary は、`generated_file_count_excluded` と `generated_file_count_excluded_authoritative` も返します。generated file を含める場合、除外数は `0` です。`files.generated` が無い legacy DB で filter が要求された場合、未実行の filter を実行済みと誤認させないよう、policy は `unavailable`、count は `null`、authoritative / available flag は `false` になります。明示的な `--include-generated` は `include` のままで、authoritative な除外数 `0` を返します。byte cap の有無にかかわらず、raw discovery array は query result row が 0 件でも SQLite trust diagnostics を維持します。 | | map の scope、depth、freshness | `map --depth ` は path、language、test、generated-code、除外条件を適用してから、指定した prefix depth で module を集計します。scope を絞った map output からは workspace 全体向けの decomposition plan を除外します。workspace HEAD metadata は map と同じ SQLite snapshot から 1 query で読み、`head_freshness` に `scope=workspace`、現在の成功 index stamp なら `indexed_head_source=latest_index`（fallback の場合だけ `legacy_full_scan`）、互換用 stamp は別名の `legacy_full_scan_head` として明示します。`issue-drafts` は scope 内の全 file を閾値評価するため、`candidate_source=evaluated_scoped_candidates`、candidate 件数、group 合計、省略数、`truncation.issue_draft_candidates` は candidate 基準になります。`truncation.largest_files` は明示的な互換 alias としてのみ残します。 | | `test-extractor` JSON | 機械可読な `test-extractor` success は versioned `{"api_version":"1","symbols":[...]}` envelope を使い、内側の symbol object は既存の property 名を維持します。`--json` failure は共通の versioned command-error 契約を使います。 | diff --git a/changelog.d/unreleased/4835.fixed.md b/changelog.d/unreleased/4835.fixed.md new file mode 100644 index 000000000..0df50c80b --- /dev/null +++ b/changelog.d/unreleased/4835.fixed.md @@ -0,0 +1,18 @@ +--- +category: fixed +issues: + - 4835 +affected: + - src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs + - src/CodeIndex/Cli/QueryCommandRunner.Discovery.cs + - tests/CodeIndex.Tests/QueryCommandRunnerIssue4835Tests.cs + - DEVELOPER_GUIDE.md +--- + +## English + +- **Projected `symbols` and `files` rows now stay consistent across JSON modes (#4835)** — array, NDJSON, and envelope output now reuse the same discovery-row serialization, preserve exact-query readiness, and keep projected rows in `results` with truthful count, truncation, and cursor metadata instead of relocating the first row to `metadata.response_context`. + +## 日本語 + +- **投影した `symbols` / `files` の row が JSON mode 間で一貫するようになりました (#4835)** — array、NDJSON、envelope 出力が同じ discovery-row serialization を再利用し、exact-query readiness を維持するようになりました。また、先頭 row を `metadata.response_context` に移動せず `results` に保持し、件数・truncation・cursor metadata を正しく返します。 diff --git a/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs b/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs index fe25d45ad..f9c2fa696 100644 --- a/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs +++ b/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs @@ -586,9 +586,13 @@ private static ResponseExtraction ExtractResponseItems(string command, JsonArray } if (command == "hotspots" && rawResults.FirstOrDefault() is JsonObject hotspotsPayload) return ExtractNestedCollection(hotspotsPayload, "hotspots"); - if (command == "symbols" && rawResults.FirstOrDefault() is JsonObject symbolsPayload) + if (command == "symbols" + && rawResults.FirstOrDefault() is JsonObject symbolsPayload + && symbolsPayload["symbols"] is JsonArray) return ExtractNestedCollection(symbolsPayload, "symbols"); - if (command == "files" && rawResults.FirstOrDefault() is JsonObject filesPayload) + if (command == "files" + && rawResults.FirstOrDefault() is JsonObject filesPayload + && filesPayload["files"] is JsonArray) return ExtractNestedCollection(filesPayload, "files"); if (command == "languages" && rawResults.FirstOrDefault() is JsonObject languagesPayload) return ExtractNestedCollection(languagesPayload, "languages"); @@ -622,6 +626,8 @@ private static ResponseExtraction ExtractResponseItems(string command, JsonArray mapPayload); } } + if (command is "symbols" or "files") + return ExtractDiscoveryRows(command, rawResults); if (rawResults.Count == 1 && rawResults[0] is JsonArray arrayPayload) { return new ResponseExtraction( @@ -641,6 +647,32 @@ private static ResponseExtraction ExtractResponseItems(string command, JsonArray return new ResponseExtraction(rows, null, null, null); } + private static ResponseExtraction ExtractDiscoveryRows(string command, JsonArray rawResults) + { + var rows = new JsonArray(); + JsonObject? context = null; + foreach (var result in rawResults) + { + if (result is JsonObject obj && IsJsonStreamTerminal(obj)) + continue; + rows.Add(result?.DeepClone()); + if (command != "symbols" || result is not JsonObject row) + continue; + + if (row.TryGetPropertyValue("exact_index_available", out var exactIndexAvailable)) + { + context ??= new JsonObject(); + context["exact_index_available"] = exactIndexAvailable?.DeepClone(); + } + if (row.TryGetPropertyValue("degraded_reason", out var degradedReason)) + { + context ??= new JsonObject(); + context["degraded_reason"] = degradedReason?.DeepClone(); + } + } + return new ResponseExtraction(rows, command, context, null); + } + private static ResponseExtraction ExtractNestedCollection(JsonObject payload, string collectionName) { var items = payload[collectionName] as JsonArray ?? []; diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Discovery.cs b/src/CodeIndex/Cli/QueryCommandRunner.Discovery.cs index 73fddb120..dbe74e9e2 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Discovery.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Discovery.cs @@ -182,6 +182,9 @@ public static int RunSymbols(string[] cmdArgs, JsonSerializerOptions jsonOptions : reader.SearchSymbols(symbolQueries, options.Limit, options.Kind, options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, options.Since, exact, visibilityFilters: options.VisibilityFilters, excludeVisibilityFilters: options.ExcludeVisibilityFilters, sortMode: options.SymbolSortMode, offset: JsonEnvelopeWrapper.GetBoundedResponseOffset("symbols")); var hasExactPredicate = exact && symbolQueries is { Count: > 0 }; var exactSignal = reader.GetSymbolsExactQuerySignal(options.Lang, options.PathPatterns, options.ExcludePaths, options.ExcludeTests, options.Since); + Func rowFactory = + result => ToSymbolDiscoveryJsonNode(result, jsonOptions, options.OutputFormat == OutputFormatCompact); + var rowExactSignal = hasExactPredicate ? exactSignal : (ExactQuerySignal?)null; var multiNameExactHint = symbolQueries != null && symbolQueries.Count > 1; var exactZeroHint = multiNameExactHint ? BuildExactZeroHint( @@ -211,14 +214,14 @@ public static int RunSymbols(string[] cmdArgs, JsonSerializerOptions jsonOptions results, totalCount: 0, fileCount: 0, - rowFactory: result => ToSymbolDiscoveryJsonNode(result, jsonOptions, options.OutputFormat == OutputFormatCompact), - exactSignal: hasExactPredicate ? exactSignal : null); + rowFactory, + exactSignal: rowExactSignal); return payloadExitCode == CommandExitCodes.Success ? ZeroResultExitCode(options) : payloadExitCode; } if (options.OutputFormat == OutputFormatJson) { if (options.JsonOutputFormat == JsonOutputFormatArray) - Console.WriteLine(SerializeQueryJson(results, CliJsonSerializerContextFactory.Create(jsonOptions).ListSymbolResult, jsonOptions)); + WriteDiscoveryJsonArray(results, rowFactory, rowExactSignal, jsonOptions); return ZeroResultExitCode(options); } if (TryWriteEmptyFormattedResult(options, jsonOptions)) @@ -247,8 +250,8 @@ public static int RunSymbols(string[] cmdArgs, JsonSerializerOptions jsonOptions "symbols", results, counts.Count, - result => ToSymbolDiscoveryJsonNode(result, jsonOptions, compact: false), - hasExactPredicate ? exactSignal : null); + rowFactory, + rowExactSignal); ndjsonTerminalLine = stream.TerminalLine; return stream.ExitCode; } @@ -267,8 +270,8 @@ public static int RunSymbols(string[] cmdArgs, JsonSerializerOptions jsonOptions results, counts.Count, counts.FileCount, - result => ToSymbolDiscoveryJsonNode(result, jsonOptions, options.OutputFormat == OutputFormatCompact), - hasExactPredicate ? exactSignal : null); + rowFactory, + rowExactSignal); } if (options.OutputFormat == OutputFormatLsp) @@ -290,7 +293,7 @@ public static int RunSymbols(string[] cmdArgs, JsonSerializerOptions jsonOptions { if (options.JsonOutputFormat == JsonOutputFormatArray) { - Console.WriteLine(SerializeQueryJson(results, CliJsonSerializerContextFactory.Create(jsonOptions).ListSymbolResult, jsonOptions)); + WriteDiscoveryJsonArray(results, rowFactory, rowExactSignal, jsonOptions); } else { @@ -470,6 +473,8 @@ public static int RunFiles(string[] cmdArgs, JsonSerializerOptions jsonOptions) options.Since, orderBySize: options.RawBytes, offset: JsonEnvelopeWrapper.GetBoundedResponseOffset("files")); + Func rowFactory = + result => ToFileDiscoveryJsonNode(result, jsonOptions, options.OutputFormat == OutputFormatCompact); if (results.Count == 0) { if (IsDiscoveryNdjson(options)) @@ -487,16 +492,12 @@ public static int RunFiles(string[] cmdArgs, JsonSerializerOptions jsonOptions) results, totalCount: 0, fileCount: 0, - rowFactory: result => ToFileDiscoveryJsonNode(result, jsonOptions, options.OutputFormat == OutputFormatCompact), + rowFactory, generatedFileCountExcluded: CountGeneratedFilesExcluded(reader, options, filesScope)); return payloadExitCode == CommandExitCodes.Success ? ZeroResultExitCode(options) : payloadExitCode; } if (options.JsonOutputFormat == JsonOutputFormatArray) - { - var emptyRows = new JsonArray(); - AddActiveSqliteDiagnostics(emptyRows); - Console.WriteLine(emptyRows.ToJsonString(EnsureJsonNodeSerializerOptions(jsonOptions))); - } + WriteDiscoveryJsonArray(results, rowFactory, exactSignal: null, jsonOptions); } else if (!options.Json) { @@ -517,7 +518,7 @@ public static int RunFiles(string[] cmdArgs, JsonSerializerOptions jsonOptions) "files", results, counts.Count, - result => ToFileDiscoveryJsonNode(result, jsonOptions, compact: false)); + rowFactory); ndjsonTerminalLine = stream.TerminalLine; return stream.ExitCode; } @@ -534,7 +535,7 @@ public static int RunFiles(string[] cmdArgs, JsonSerializerOptions jsonOptions) results, counts.Count, counts.FileCount, - result => ToFileDiscoveryJsonNode(result, jsonOptions, options.OutputFormat == OutputFormatCompact), + rowFactory, generatedFileCountExcluded: CountGeneratedFilesExcluded(reader, options, filesScope)); } @@ -543,7 +544,7 @@ public static int RunFiles(string[] cmdArgs, JsonSerializerOptions jsonOptions) var context = CliJsonSerializerContextFactory.Create(jsonOptions); if (options.JsonOutputFormat == JsonOutputFormatArray) { - Console.WriteLine(SerializeQueryJson(results, context.ListFileResult, jsonOptions)); + WriteDiscoveryJsonArray(results, rowFactory, exactSignal: null, jsonOptions); } else { @@ -1004,6 +1005,21 @@ private static JsonArray BuildDiscoveryRows( return rows; } + private static void WriteDiscoveryJsonArray( + IReadOnlyList results, + Func rowFactory, + ExactQuerySignal? exactSignal, + JsonSerializerOptions jsonOptions) + { + var rows = BuildDiscoveryRows( + results, + results.Count, + rowFactory, + exactSignal, + addActiveSqliteDiagnostics: true); + Console.WriteLine(rows.ToJsonString(EnsureJsonNodeSerializerOptions(jsonOptions))); + } + private static JsonNode? BuildDiscoveryRow( T result, Func rowFactory, diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerIssue4835Tests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerIssue4835Tests.cs new file mode 100644 index 000000000..6987769e1 --- /dev/null +++ b/tests/CodeIndex.Tests/QueryCommandRunnerIssue4835Tests.cs @@ -0,0 +1,220 @@ +using System.Text.Json; +using CodeIndex.Cli; + +namespace CodeIndex.Tests; + +[Collection("Console sensitive")] +public sealed class QueryCommandRunnerIssue4835Tests +{ + private readonly JsonSerializerOptions _jsonOptions = ProgramRunner.CreateDefaultJsonOptions(); + + [Theory] + [InlineData("symbols")] + [InlineData("files")] + public void DiscoveryRows_AgreeAcrossArrayNdjsonAndProjectedEnvelope_Issue4835(string command) + { + var projectRoot = TestProjectHelper.CreateTempProject($"discovery_json_shapes_4835_{command}"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + for (var index = 1; index <= 3; index++) + { + TestProjectHelper.InsertIndexedFile( + dbPath, + $"src/SharedType{index}.cs", + "csharp", + "public sealed class SharedType { }\n"); + } + + var baseArgs = BuildQueryArgs(command, dbPath, "SharedType"); + var (ndjsonExitCode, ndjsonStdout, ndjsonStderr) = CaptureConsole(() => + ProgramRunner.Run([.. baseArgs, "--json"], _jsonOptions, "1.0.0-test")); + + Assert.Equal(CommandExitCodes.Success, ndjsonExitCode); + Assert.Equal(string.Empty, ndjsonStderr); + var ndjsonRecords = ParseNdjson(ndjsonStdout); + var terminal = ndjsonRecords[^1]; + var ndjsonRows = ndjsonRecords[..^1]; + Assert.Equal(2, ndjsonRows.Length); + Assert.True(terminal.GetProperty("terminal_record").GetBoolean()); + Assert.Equal(2, terminal.GetProperty("count").GetInt32()); + Assert.Equal(3, terminal.GetProperty("total_count").GetInt32()); + Assert.True(terminal.GetProperty("truncated").GetBoolean()); + Assert.True(terminal.GetProperty("has_more").GetBoolean()); + + var (arrayExitCode, arrayStdout, arrayStderr) = CaptureConsole(() => + ProgramRunner.Run([.. baseArgs, "--json=array"], _jsonOptions, "1.0.0-test")); + + Assert.Equal(CommandExitCodes.Success, arrayExitCode); + Assert.Equal(string.Empty, arrayStderr); + using var arrayDocument = JsonDocument.Parse(arrayStdout); + var arrayRows = arrayDocument.RootElement.EnumerateArray().Select(row => row.Clone()).ToArray(); + Assert.Equal(ndjsonRows.Select(row => row.GetRawText()), arrayRows.Select(row => row.GetRawText())); + bool? expectedExactIndexAvailable = null; + if (command == "symbols") + { + expectedExactIndexAvailable = ndjsonRows[0].GetProperty("exact_index_available").GetBoolean(); + Assert.All( + ndjsonRows, + row => Assert.Equal(expectedExactIndexAvailable, row.GetProperty("exact_index_available").GetBoolean())); + Assert.All( + arrayRows, + row => Assert.Equal(expectedExactIndexAvailable, row.GetProperty("exact_index_available").GetBoolean())); + } + + var (envelopeExitCode, envelopeStdout, envelopeStderr) = CaptureConsole(() => + ProgramRunner.Run([.. baseArgs, "--json-envelope"], _jsonOptions, "1.0.0-test")); + + Assert.Equal(CommandExitCodes.Success, envelopeExitCode); + Assert.Equal(string.Empty, envelopeStderr); + using var envelopeDocument = JsonDocument.Parse(envelopeStdout); + var envelopeRows = envelopeDocument.RootElement.GetProperty("results") + .EnumerateArray() + .Select(row => row.Clone()) + .ToArray(); + Assert.Equal(ndjsonRows.Select(row => row.GetRawText()), envelopeRows.Select(row => row.GetRawText())); + var envelopeMetadata = envelopeDocument.RootElement.GetProperty("metadata"); + Assert.Equal(terminal.GetProperty("count").GetInt32(), envelopeMetadata.GetProperty("result_count").GetInt32()); + Assert.Equal( + terminal.GetProperty("total_count").GetInt32(), + envelopeMetadata.GetProperty("stream_terminal").GetProperty("total_count").GetInt32()); + Assert.Equal( + terminal.GetProperty("truncated").GetBoolean(), + envelopeMetadata.GetProperty("stream_terminal").GetProperty("truncated").GetBoolean()); + + var fields = command == "symbols" ? "name,path" : "path,lang"; + var projectedArgs = baseArgs.Concat(["--json-envelope", "--fields", fields]).ToArray(); + var (projectedExitCode, projectedStdout, projectedStderr) = CaptureConsole(() => + ProgramRunner.Run(projectedArgs, _jsonOptions, "1.0.0-test")); + + Assert.Equal(CommandExitCodes.Success, projectedExitCode); + Assert.Equal(string.Empty, projectedStderr); + using var projectedDocument = JsonDocument.Parse(projectedStdout); + var projectedMetadata = projectedDocument.RootElement.GetProperty("metadata"); + AssertMetadataMatchesTerminal(projectedMetadata, terminal, command); + var expectedFields = fields.Split(','); + var projectedRows = projectedDocument.RootElement.GetProperty("results").EnumerateArray().ToArray(); + Assert.Equal(2, projectedRows.Length); + Assert.All( + projectedRows, + row => Assert.Equal(expectedFields, row.EnumerateObject().Select(property => property.Name))); + AssertResponseContextContainsNoRow(projectedMetadata); + if (command == "symbols") + { + Assert.Equal( + expectedExactIndexAvailable, + projectedMetadata.GetProperty("response_context").GetProperty("exact_index_available").GetBoolean()); + } + + var cursor = Assert.IsType(projectedMetadata.GetProperty("next_cursor").GetString()); + var (nextExitCode, nextStdout, nextStderr) = CaptureConsole(() => + ProgramRunner.Run([.. projectedArgs, "--cursor", cursor], _jsonOptions, "1.0.0-test")); + + Assert.Equal(CommandExitCodes.Success, nextExitCode); + Assert.Equal(string.Empty, nextStderr); + using var nextDocument = JsonDocument.Parse(nextStdout); + var nextMetadata = nextDocument.RootElement.GetProperty("metadata"); + Assert.Single(nextDocument.RootElement.GetProperty("results").EnumerateArray()); + Assert.Equal(1, nextMetadata.GetProperty("returned_count").GetInt32()); + Assert.Equal(3, nextMetadata.GetProperty("total_count").GetInt32()); + Assert.Equal(2, nextMetadata.GetProperty("cursor_offset").GetInt32()); + Assert.False(nextMetadata.GetProperty("has_more").GetBoolean()); + Assert.Equal(JsonValueKind.Null, nextMetadata.GetProperty("next_cursor").ValueKind); + AssertResponseContextContainsNoRow(nextMetadata); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + + [Theory] + [InlineData("symbols")] + [InlineData("files")] + public void DiscoveryRows_EmptyShapesRemainGenuinelyEmpty_Issue4835(string command) + { + var projectRoot = TestProjectHelper.CreateTempProject($"discovery_json_empty_4835_{command}"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + TestProjectHelper.InsertIndexedFile( + dbPath, + "src/SharedType.cs", + "csharp", + "public sealed class SharedType { }\n"); + + var baseArgs = BuildQueryArgs(command, dbPath, "Missing4835"); + var (ndjsonExitCode, ndjsonStdout, ndjsonStderr) = CaptureConsole(() => + ProgramRunner.Run([.. baseArgs, "--json"], _jsonOptions, "1.0.0-test")); + + Assert.Equal(CommandExitCodes.Success, ndjsonExitCode); + Assert.Equal(string.Empty, ndjsonStdout); + Assert.Equal(string.Empty, ndjsonStderr); + + var (arrayExitCode, arrayStdout, arrayStderr) = CaptureConsole(() => + ProgramRunner.Run([.. baseArgs, "--json=array"], _jsonOptions, "1.0.0-test")); + + Assert.Equal(CommandExitCodes.Success, arrayExitCode); + Assert.Equal(string.Empty, arrayStderr); + using (var arrayDocument = JsonDocument.Parse(arrayStdout)) + Assert.Empty(arrayDocument.RootElement.EnumerateArray()); + + var fields = command == "symbols" ? "name,path" : "path,lang"; + var (envelopeExitCode, envelopeStdout, envelopeStderr) = CaptureConsole(() => + ProgramRunner.Run( + [.. baseArgs, "--json-envelope", "--fields", fields], + _jsonOptions, + "1.0.0-test")); + + Assert.Equal(CommandExitCodes.Success, envelopeExitCode); + Assert.Equal(string.Empty, envelopeStderr); + using var envelopeDocument = JsonDocument.Parse(envelopeStdout); + var metadata = envelopeDocument.RootElement.GetProperty("metadata"); + Assert.Empty(envelopeDocument.RootElement.GetProperty("results").EnumerateArray()); + Assert.Equal(0, metadata.GetProperty("returned_count").GetInt32()); + Assert.Equal(0, metadata.GetProperty("total_count").GetInt32()); + Assert.Equal(0, metadata.GetProperty("omitted_count").GetInt32()); + Assert.False(metadata.GetProperty("truncated").GetBoolean()); + Assert.False(metadata.GetProperty("has_more").GetBoolean()); + Assert.Equal(JsonValueKind.Null, metadata.GetProperty("next_cursor").ValueKind); + AssertResponseContextContainsNoRow(metadata); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + + private static string[] BuildQueryArgs(string command, string dbPath, string query) + => command == "symbols" + ? [command, query, "--db", dbPath, "--exact-name", "--kind", "class", "--limit", "2"] + : [command, query, "--db", dbPath, "--limit", "2"]; + + private static void AssertMetadataMatchesTerminal(JsonElement metadata, JsonElement terminal, string command) + { + Assert.Equal(command, metadata.GetProperty("primary_collection").GetString()); + Assert.Equal(terminal.GetProperty("count").GetInt32(), metadata.GetProperty("returned_count").GetInt32()); + Assert.Equal(terminal.GetProperty("total_count").GetInt32(), metadata.GetProperty("total_count").GetInt32()); + Assert.Equal(terminal.GetProperty("truncated").GetBoolean(), metadata.GetProperty("truncated").GetBoolean()); + Assert.Equal(terminal.GetProperty("has_more").GetBoolean(), metadata.GetProperty("has_more").GetBoolean()); + Assert.False(string.IsNullOrWhiteSpace(metadata.GetProperty("next_cursor").GetString())); + } + + private static void AssertResponseContextContainsNoRow(JsonElement metadata) + { + if (!metadata.TryGetProperty("response_context", out var responseContext)) + return; + Assert.False(responseContext.TryGetProperty("name", out _)); + Assert.False(responseContext.TryGetProperty("path", out _)); + Assert.False(responseContext.TryGetProperty("symbol_id", out _)); + Assert.False(responseContext.TryGetProperty("checksum", out _)); + } + + private static JsonElement[] ParseNdjson(string stdout) + => stdout.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries) + .Select(line => JsonDocument.Parse(line).RootElement.Clone()) + .ToArray(); + + private static (int ExitCode, string Stdout, string Stderr) CaptureConsole(Func action) + => ConsoleCapture.Capture(action); +} From f55e4a2b232fb14363aafc6470352a6e97b1a37a Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 27 Jul 2026 16:12:37 +0900 Subject: [PATCH 2/2] Address JSON review findings (#4835) --- .../Cli/JsonEnvelopeWrapper.Bounded.cs | 38 +++++++++++++++++-- .../QueryCommandRunnerIssue4835Tests.cs | 29 ++++++++++++++ 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs b/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs index f9c2fa696..eed13127b 100644 --- a/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs +++ b/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs @@ -256,6 +256,8 @@ private static int RunBoundedResponse( reportedTotalCount, executionContext.ReportedTotalCountAuthoritative) : ResolveTotalCount(command, args, runInner, extraction, availableItems.Count, controls.Offset, streamTerminal); + if (count.Context is not null) + extraction = extraction with { Context = MergeResponseContexts(extraction.Context, count.Context) }; var totalCount = Math.Max(count.TotalCount, controls.Offset + pageItems.Count); var totalAuthoritative = count.Authoritative; var completedSnapshot = SafeReadResponseSnapshot(resolvedDbPath, dbPathExplicit, appVersion); @@ -588,11 +590,11 @@ private static ResponseExtraction ExtractResponseItems(string command, JsonArray return ExtractNestedCollection(hotspotsPayload, "hotspots"); if (command == "symbols" && rawResults.FirstOrDefault() is JsonObject symbolsPayload - && symbolsPayload["symbols"] is JsonArray) + && (symbolsPayload["symbols"] is JsonArray || ReadOptionalBool(symbolsPayload, "summary_only"))) return ExtractNestedCollection(symbolsPayload, "symbols"); if (command == "files" && rawResults.FirstOrDefault() is JsonObject filesPayload - && filesPayload["files"] is JsonArray) + && (filesPayload["files"] is JsonArray || ReadOptionalBool(filesPayload, "summary_only"))) return ExtractNestedCollection(filesPayload, "files"); if (command == "languages" && rawResults.FirstOrDefault() is JsonObject languagesPayload) return ExtractNestedCollection(languagesPayload, "languages"); @@ -673,6 +675,14 @@ private static ResponseExtraction ExtractDiscoveryRows(string command, JsonArray return new ResponseExtraction(rows, command, context, null); } + private static JsonObject MergeResponseContexts(JsonObject? existing, JsonObject additional) + { + var merged = existing is null ? new JsonObject() : (JsonObject)existing.DeepClone(); + foreach (var property in additional) + merged[property.Key] = property.Value?.DeepClone(); + return merged; + } + private static ResponseExtraction ExtractNestedCollection(JsonObject payload, string collectionName) { var items = payload[collectionName] as JsonArray ?? []; @@ -809,7 +819,7 @@ private static ResponseCount ResolveTotalCount( && !ReadOptionalBool(countPayload, "degraded") && ReadOptionalBool(countPayload, "graph_table_available", defaultValue: true) && ReadOptionalBool(countPayload, "hotspot_family_ready", defaultValue: true); - return new ResponseCount(total, authoritative); + return new ResponseCount(total, authoritative, ExtractCountResponseContext(command, countPayload)); } catch { @@ -834,6 +844,23 @@ private static bool TryReadBool(JsonObject obj, string propertyName, out bool va private static bool ReadOptionalBool(JsonObject obj, string propertyName, bool defaultValue = false) => TryReadBool(obj, propertyName, out var value) ? value : defaultValue; + private static JsonObject? ExtractCountResponseContext(string command, JsonObject countPayload) + { + if (command != "symbols" + || !countPayload.TryGetPropertyValue("exact_index_available", out var exactIndexAvailable)) + { + return null; + } + + var context = new JsonObject + { + ["exact_index_available"] = exactIndexAvailable?.DeepClone(), + }; + if (countPayload.TryGetPropertyValue("degraded_reason", out var degradedReason)) + context["degraded_reason"] = degradedReason?.DeepClone(); + return context; + } + private static string? ReadString(JsonObject? obj, string propertyName) => obj?[propertyName] is JsonValue value && value.TryGetValue(out var text) ? text @@ -1345,7 +1372,10 @@ private sealed record ResponseExtraction( JsonObject? Context, JsonObject? SourcePayload); - private readonly record struct ResponseCount(int TotalCount, bool Authoritative); + private readonly record struct ResponseCount( + int TotalCount, + bool Authoritative, + JsonObject? Context = null); private readonly record struct ResponseSnapshot( string GenerationFingerprint, diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerIssue4835Tests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerIssue4835Tests.cs index 6987769e1..6184f2c8b 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerIssue4835Tests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerIssue4835Tests.cs @@ -121,6 +121,23 @@ public void DiscoveryRows_AgreeAcrossArrayNdjsonAndProjectedEnvelope_Issue4835(s Assert.False(nextMetadata.GetProperty("has_more").GetBoolean()); Assert.Equal(JsonValueKind.Null, nextMetadata.GetProperty("next_cursor").ValueKind); AssertResponseContextContainsNoRow(nextMetadata); + + var (summaryExitCode, summaryStdout, summaryStderr) = CaptureConsole(() => + ProgramRunner.Run( + [.. baseArgs, "--json-envelope", "--summary-only", "--fields", fields], + _jsonOptions, + "1.0.0-test")); + + Assert.Equal(CommandExitCodes.Success, summaryExitCode); + Assert.Equal(string.Empty, summaryStderr); + using var summaryDocument = JsonDocument.Parse(summaryStdout); + var summaryMetadata = summaryDocument.RootElement.GetProperty("metadata"); + Assert.Empty(summaryDocument.RootElement.GetProperty("results").EnumerateArray()); + Assert.Equal(0, summaryMetadata.GetProperty("returned_count").GetInt32()); + Assert.Equal(3, summaryMetadata.GetProperty("total_count").GetInt32()); + Assert.False(summaryMetadata.GetProperty("has_more").GetBoolean()); + Assert.Equal(JsonValueKind.Null, summaryMetadata.GetProperty("next_cursor").ValueKind); + AssertResponseContextContainsNoRow(summaryMetadata); } finally { @@ -178,6 +195,18 @@ public void DiscoveryRows_EmptyShapesRemainGenuinelyEmpty_Issue4835(string comma Assert.False(metadata.GetProperty("has_more").GetBoolean()); Assert.Equal(JsonValueKind.Null, metadata.GetProperty("next_cursor").ValueKind); AssertResponseContextContainsNoRow(metadata); + if (command == "symbols") + { + var (countExitCode, countStdout, countStderr) = CaptureConsole(() => + ProgramRunner.Run([.. baseArgs, "--count", "--json"], _jsonOptions, "1.0.0-test")); + + Assert.Equal(CommandExitCodes.Success, countExitCode); + Assert.Equal(string.Empty, countStderr); + using var countDocument = JsonDocument.Parse(countStdout); + Assert.Equal( + countDocument.RootElement.GetProperty("exact_index_available").GetBoolean(), + metadata.GetProperty("response_context").GetProperty("exact_index_available").GetBoolean()); + } } finally {