From e98cce276f272515badf07865bf3b08b3cfff3e7 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 28 Jul 2026 12:08:20 +0900 Subject: [PATCH 1/3] Fix C# semantic token classification across surfaces (#4852) --- DEVELOPER_GUIDE.md | 4 +- README.md | 4 +- TESTING_GUIDE.md | 2 + changelog.d/unreleased/4852.fixed.md | 26 + .../Cli/QueryCommandRunner.Excerpt.cs | 54 +- .../Cli/QueryCommandRunner.Inspect.cs | 2 +- .../Database/DbReader.FilesStatus.cs | 19 + .../Lsp/LspServer.NavigationFeatures.cs | 36 +- src/CodeIndex/Lsp/LspServer.SemanticTokens.cs | 208 +--- .../CSharpSemanticTokenClassifier.cs | 928 ++++++++++++++++++ tests/CodeIndex.Tests/LspServerTests.cs | 76 ++ .../QueryCommandRunnerSearchTests.cs | 104 ++ tests/CodeIndex.Tests/golden/excerpt.json | 48 +- 13 files changed, 1273 insertions(+), 238 deletions(-) create mode 100644 changelog.d/unreleased/4852.fixed.md create mode 100644 src/CodeIndex/Semantics/CSharpSemanticTokenClassifier.cs diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 2d416f8780..638fdf668f 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1651,7 +1651,7 @@ When the match line falls inside an indexed symbol range, `search --json` and MC `find --json` remains line-delimited for repeated matches and adds bounded match-span/truncation metadata to each row: `length` reports the 1-based `column` span length, `original_line_length` reports the source line length before any line-width clamp, and `snippet_truncation_context.line_count` / `char_counts` / `total_chars` / optional `reason` describe snippet clamping. `reason` is `line_width` when `--max-line-width` elides one or more snippet lines. -`excerpt --json` includes `semantic_tokens`, a lightweight range list with 1-based source start/end positions, token `type`, and `modifiers`, so IDE and LLM clients can render or post-process excerpt spans without reparsing the raw `content` string. `semantic_token_coordinate_space` is `source`; when `--max-line-width` clamps returned content, `content_line_spans` maps each returned content line and visible content-column span back to the matching source line and source-column span, while clamp markers remain unmapped and are not emitted as semantic tokens. Excerpt rows also expose `requested_start_line`, `requested_end_line`, `effective_start_line`, `effective_end_line`, `content_truncation_reasons`, and optional `content_recovery` so clients can tell when `--max-line-width` caused `line_width_cap` and replay the omitted text. `content_recovery` and the matching `body_content_recovery` use `argv` as their primary machine-readable contract; the array begins with the current apphost or `dotnet` plus the running assembly, so recovery stays on the same build and database contract. Root-level paths beginning with `-` are preceded by the supported `--` end-of-options marker so the recovery CLI does not parse them as options. The legacy `command` is display-only, accompanied by `command_display_only: true` and `command_shell` (`posix-sh` or `powershell`), and is escaped for that declared shell. MCP emits the equivalent camelCase field names. Body-bearing JSON rows also use matching `body_requested_*`, `body_effective_*`, and `body_content_truncation_reasons` fields; body reasons include `body_line_cap` for snippet/body line caps and `body_byte_cap` for definition body byte caps. +`excerpt --json` includes `semantic_tokens`, a lightweight range list with 1-based source start/end positions, token `type`, and `modifiers`, so IDE and LLM clients can render or post-process excerpt spans without reparsing the raw `content` string. C# excerpts and LSP `textDocument/semanticTokens/full` share the same source classifier for keywords/modifiers, namespaces and types, methods and properties, parameters, variables and fields, and declaration modifiers; excerpt classification uses the full indexed source before filtering to the visible range, so a narrow or clamped excerpt retains declaration context. Excerpt range mapping and LSP delta encoding only translate coordinates and do not choose semantic kinds. `semantic_token_coordinate_space` is `source`; when `--max-line-width` clamps returned content, `content_line_spans` maps each returned content line and visible content-column span back to the matching source line and source-column span, while clamp markers remain unmapped and are not emitted as semantic tokens. Excerpt rows also expose `requested_start_line`, `requested_end_line`, `effective_start_line`, `effective_end_line`, `content_truncation_reasons`, and optional `content_recovery` so clients can tell when `--max-line-width` caused `line_width_cap` and replay the omitted text. `content_recovery` and the matching `body_content_recovery` use `argv` as their primary machine-readable contract; the array begins with the current apphost or `dotnet` plus the running assembly, so recovery stays on the same build and database contract. Root-level paths beginning with `-` are preceded by the supported `--` end-of-options marker so the recovery CLI does not parse them as options. The legacy `command` is display-only, accompanied by `command_display_only: true` and `command_shell` (`posix-sh` or `powershell`), and is escaped for that declared shell. MCP emits the equivalent camelCase field names. Body-bearing JSON rows also use matching `body_requested_*`, `body_effective_*`, and `body_content_truncation_reasons` fields; body reasons include `body_line_cap` for snippet/body line caps and `body_byte_cap` for definition body byte caps. `inspect` and MCP `analyze_symbol` bundle the primary definition, nearby symbols from the same file, references, callers, callees, file metadata, workspace freshness/git metadata, and graph-support metadata into one response. When those bundled graph sections actually depend on SQL-backed reads, the payload also mirrors `sql_graph_contract_ready` / `sql_graph_contract_degraded_reason` (plus the existing camelCase aliases on MCP responses); mixed-language bundles that only return C# / JS / etc. graph rows omit the SQL trust signal entirely. This is intended for symbol-oriented AI workflows that would otherwise need several back-to-back calls. Call graph sections remain language-aware: for unsupported languages, clients can now distinguish "unsupported" from "no hits" via `graphSupported` / `graphSupportReason`, and should prefer `search` instead of assuming graph data will exist. @@ -4887,7 +4887,7 @@ literal-safe な `search` query は reader 層で FTS5 sanitization 前に 1000 `find --json` は繰り返し一致でも line-delimited のまま維持し、各 row に bounded な match span / truncation metadata を追加します。`length` は 1-based の `column` から始まる一致長、`original_line_length` は行幅クランプ前のソース行長、`snippet_truncation_context.line_count` / `char_counts` / `total_chars` / 任意の `reason` は snippet クランプを表します。`--max-line-width` によって snippet 行が省略された場合、`reason` は `line_width` になります。 -`excerpt --json` は 1-based の source 開始/終了位置、token `type`、`modifiers` を持つ軽量 range list の `semantic_tokens` を返すため、IDE や LLM クライアントは生の `content` 文字列を再パースせずに抜粋範囲を描画・後処理できます。`semantic_token_coordinate_space` は `source` です。`--max-line-width` で返却内容がクランプされた場合、`content_line_spans` は返却 content 行と可視 content column span を、対応する source 行と source column span に対応付けます。clamp marker は未対応領域として扱い、semantic token には含めません。excerpt row は `requested_start_line`、`requested_end_line`、`effective_start_line`、`effective_end_line`、`content_truncation_reasons`、任意の `content_recovery` も返すため、`--max-line-width` による `line_width_cap` を検出して省略部分を再取得できます。`content_recovery` と対応する `body_content_recovery` では、`argv` が一次的な機械可読契約です。この配列は現在の apphost、または `dotnet` と実行中 assembly の組から始まるため、同じ build と database contract で復旧できます。先頭が `-` の root-level path の前には、対応済みの `--` end-of-options marker を置き、復旧 CLI が option として解釈しないようにします。従来の `command` は表示専用で、`command_display_only: true` と `command_shell`（`posix-sh` または `powershell`）を伴い、宣言した shell 向けに escape されます。MCP では同等の camelCase field 名を返します。body を持つ JSON row は対応する `body_requested_*`、`body_effective_*`、`body_content_truncation_reasons` も返します。body reason には snippet/body 行数上限の `body_line_cap` と definition body byte 上限の `body_byte_cap` があります。 +`excerpt --json` は 1-based の source 開始/終了位置、token `type`、`modifiers` を持つ軽量 range list の `semantic_tokens` を返すため、IDE や LLM クライアントは生の `content` 文字列を再パースせずに抜粋範囲を描画・後処理できます。C# の excerpt と LSP `textDocument/semanticTokens/full` は、keyword/modifier、namespace と type、method と property、parameter、variable と field、declaration modifier を判定する同じ source classifier を共有します。excerpt は indexed source 全体を分類してから可視範囲へ絞るため、狭い範囲やクランプされた範囲でも declaration context を維持します。excerpt の range mapping と LSP の delta encoding は座標変換だけを担当し、semantic kind を選びません。`semantic_token_coordinate_space` は `source` です。`--max-line-width` で返却内容がクランプされた場合、`content_line_spans` は返却 content 行と可視 content column span を、対応する source 行と source column span に対応付けます。clamp marker は未対応領域として扱い、semantic token には含めません。excerpt row は `requested_start_line`、`requested_end_line`、`effective_start_line`、`effective_end_line`、`content_truncation_reasons`、任意の `content_recovery` も返すため、`--max-line-width` による `line_width_cap` を検出して省略部分を再取得できます。`content_recovery` と対応する `body_content_recovery` では、`argv` が一次的な機械可読契約です。この配列は現在の apphost、または `dotnet` と実行中 assembly の組から始まるため、同じ build と database contract で復旧できます。先頭が `-` の root-level path の前には、対応済みの `--` end-of-options marker を置き、復旧 CLI が option として解釈しないようにします。従来の `command` は表示専用で、`command_display_only: true` と `command_shell`（`posix-sh` または `powershell`）を伴い、宣言した shell 向けに escape されます。MCP では同等の camelCase field 名を返します。body を持つ JSON row は対応する `body_requested_*`、`body_effective_*`、`body_content_truncation_reasons` も返します。body reason には snippet/body 行数上限の `body_line_cap` と definition body byte 上限の `body_byte_cap` があります。 `inspect` と MCP の `analyze_symbol` は、主定義、同一ファイル内の近傍シンボル、参照、caller、callee、ファイルメタデータ、さらにワークスペース鮮度/git メタデータと graph 対応メタデータを1レスポンスにまとめます。bundle 内の graph 節が実際に SQL ベースの read に依存する場合だけ、`sql_graph_contract_ready` / `sql_graph_contract_degraded_reason`（MCP では既存の camelCase alias も）も返します。mixed-language bundle で C# / JS などの graph row しか返っていない場合は SQL trust signal を出さないため、無関係なクエリが stale SQL state に引きずられません。複数の連続クエリを避けたい AI ワークフロー向けです。call graph 系の節は言語差分を考慮しており、未対応言語では `graphSupported` / `graphSupportReason` によって「未対応」と「ヒットなし」を区別できます。その場合は `search` を優先して使う前提です。 diff --git a/README.md b/README.md index 8fb848a92c..cab2e8f5db 100644 --- a/README.md +++ b/README.md @@ -170,7 +170,7 @@ incomplete generation; JSON still reports `status: "partial"`. |---|---| | Search and navigation | `search`, `find`, `excerpt`, `symbols`, `definition`, `references`, `callers`, `callees`, `inspect`, `map`, `deps`, `impact`, `unused`, and `hotspots`. See the [command reference](USER_GUIDE.md#command-reference). | | AI integration | `cdidx mcp` exposes indexed search tools for Claude Code, Cursor, Windsurf, Copilot, Codex, and other MCP clients. See [AI Integration](USER_GUIDE.md#ai-integration). | -| Editor lookup | `cdidx lsp --db .cdidx/codeindex.db` starts a read-only LSP shim for editors that can launch an LSP command. Large document/workspace symbol requests support bounded partial-result chunks, work-done progress, and request cancellation. References use indexed symbol identity, symbol locations select identifiers, and type hints omit explicitly declared types. C# semantic tokens distinguish keywords, modifiers, namespace components, types, fields, methods, and declarations. | +| Editor lookup | `cdidx lsp --db .cdidx/codeindex.db` starts a read-only LSP shim for editors that can launch an LSP command. Large document/workspace symbol requests support bounded partial-result chunks, work-done progress, and request cancellation. References use indexed symbol identity, symbol locations select identifiers, and type hints omit explicitly declared types. LSP and `excerpt --json` share one C# semantic classifier that distinguishes keywords, modifiers, namespaces/types, methods/properties, parameters, variables/fields, and declarations. | | Freshness | `status --check`, `--files`, `--commits`, `--changed-between`, and `--watch` keep the DB aligned with the workspace. | | Validation | `cdidx validate` reports encoding and line-ending issues in indexed files. See [Validate indexed files](USER_GUIDE.md#validate-indexed-files). | | Language coverage | `cdidx languages --json` is the live capability probe; add `--format count`, `--summary-only`, `--capability `, `--language`, `--extension`, or `--alias` to narrow output. See [Supported languages](USER_GUIDE.md#supported-languages). | @@ -583,7 +583,7 @@ commit し、構造化 `file_errors` を返して partial-result 終了コード |---|---| | 検索とナビゲーション | `search`、`find`、`excerpt`、`symbols`、`definition`、`references`、`callers`、`callees`、`inspect`、`map`、`deps`、`impact`、`unused`、`hotspots`。詳細は [コマンドリファレンス](USER_GUIDE.md#コマンドリファレンス)。 | | AI 連携 | `cdidx mcp` は Claude Code、Cursor、Windsurf、Copilot、Codex などの MCP client に indexed search tool を提供します。詳細は [AIとの連携](USER_GUIDE.md#aiとの連携)。 | -| editor lookup | `cdidx lsp --db .cdidx/codeindex.db` は、LSP command を起動できる editor 向けの read-only LSP shim です。大きな document/workspace symbol request は上限付き partial-result chunk、work-done progress、request cancellation に対応します。reference は indexed symbol identity を使い、symbol location は identifier を選択し、type hint は明示的に宣言された型を省略します。C# semantic token は keyword、modifier、namespace component、type、field、method、declaration を区別します。 | +| editor lookup | `cdidx lsp --db .cdidx/codeindex.db` は、LSP command を起動できる editor 向けの read-only LSP shim です。大きな document/workspace symbol request は上限付き partial-result chunk、work-done progress、request cancellation に対応します。reference は indexed symbol identity を使い、symbol location は identifier を選択し、type hint は明示的に宣言された型を省略します。LSP と `excerpt --json` は同じ C# semantic classifier を共有し、keyword、modifier、namespace/type、method/property、parameter、variable/field、declaration を区別します。 | | 鮮度管理 | `status --check`、`--files`、`--commits`、`--changed-between`、`--watch` で DB と workspace を揃えます。 | | validation | `cdidx validate` は indexed file の encoding / line-ending 問題を報告します。詳細は [Indexed files を validate する](USER_GUIDE.md#indexed-files-を-validate-する)。 | | 対応言語 | `cdidx languages --json` が live capability probe です。`--language`、`--extension`、`--alias` で 1 行を lookup できます。詳細は [対応言語](USER_GUIDE.md#対応言語)。 | diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 1a9218e46a..6921fccb2e 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -520,6 +520,7 @@ Use `docs/test-doc-maintenance-plan.md` before moving oversized suites or adding indexes one real C# fixture containing user-defined generic classes and delegates, block-comment generic trivia, repeated same-line generic names under indentation, unrelated same-name properties, ordinal-case type/property collisions, `Action` / `Stream` references, and uppercase property receivers split across partial-class and inherited-base files. Keep CLI `references`, `inspect`, and `deps` assertions together with the MCP `references`, `analyze_symbol`, and `deps` wrappers so unresolved diagnostics, generic-arity selection, case-sensitive receiver handling, false dependency-edge suppression, and preservation of real member dependencies cannot drift across surfaces. - `LspServerTests.cs` LSP JSON-RPC behavior and indexed editor semantics. Reference parity coverage must compare LSP locations with the matching CLI candidate bundle, including overload identity and both `includeDeclaration` states. Keep document/workspace identifier ranges and explicit-versus-inferred inlay hints in one source-semantics fixture so persisted-column anchoring, source confirmation, and hint suppression cannot drift independently. + C# semantic-token coverage decodes LSP delta coordinates and pairs the same keyword/modifier, namespace/type, method/property, parameter/variable, generic/record, and declaration-kind fixture with `QueryCommandRunnerTests.RunExcerpt_JsonCSharpSemanticKindsMatchLspClassifier_Issue4852`; keep both surfaces on the shared source classifier, retain ordering/non-overlap assertions, and preserve the narrow excerpt check whose parameter/type-parameter declarations are outside the returned range. Document-symbol hierarchy coverage keeps positional and body record members in one same-line fixture, asserts the complete root-to-member shape and deterministic sibling order, and retains the separate same-range top-level controls so order-independent parent resolution cannot over-nest unrelated symbols. Keep a same-line duplicate-container fixture with distinct positional members so selection-column disambiguation cannot move an earlier member beneath a later same-named container. Symbol-progress coverage crosses the production chunk item limit with one sentinel, verifies the complete deterministic sequence across partial-result frames, and keeps live work-done delivery, truncation, typed request-ID cancellation, full-queue cancellation, queue-pressure notification preservation, backpressured `Server busy` response retention, emitted-count cancellation reporting, output-failure propagation, and invalid-token variants in the same protocol fixture. Assert both the item and UTF-8 JSON body budgets for every emitted chunk. The server-busy backpressure cancellation fixture must stage the cancel frame until both the active symbol request and blocked response write have been observed. Block the request on its cancellation token instead of matching fixed delays so suite load cannot convert the expected cancellation into a successful response. @@ -1415,6 +1416,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" は user-defined generic class / delegate、generic 間の block comment trivia、indent 下で同一行に反復する同名 generic、無関係な同名 property、大文字小文字だけ異なる type / property collision、`Action` / `Stream` reference、partial class の別ファイルおよび継承 base file に分かれた大文字始まりの property receiver を含む実 C# fixture を1回 index します。CLI の `references`、`inspect`、`deps` と MCP wrapper の `references`、`analyze_symbol`、`deps` を同じ fixture で検証し、unresolved 診断、generic arity による選択、case-sensitive receiver 判定、偽 dependency edge の抑制、実在する member dependency の維持が surface 間でずれないよう維持してください。 - `LspServerTests.cs` LSP の JSON-RPC 挙動と indexed editor semantics のテスト。reference parity coverage では、overload identity と `includeDeclaration` の両状態を含め、LSP location と対応する CLI candidate bundle を比較してください。document/workspace の identifier range と explicit/inferred inlay hint は 1 つの source-semantics fixture にまとめ、保存済み column の anchoring、source 上の確認、hint 抑制が別々に drift しないようにします。 + C# semantic-token coverage は LSP の delta 座標を decode し、同じ keyword/modifier、namespace/type、method/property、parameter/variable、generic/record、declaration kind の fixture を `QueryCommandRunnerTests.RunExcerpt_JsonCSharpSemanticKindsMatchLspClassifier_Issue4852` と対にします。両 surface が shared source classifier を使い続け、順序と非 overlap の assertion、および parameter / type-parameter の declaration が返却範囲外にある narrow excerpt の検証を維持してください。 document-symbol hierarchy coverage は positional member と body member を同一行の record fixture にまとめ、root から member までの完全な形状と決定的な sibling 順序を assert します。順序非依存の親解決が無関係な symbol を過剰に nest しないよう、same-range top-level の control は独立したまま維持してください。selection column による曖昧性解消で前の member が行内で後にある同名 container の配下へ移動しないよう、異なる positional member を持つ同一行の duplicate-container fixture も維持してください。 symbol-progress coverage は production の chunk item limit を sentinel 1 件だけで超え、partial-result frame 全体の決定的な完全 sequence を検証します。live work-done delivery、truncation、型付き request ID の cancellation、full-queue cancellation、queue pressure 下の notification 保持、backpressure された `Server busy` response の保持、cancellation 時の送信済み件数、output-failure propagation、invalid-token variant は同じ protocol fixture にまとめ、送信した全 chunk の item 上限と UTF-8 JSON body budget の両方を assert してください。 server-busy backpressure cancellation fixture では、active symbol request と block された response write の両方を観測するまで cancel frame を stage してください。固定 delay の一致に依存せず cancellation token 上で request を block することで、suite load により期待する cancellation が成功 response に変わる競合を防ぎます。 diff --git a/changelog.d/unreleased/4852.fixed.md b/changelog.d/unreleased/4852.fixed.md new file mode 100644 index 0000000000..9a198c61ae --- /dev/null +++ b/changelog.d/unreleased/4852.fixed.md @@ -0,0 +1,26 @@ +--- +category: fixed +issues: + - 4852 +affected: + - src/CodeIndex/Semantics/CSharpSemanticTokenClassifier.cs + - src/CodeIndex/Lsp/LspServer.SemanticTokens.cs + - src/CodeIndex/Lsp/LspServer.NavigationFeatures.cs + - src/CodeIndex/Cli/QueryCommandRunner.Excerpt.cs + - src/CodeIndex/Cli/QueryCommandRunner.Inspect.cs + - src/CodeIndex/Database/DbReader.FilesStatus.cs + - tests/CodeIndex.Tests/LspServerTests.cs + - tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs + - tests/CodeIndex.Tests/golden/excerpt.json + - README.md + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Restored consistent C# semantic token kinds across LSP and excerpt output (#4852)** — a shared source classifier now identifies keywords and modifiers, namespaces and types, methods and properties, parameters, variables and fields, generic/record constructs, contextual keywords, and declarations before LSP delta encoding or excerpt source-coordinate mapping. Excerpts classify the full indexed source before filtering visible spans, preserving declaration context outside narrow ranges. This fixes the direct regression of #4444 where `private` and `using` appeared as variables and method names could appear as types. + +## 日本語 + +- **LSP と excerpt 出力で一貫した C# semantic token kind を復元しました (#4852)** — shared source classifier が keyword / modifier、namespace / type、method / property、parameter、variable / field、generic / record 構文、contextual keyword、declaration を判定してから、LSP の delta encoding または excerpt の source 座標 mapping を行うようになりました。excerpt は indexed source 全体を分類してから可視 span へ絞り、狭い範囲の外にある declaration context も維持します。`private` と `using` が variable、method 名が type として表示されることがあった #4444 の直接 regression を修正します。 diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Excerpt.cs b/src/CodeIndex/Cli/QueryCommandRunner.Excerpt.cs index cab7e86895..c998aa2e68 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Excerpt.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Excerpt.cs @@ -2,6 +2,7 @@ using System.Text.Json; using System.Text.Json.Nodes; using CodeIndex.Database; +using CodeIndex.Semantics; namespace CodeIndex.Cli; @@ -148,7 +149,7 @@ public static int RunExcerpt(string[] cmdArgs, JsonSerializerOptions jsonOptions { ExcerptRecoveryCommandFormatter.ApplyDbPath(excerpt, options.DbPath); if (!options.NoSemanticTokens) - excerpt.SemanticTokens = BuildExcerptSemanticTokens(excerpt); + excerpt.SemanticTokens = BuildExcerptSemanticTokens(excerpt, reader); } if (options.Json) @@ -217,13 +218,26 @@ private static bool TryParseExcerptLocationArgument( private static bool TryParsePositiveLine(string value, out int line) => int.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out line) && line > 0; - private static List BuildExcerptSemanticTokens(FileExcerptResult excerpt) + private static List BuildExcerptSemanticTokens( + FileExcerptResult excerpt, + DbReader reader) { var tokens = new List(); var lines = excerpt.Content.Replace("\r\n", "\n").Split('\n'); var spans = excerpt.ContentLineSpans.Count == 0 ? BuildIdentityExcerptContentLineSpans(excerpt, lines) : excerpt.ContentLineSpans; + var isCSharp = string.Equals(excerpt.Lang, "csharp", StringComparison.OrdinalIgnoreCase) || + string.Equals(Path.GetExtension(excerpt.Path), ".cs", StringComparison.OrdinalIgnoreCase); + var indexedSourceLines = isCSharp + ? reader.GetIndexedSourceLinesForSemanticTokens(excerpt.Path) + : []; + var classifiesIndexedSource = indexedSourceLines.Count > 0; + var classifiedCSharpTokens = isCSharp + ? CSharpSemanticTokenClassifier.Classify( + classifiesIndexedSource ? indexedSourceLines : lines, + CSharpSemanticTokenClassifier.DefaultExcerptTokenLimit) + : []; foreach (var span in spans) { if (span.ContentLine <= 0 || span.ContentLine > lines.Length) @@ -232,6 +246,42 @@ private static List BuildExcerptSemanticTokens(FileExcerpt var line = lines[span.ContentLine - 1]; var startColumn = Math.Clamp(span.ContentStartColumn - 1, 0, line.Length); var endColumn = Math.Clamp(span.ContentEndColumn - 1, startColumn, line.Length); + if (isCSharp) + { + var classifiedLine = classifiesIndexedSource + ? span.SourceLine - 1 + : span.ContentLine - 1; + var classifiedStartColumn = classifiesIndexedSource + ? span.SourceStartColumn - 1 + : startColumn; + var classifiedEndColumn = classifiesIndexedSource + ? span.SourceEndColumn - 1 + : endColumn; + foreach (var token in classifiedCSharpTokens) + { + if (token.Line != classifiedLine || + token.StartCharacter < classifiedStartColumn || + token.StartCharacter + token.Length > classifiedEndColumn) + { + continue; + } + + var sourceStartColumn = span.SourceStartColumn + + token.StartCharacter - + classifiedStartColumn; + tokens.Add(new ExcerptSemanticToken + { + StartLine = span.SourceLine, + StartColumn = sourceStartColumn, + EndLine = span.SourceLine, + EndColumn = sourceStartColumn + token.Length, + Type = CSharpSemanticTokenClassifier.ToProtocolName(token.Kind), + Modifiers = token.IsDeclaration ? ["declaration"] : [], + }); + } + continue; + } + var column = startColumn; while (column < endColumn) { diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Inspect.cs b/src/CodeIndex/Cli/QueryCommandRunner.Inspect.cs index f47a307119..a3925cad68 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Inspect.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Inspect.cs @@ -247,7 +247,7 @@ public static int RunInspect(string[] cmdArgs, JsonSerializerOptions jsonOptions if (sourceExcerpt != null) { ExcerptRecoveryCommandFormatter.ApplyDbPath(sourceExcerpt, options.DbPath); - sourceExcerpt.SemanticTokens = BuildExcerptSemanticTokens(sourceExcerpt); + sourceExcerpt.SemanticTokens = BuildExcerptSemanticTokens(sourceExcerpt, reader); payload["source_excerpt"] = JsonSerializer.SerializeToNode(sourceExcerpt, CliJsonSerializerContextFactory.Create(jsonOptions).FileExcerptResult); } ApplyInspectFieldSelection(payload, options, jsonOptions); diff --git a/src/CodeIndex/Database/DbReader.FilesStatus.cs b/src/CodeIndex/Database/DbReader.FilesStatus.cs index 40ae807022..79fe4e97d7 100644 --- a/src/CodeIndex/Database/DbReader.FilesStatus.cs +++ b/src/CodeIndex/Database/DbReader.FilesStatus.cs @@ -743,6 +743,25 @@ FROM chunks c return lineMap.Count > 0; } + /// + /// Reconstruct all indexed source lines while preserving absolute line positions for semantic classification. + /// semantic 分類向けに絶対行位置を維持しながら、indexed source 全行を再構成する。 + /// + internal IReadOnlyList GetIndexedSourceLinesForSemanticTokens(string path) + { + if (!TryLoadIndexedFileLines(path, out _, out var totalLines, out var lineMap) || totalLines <= 0) + return []; + + var sourceLines = new string?[totalLines]; + foreach (var (line, content) in lineMap) + { + if (line > 0 && line <= sourceLines.Length) + sourceLines[line - 1] = content; + } + + return sourceLines; + } + /// /// Reconstruct a file excerpt from indexed chunks. /// インデックス済みチャンクからファイル抜粋を再構成する。 diff --git a/src/CodeIndex/Lsp/LspServer.NavigationFeatures.cs b/src/CodeIndex/Lsp/LspServer.NavigationFeatures.cs index a8222def1d..c9738fad13 100644 --- a/src/CodeIndex/Lsp/LspServer.NavigationFeatures.cs +++ b/src/CodeIndex/Lsp/LspServer.NavigationFeatures.cs @@ -136,20 +136,14 @@ private JsonObject SemanticTokensFull(JsonElement root) if (!TryResolveIndexedDocument(root, out var document)) return new JsonObject { ["data"] = new JsonArray() }; - var lineCache = new Dictionary(); - var symbols = GetDocumentSymbols(document.IndexedPath, MaxSemanticTokenItems) - .Where(symbol => !string.IsNullOrWhiteSpace(symbol.Name)) - .Take(MaxSemanticTokenItems) - .Select(symbol => BuildSemanticToken(document, symbol, lineCache)) - .Where(token => token.HasValue) - .Select(token => token!.Value) - .OrderBy(token => token.Line) - .ThenBy(token => token.StartCharacter) - .ToList(); - IEnumerable lexicalTokens = string.Equals(Path.GetExtension(document.ResolvedPath), ".cs", StringComparison.OrdinalIgnoreCase) - ? BuildCSharpLexicalSemanticTokens(document, lineCache) - : []; - symbols = RemoveOverlappingSemanticTokens(lexicalTokens.Concat(symbols)) + var isCSharp = string.Equals( + Path.GetExtension(document.ResolvedPath), + ".cs", + StringComparison.OrdinalIgnoreCase); + var symbols = isCSharp + ? BuildCSharpSemanticTokens(document).ToList() + : BuildIndexedSemanticTokens(document); + symbols = symbols .OrderBy(token => token.Line) .ThenBy(token => token.StartCharacter) .ToList(); @@ -172,6 +166,20 @@ private JsonObject SemanticTokensFull(JsonElement root) return new JsonObject { ["data"] = data }; } + private List BuildIndexedSemanticTokens(IndexedDocumentContext document) + { + var lineCache = new Dictionary(); + return GetDocumentSymbols(document.IndexedPath, MaxSemanticTokenItems) + .Where(symbol => !string.IsNullOrWhiteSpace(symbol.Name)) + .Take(MaxSemanticTokenItems) + .Select(symbol => BuildSemanticToken(document, symbol, lineCache)) + .Where(token => token.HasValue) + .Select(token => token!.Value) + .OrderBy(token => token.Line) + .ThenBy(token => token.StartCharacter) + .ToList(); + } + private JsonArray InlayHint(JsonElement root) { if (!TryResolveIndexedDocument(root, out var document)) diff --git a/src/CodeIndex/Lsp/LspServer.SemanticTokens.cs b/src/CodeIndex/Lsp/LspServer.SemanticTokens.cs index 0c05a4df1c..4bff351107 100644 --- a/src/CodeIndex/Lsp/LspServer.SemanticTokens.cs +++ b/src/CodeIndex/Lsp/LspServer.SemanticTokens.cs @@ -11,6 +11,7 @@ using CodeIndex.Mcp; using CodeIndex.Models; using CodeIndex.Security; +using CodeIndex.Semantics; namespace CodeIndex.Lsp; @@ -18,207 +19,20 @@ internal sealed partial class LspServer : IDisposable { private readonly record struct SemanticToken(int Line, int StartCharacter, int Length, int TokenType, int TokenModifiers); - private static IEnumerable RemoveOverlappingSemanticTokens(IEnumerable candidates) + private IEnumerable BuildCSharpSemanticTokens(IndexedDocumentContext document) { - var selected = new List(); - foreach (var candidate in candidates) - { - if (selected.Any(existing => - existing.Line == candidate.Line && - existing.StartCharacter < candidate.StartCharacter + candidate.Length && - candidate.StartCharacter < existing.StartCharacter + existing.Length)) - { - continue; - } - - selected.Add(candidate); - if (selected.Count == MaxSemanticTokenItems) - break; - } - return selected; - } - - private static readonly HashSet CSharpModifiers = new(StringComparer.Ordinal) - { - "abstract", "async", "const", "extern", "file", "internal", "override", "partial", - "private", "protected", "public", "readonly", "required", "sealed", "static", "unsafe", "virtual", "volatile", - }; - - private static readonly HashSet CSharpKeywords = new(StringComparer.Ordinal) - { - "as", "base", "bool", "break", "byte", "case", "catch", "char", "checked", "class", "continue", - "decimal", "default", "delegate", "do", "double", "else", "enum", "event", "explicit", "false", "finally", - "fixed", "float", "for", "foreach", "goto", "if", "implicit", "in", "int", "interface", "is", "lock", - "long", "namespace", "new", "null", "object", "operator", "out", "params", "record", "ref", "return", "sbyte", - "short", "sizeof", "stackalloc", "string", "struct", "switch", "this", "throw", "true", "try", "typeof", - "uint", "ulong", "unchecked", "using", "ushort", "void", "while", "with", "yield", - }; - - private IEnumerable BuildCSharpLexicalSemanticTokens( - IndexedDocumentContext document, - Dictionary lineCache) - { - var inBlockComment = false; - var stringMode = CSharpStringMode.None; - var rawQuoteCount = 0; - var ordinaryQuote = '\0'; - for (var line = 0; line < MaxSemanticTokenItems; line++) - { - if (!TryReadPositionLineCached(document.ResolvedPath, line, lineCache, out var sourceLine)) - yield break; - - for (var index = 0; index < sourceLine.Length;) - { - if (inBlockComment) - { - var end = sourceLine.IndexOf("*/", index, StringComparison.Ordinal); - if (end < 0) - break; - inBlockComment = false; - index = end + 2; - continue; - } - - if (stringMode == CSharpStringMode.Raw) - { - var end = FindRawStringEnd(sourceLine, index, rawQuoteCount); - if (end < 0) - break; - stringMode = CSharpStringMode.None; - index = end; - continue; - } - - if (stringMode == CSharpStringMode.Verbatim) - { - var end = sourceLine.IndexOf('"', index); - if (end < 0) - break; - if (end + 1 < sourceLine.Length && sourceLine[end + 1] == '"') - { - index = end + 2; - continue; - } - stringMode = CSharpStringMode.None; - index = end + 1; - continue; - } - - if (stringMode == CSharpStringMode.Ordinary) - { - if (sourceLine[index] == '\\') - { - index = Math.Min(index + 2, sourceLine.Length); - continue; - } - if (sourceLine[index++] == ordinaryQuote) - stringMode = CSharpStringMode.None; - continue; - } + if (!TryReadAllPositionLines(document.ResolvedPath, out var sourceLines)) + yield break; - if (index + 1 < sourceLine.Length && sourceLine[index] == '/' && sourceLine[index + 1] == '/') - break; - if (index + 1 < sourceLine.Length && sourceLine[index] == '/' && sourceLine[index + 1] == '*') - { - inBlockComment = true; - index += 2; - continue; - } - var quoteCount = CountConsecutive(sourceLine, index, '"'); - if (quoteCount >= 3) - { - stringMode = CSharpStringMode.Raw; - rawQuoteCount = quoteCount; - index += quoteCount; - continue; - } - if (sourceLine[index] == '@' && index + 1 < sourceLine.Length && sourceLine[index + 1] == '"') - { - stringMode = CSharpStringMode.Verbatim; - index += 2; - continue; - } - if (sourceLine[index] == '@' && index + 2 < sourceLine.Length && sourceLine[index + 1] == '$' && sourceLine[index + 2] == '"') - { - stringMode = CSharpStringMode.Verbatim; - index += 3; - continue; - } - if (sourceLine[index] is '\'' or '"') - { - stringMode = CSharpStringMode.Ordinary; - ordinaryQuote = sourceLine[index]; - index++; - continue; - } - if (!IsCSharpIdentifierStart(sourceLine[index])) - { - index++; - continue; - } - - var start = index++; - while (index < sourceLine.Length && IsTokenChar(sourceLine[index])) - index++; - var word = sourceLine[start..index].TrimStart('@'); - if (CSharpModifiers.Contains(word)) - yield return new SemanticToken(line, start, index - start, 16, 0); - else if (CSharpKeywords.Contains(word)) - yield return new SemanticToken(line, start, index - start, 15, 0); - else if (IsCSharpNamespaceComponent(sourceLine, start)) - yield return new SemanticToken(line, start, index - start, 0, 0); - } - } - } - - private static bool IsCSharpIdentifierStart(char value) => char.IsLetter(value) || value is '_' or '@'; - - private enum CSharpStringMode - { - None, - Ordinary, - Verbatim, - Raw, - } - - private static int CountConsecutive(string text, int start, char value) - { - var index = start; - while (index < text.Length && text[index] == value) - index++; - return index - start; - } - - private static int FindRawStringEnd(string line, int start, int quoteCount) - { - for (var index = start; index < line.Length; index++) + foreach (var token in CSharpSemanticTokenClassifier.Classify(sourceLines, MaxSemanticTokenItems)) { - if (line[index] == '"' && CountConsecutive(line, index, '"') >= quoteCount) - return index + quoteCount; + yield return new SemanticToken( + token.Line, + token.StartCharacter, + token.Length, + CSharpSemanticTokenClassifier.ToLspTokenType(token.Kind), + token.IsDeclaration ? 1 << 0 : 0); } - return -1; - } - - private static bool IsCSharpNamespaceComponent(string line, int start) - { - var trimmedStart = line.Length - line.AsSpan().TrimStart().Length; - var trimmedLine = line.AsSpan(trimmedStart); - var nameStart = trimmedLine.StartsWith("global using ", StringComparison.Ordinal) - ? trimmedStart + "global using ".Length - : trimmedLine.StartsWith("using ", StringComparison.Ordinal) - ? trimmedStart + "using ".Length - : trimmedLine.StartsWith("namespace ", StringComparison.Ordinal) - ? trimmedStart + "namespace ".Length - : -1; - if (nameStart < 0 || start < nameStart) - return false; - - var semicolon = line.IndexOf(';', nameStart); - var brace = line.IndexOf('{', nameStart); - var nameEnd = new[] { semicolon, brace }.Where(value => value >= 0).DefaultIfEmpty(line.Length).Min(); - var alias = line.IndexOf('=', nameStart, Math.Max(0, nameEnd - nameStart)); - var qualifiedNameStart = alias >= 0 ? alias + 1 : nameStart; - return start >= qualifiedNameStart && start < nameEnd; } private SemanticToken? BuildSemanticToken(IndexedDocumentContext document, SymbolResult symbol, Dictionary lineCache) diff --git a/src/CodeIndex/Semantics/CSharpSemanticTokenClassifier.cs b/src/CodeIndex/Semantics/CSharpSemanticTokenClassifier.cs new file mode 100644 index 0000000000..cdbb4da370 --- /dev/null +++ b/src/CodeIndex/Semantics/CSharpSemanticTokenClassifier.cs @@ -0,0 +1,928 @@ +using System.Globalization; + +namespace CodeIndex.Semantics; + +internal enum CSharpSemanticTokenKind +{ + Namespace, + Type, + Class, + Enum, + Interface, + Struct, + TypeParameter, + Parameter, + Variable, + Property, + EnumMember, + Event, + Function, + Method, + Keyword, + Modifier, + Number, + Field, +} + +internal readonly record struct ClassifiedCSharpSemanticToken( + int Line, + int StartCharacter, + int Length, + CSharpSemanticTokenKind Kind, + bool IsDeclaration); + +internal static class CSharpSemanticTokenClassifier +{ + internal const int DefaultExcerptTokenLimit = 10_000; + + private static readonly HashSet Modifiers = new(StringComparer.Ordinal) + { + "abstract", "async", "const", "extern", "file", "internal", "override", "partial", + "private", "protected", "public", "readonly", "required", "sealed", "static", + "unsafe", "virtual", "volatile", + }; + + private static readonly HashSet Keywords = new(StringComparer.Ordinal) + { + "add", "alias", "and", "as", "ascending", "async", "await", "base", "bool", "break", + "by", "byte", "case", "catch", "char", "checked", "class", "const", "continue", "decimal", + "default", "delegate", "descending", "do", "double", "dynamic", "else", "enum", "equals", + "event", "explicit", "false", "finally", "fixed", "float", "for", "foreach", "from", "get", + "global", "goto", "group", "if", "implicit", "in", "init", "int", "interface", "internal", + "into", "is", "join", "let", "lock", "long", "managed", "nameof", "namespace", "new", "not", + "notnull", "null", "object", "on", "operator", "or", "orderby", "out", "override", "params", + "partial", "private", "protected", "public", "readonly", "record", "ref", "remove", "required", + "return", "sbyte", "scoped", "sealed", "select", "set", "short", "sizeof", "stackalloc", + "static", "string", "struct", "switch", "this", "throw", "true", "try", "typeof", "uint", + "ulong", "unchecked", "unmanaged", "unsafe", "using", "ushort", "value", "var", "virtual", + "void", "volatile", "when", "where", "while", "with", "yield", + }; + + private static readonly HashSet BuiltInTypeKeywords = new(StringComparer.Ordinal) + { + "bool", "byte", "char", "decimal", "double", "dynamic", "float", "int", "long", + "nint", "nuint", "object", "sbyte", "short", "string", "uint", "ulong", "ushort", "void", + }; + + private static readonly HashSet TypeDeclarationKeywords = new(StringComparer.Ordinal) + { + "class", "enum", "interface", "record", "struct", + }; + + internal static IReadOnlyList Classify( + IReadOnlyList sourceLines, + int maxTokens) + { + if (maxTokens <= 0 || sourceLines.Count == 0) + return []; + + var lexemes = Tokenize(sourceLines, maxTokens); + if (lexemes.Count == 0) + return []; + + var kinds = new CSharpSemanticTokenKind?[lexemes.Count]; + var declarations = new bool[lexemes.Count]; + var declaredTypeNames = new HashSet(StringComparer.Ordinal); + var declaredTypeParameterNames = new HashSet(StringComparer.Ordinal); + var declaredParameterNames = new HashSet(StringComparer.Ordinal); + var declaredPropertyNames = new HashSet(StringComparer.Ordinal); + var declaredFieldNames = new HashSet(StringComparer.Ordinal); + var methodBodies = new List<(int Start, int End)>(); + + ClassifyKeywordsAndNumbers(lexemes, kinds); + ClassifyDirectives(lexemes, kinds); + ClassifyTypeDeclarations( + lexemes, + kinds, + declarations, + declaredTypeNames, + declaredTypeParameterNames, + declaredParameterNames); + ClassifyProperties( + lexemes, + kinds, + declarations, + declaredPropertyNames); + ClassifyMethods( + lexemes, + kinds, + declarations, + declaredTypeNames, + declaredTypeParameterNames, + declaredParameterNames, + methodBodies); + ClassifyVariablesAndFields( + lexemes, + kinds, + declarations, + methodBodies, + declaredFieldNames); + ClassifyRemainingIdentifiers( + lexemes, + kinds, + declaredTypeNames, + declaredTypeParameterNames, + declaredParameterNames, + declaredPropertyNames, + declaredFieldNames); + + var result = new List(Math.Min(maxTokens, lexemes.Count)); + for (var index = 0; index < lexemes.Count && result.Count < maxTokens; index++) + { + if (kinds[index] is not { } kind) + continue; + + var lexeme = lexemes[index]; + result.Add(new ClassifiedCSharpSemanticToken( + lexeme.Line, + lexeme.Start, + lexeme.Length, + kind, + declarations[index])); + } + + return result; + } + + internal static string ToProtocolName(CSharpSemanticTokenKind kind) => kind switch + { + CSharpSemanticTokenKind.Namespace => "namespace", + CSharpSemanticTokenKind.Type => "type", + CSharpSemanticTokenKind.Class => "class", + CSharpSemanticTokenKind.Enum => "enum", + CSharpSemanticTokenKind.Interface => "interface", + CSharpSemanticTokenKind.Struct => "struct", + CSharpSemanticTokenKind.TypeParameter => "typeParameter", + CSharpSemanticTokenKind.Parameter => "parameter", + CSharpSemanticTokenKind.Variable => "variable", + CSharpSemanticTokenKind.Property => "property", + CSharpSemanticTokenKind.EnumMember => "enumMember", + CSharpSemanticTokenKind.Event => "event", + CSharpSemanticTokenKind.Function => "function", + CSharpSemanticTokenKind.Method => "method", + CSharpSemanticTokenKind.Keyword => "keyword", + CSharpSemanticTokenKind.Modifier => "modifier", + CSharpSemanticTokenKind.Number => "number", + CSharpSemanticTokenKind.Field => "field", + _ => "variable", + }; + + internal static int ToLspTokenType(CSharpSemanticTokenKind kind) => kind switch + { + CSharpSemanticTokenKind.Namespace => 0, + CSharpSemanticTokenKind.Type => 1, + CSharpSemanticTokenKind.Class => 2, + CSharpSemanticTokenKind.Enum => 3, + CSharpSemanticTokenKind.Interface => 4, + CSharpSemanticTokenKind.Struct => 5, + CSharpSemanticTokenKind.TypeParameter => 6, + CSharpSemanticTokenKind.Parameter => 7, + CSharpSemanticTokenKind.Variable => 8, + CSharpSemanticTokenKind.Property => 9, + CSharpSemanticTokenKind.EnumMember => 10, + CSharpSemanticTokenKind.Event => 11, + CSharpSemanticTokenKind.Function => 12, + CSharpSemanticTokenKind.Method => 13, + CSharpSemanticTokenKind.Keyword => 15, + CSharpSemanticTokenKind.Modifier => 16, + CSharpSemanticTokenKind.Number => 19, + CSharpSemanticTokenKind.Field => 23, + _ => 8, + }; + + private static void ClassifyKeywordsAndNumbers( + IReadOnlyList lexemes, + CSharpSemanticTokenKind?[] kinds) + { + for (var index = 0; index < lexemes.Count; index++) + { + var lexeme = lexemes[index]; + if (lexeme.Kind == LexemeKind.Number) + { + kinds[index] = CSharpSemanticTokenKind.Number; + continue; + } + + if (lexeme.Kind != LexemeKind.Identifier) + continue; + + var word = NormalizeIdentifier(lexeme.Text); + if (Modifiers.Contains(word)) + kinds[index] = CSharpSemanticTokenKind.Modifier; + else if (Keywords.Contains(word)) + kinds[index] = CSharpSemanticTokenKind.Keyword; + } + } + + private static void ClassifyDirectives( + IReadOnlyList lexemes, + CSharpSemanticTokenKind?[] kinds) + { + for (var index = 0; index < lexemes.Count; index++) + { + var word = NormalizeIdentifier(lexemes[index].Text); + if (word == "namespace") + { + MarkQualifiedName(lexemes, kinds, index + 1, stopAtEquals: false); + } + else if (word == "using" && IsDirectiveStart(lexemes, index)) + { + MarkQualifiedName(lexemes, kinds, index + 1, stopAtEquals: true); + } + } + } + + private static void ClassifyTypeDeclarations( + IReadOnlyList lexemes, + CSharpSemanticTokenKind?[] kinds, + bool[] declarations, + HashSet declaredTypeNames, + HashSet declaredTypeParameterNames, + HashSet declaredParameterNames) + { + for (var index = 0; index < lexemes.Count; index++) + { + var keyword = NormalizeIdentifier(lexemes[index].Text); + if (!TypeDeclarationKeywords.Contains(keyword)) + continue; + + var nameIndex = FindNextUnclassifiedIdentifier(lexemes, kinds, index + 1); + if (nameIndex < 0) + continue; + + var kind = keyword switch + { + "enum" => CSharpSemanticTokenKind.Enum, + "interface" => CSharpSemanticTokenKind.Interface, + "struct" => CSharpSemanticTokenKind.Struct, + _ => CSharpSemanticTokenKind.Class, + }; + kinds[nameIndex] = kind; + declarations[nameIndex] = true; + declaredTypeNames.Add(NormalizeIdentifier(lexemes[nameIndex].Text)); + + MarkTypeParametersAfter( + lexemes, + kinds, + declarations, + nameIndex, + declaredTypeParameterNames); + + if (TryFindInvocationOpenParen(lexemes, nameIndex, out var openParen)) + { + MarkParameters( + lexemes, + kinds, + declarations, + openParen, + declaredParameterNames); + } + } + } + + private static void ClassifyProperties( + IReadOnlyList lexemes, + CSharpSemanticTokenKind?[] kinds, + bool[] declarations, + HashSet declaredPropertyNames) + { + for (var index = 0; index < lexemes.Count; index++) + { + if (lexemes[index].Kind != LexemeKind.Identifier || + kinds[index].HasValue || + !IsDeclarationTypeBefore(lexemes, kinds, index)) + { + continue; + } + + var next = index + 1; + if (next >= lexemes.Count) + continue; + + var isProperty = lexemes[next].Text == "{" && ContainsPropertyAccessor(lexemes, next); + isProperty |= lexemes[next].Text == "=>"; + if (!isProperty) + continue; + + kinds[index] = CSharpSemanticTokenKind.Property; + declarations[index] = true; + declaredPropertyNames.Add(NormalizeIdentifier(lexemes[index].Text)); + } + } + + private static void ClassifyMethods( + IReadOnlyList lexemes, + CSharpSemanticTokenKind?[] kinds, + bool[] declarations, + HashSet declaredTypeNames, + HashSet declaredTypeParameterNames, + HashSet declaredParameterNames, + List<(int Start, int End)> methodBodies) + { + for (var index = 0; index < lexemes.Count; index++) + { + if (lexemes[index].Kind != LexemeKind.Identifier || kinds[index].HasValue) + continue; + if (!TryFindInvocationOpenParen(lexemes, index, out var openParen)) + continue; + + var previous = index - 1; + if (previous >= 0 && NormalizeIdentifier(lexemes[previous].Text) == "new") + { + kinds[index] = CSharpSemanticTokenKind.Type; + continue; + } + + var name = NormalizeIdentifier(lexemes[index].Text); + var isDeclaration = declaredTypeNames.Contains(name) || + IsDeclarationTypeBefore(lexemes, kinds, index); + kinds[index] = CSharpSemanticTokenKind.Method; + declarations[index] = isDeclaration; + + if (!isDeclaration) + continue; + + MarkTypeParametersAfter( + lexemes, + kinds, + declarations, + index, + declaredTypeParameterNames); + var closeParen = MarkParameters( + lexemes, + kinds, + declarations, + openParen, + declaredParameterNames); + if (closeParen >= 0 && TryFindBodyRange(lexemes, closeParen, out var body)) + methodBodies.Add(body); + } + } + + private static void ClassifyVariablesAndFields( + IReadOnlyList lexemes, + CSharpSemanticTokenKind?[] kinds, + bool[] declarations, + IReadOnlyList<(int Start, int End)> methodBodies, + HashSet declaredFieldNames) + { + for (var index = 0; index < lexemes.Count; index++) + { + if (lexemes[index].Kind != LexemeKind.Identifier || + kinds[index].HasValue || + !IsDeclarationTypeBefore(lexemes, kinds, index) || + !HasDeclarationTerminatorAfter(lexemes, index)) + { + continue; + } + + var insideMethod = methodBodies.Any(body => index > body.Start && index < body.End); + var kind = insideMethod + ? CSharpSemanticTokenKind.Variable + : CSharpSemanticTokenKind.Field; + kinds[index] = kind; + declarations[index] = true; + if (kind == CSharpSemanticTokenKind.Field) + declaredFieldNames.Add(NormalizeIdentifier(lexemes[index].Text)); + } + } + + private static void ClassifyRemainingIdentifiers( + IReadOnlyList lexemes, + CSharpSemanticTokenKind?[] kinds, + IReadOnlySet declaredTypeNames, + IReadOnlySet declaredTypeParameterNames, + IReadOnlySet declaredParameterNames, + IReadOnlySet declaredPropertyNames, + IReadOnlySet declaredFieldNames) + { + for (var index = 0; index < lexemes.Count; index++) + { + if (lexemes[index].Kind != LexemeKind.Identifier || kinds[index].HasValue) + continue; + + var word = NormalizeIdentifier(lexemes[index].Text); + if (declaredTypeParameterNames.Contains(word)) + { + kinds[index] = CSharpSemanticTokenKind.TypeParameter; + } + else if (declaredParameterNames.Contains(word)) + { + kinds[index] = CSharpSemanticTokenKind.Parameter; + } + else if (declaredPropertyNames.Contains(word)) + { + kinds[index] = CSharpSemanticTokenKind.Property; + } + else if (declaredFieldNames.Contains(word)) + { + kinds[index] = CSharpSemanticTokenKind.Field; + } + else if (declaredTypeNames.Contains(word) || + IsTypeContext(lexemes, index) || + StartsWithUppercase(word)) + { + kinds[index] = CSharpSemanticTokenKind.Type; + } + else if (index > 0 && lexemes[index - 1].Text is "." or "?.") + { + kinds[index] = CSharpSemanticTokenKind.Property; + } + else + { + kinds[index] = CSharpSemanticTokenKind.Variable; + } + } + } + + private static List Tokenize(IReadOnlyList sourceLines, int maxTokens) + { + var maxLexemes = Math.Max(4_096, checked(Math.Min(maxTokens, 100_000) * 32)); + var result = new List(Math.Min(maxLexemes, 16_384)); + var inBlockComment = false; + var stringMode = StringMode.None; + var rawQuoteCount = 0; + var ordinaryQuote = '\0'; + + for (var lineIndex = 0; lineIndex < sourceLines.Count && result.Count < maxLexemes; lineIndex++) + { + var line = sourceLines[lineIndex] ?? string.Empty; + for (var index = 0; index < line.Length && result.Count < maxLexemes;) + { + if (inBlockComment) + { + var end = line.IndexOf("*/", index, StringComparison.Ordinal); + if (end < 0) + break; + inBlockComment = false; + index = end + 2; + continue; + } + + if (stringMode == StringMode.Raw) + { + var end = FindRawStringEnd(line, index, rawQuoteCount); + if (end < 0) + break; + stringMode = StringMode.None; + index = end; + continue; + } + + if (stringMode == StringMode.Verbatim) + { + var end = line.IndexOf('"', index); + if (end < 0) + break; + if (end + 1 < line.Length && line[end + 1] == '"') + { + index = end + 2; + continue; + } + stringMode = StringMode.None; + index = end + 1; + continue; + } + + if (stringMode == StringMode.Ordinary) + { + if (line[index] == '\\') + { + index = Math.Min(index + 2, line.Length); + continue; + } + if (line[index++] == ordinaryQuote) + stringMode = StringMode.None; + continue; + } + + if (index + 1 < line.Length && line[index] == '/' && line[index + 1] == '/') + break; + if (index + 1 < line.Length && line[index] == '/' && line[index + 1] == '*') + { + inBlockComment = true; + index += 2; + continue; + } + + var quoteCount = CountConsecutive(line, index, '"'); + if (quoteCount >= 3) + { + stringMode = StringMode.Raw; + rawQuoteCount = quoteCount; + index += quoteCount; + continue; + } + + if (StartsVerbatimString(line, index, out var prefixLength)) + { + stringMode = StringMode.Verbatim; + index += prefixLength; + continue; + } + + if (line[index] is '\'' or '"') + { + stringMode = StringMode.Ordinary; + ordinaryQuote = line[index++]; + continue; + } + + if (IsIdentifierStart(line[index])) + { + var start = index++; + while (index < line.Length && IsIdentifierPart(line[index])) + index++; + result.Add(new Lexeme( + lineIndex, + start, + index - start, + line[start..index], + LexemeKind.Identifier)); + continue; + } + + if (char.IsDigit(line[index])) + { + var start = index++; + while (index < line.Length && + (char.IsLetterOrDigit(line[index]) || line[index] is '_' or '.')) + { + index++; + } + result.Add(new Lexeme( + lineIndex, + start, + index - start, + line[start..index], + LexemeKind.Number)); + continue; + } + + if (char.IsWhiteSpace(line[index])) + { + index++; + continue; + } + + var punctuationLength = index + 1 < line.Length && + line.AsSpan(index, 2) is "=>" or "::" or "?." or "??" + ? 2 + : 1; + result.Add(new Lexeme( + lineIndex, + index, + punctuationLength, + line.Substring(index, punctuationLength), + LexemeKind.Punctuation)); + index += punctuationLength; + } + } + + return result; + } + + private static bool IsDirectiveStart(IReadOnlyList lexemes, int index) + { + if (index + 1 < lexemes.Count && lexemes[index + 1].Text == "(") + return false; + + for (var previous = index - 1; + previous >= 0 && lexemes[previous].Line == lexemes[index].Line; + previous--) + { + if (NormalizeIdentifier(lexemes[previous].Text) != "global") + return false; + } + + return true; + } + + private static void MarkQualifiedName( + IReadOnlyList lexemes, + CSharpSemanticTokenKind?[] kinds, + int start, + bool stopAtEquals) + { + var equals = -1; + var end = start; + while (end < lexemes.Count && lexemes[end].Text is not ";" and not "{") + { + if (lexemes[end].Text == "=") + equals = end; + end++; + } + + var nameStart = stopAtEquals && equals >= 0 ? equals + 1 : start; + for (var index = nameStart; index < end; index++) + { + if (lexemes[index].Kind == LexemeKind.Identifier && + NormalizeIdentifier(lexemes[index].Text) != "static") + { + kinds[index] = CSharpSemanticTokenKind.Namespace; + } + } + } + + private static int FindNextUnclassifiedIdentifier( + IReadOnlyList lexemes, + CSharpSemanticTokenKind?[] kinds, + int start) + { + for (var index = start; index < lexemes.Count; index++) + { + if (lexemes[index].Text is ";" or "{" or "}") + return -1; + if (lexemes[index].Kind == LexemeKind.Identifier && + !Keywords.Contains(NormalizeIdentifier(lexemes[index].Text)) && + !kinds[index].HasValue) + { + return index; + } + } + return -1; + } + + private static void MarkTypeParametersAfter( + IReadOnlyList lexemes, + CSharpSemanticTokenKind?[] kinds, + bool[] declarations, + int nameIndex, + HashSet declaredTypeParameterNames) + { + var open = nameIndex + 1; + if (open >= lexemes.Count || lexemes[open].Text != "<") + return; + + var close = FindMatching(lexemes, open, "<", ">"); + if (close < 0) + return; + + for (var index = open + 1; index < close; index++) + { + if (lexemes[index].Kind != LexemeKind.Identifier || + Keywords.Contains(NormalizeIdentifier(lexemes[index].Text))) + { + continue; + } + kinds[index] = CSharpSemanticTokenKind.TypeParameter; + declarations[index] = true; + declaredTypeParameterNames.Add(NormalizeIdentifier(lexemes[index].Text)); + } + } + + private static int MarkParameters( + IReadOnlyList lexemes, + CSharpSemanticTokenKind?[] kinds, + bool[] declarations, + int openParen, + HashSet declaredParameterNames) + { + var closeParen = FindMatching(lexemes, openParen, "(", ")"); + if (closeParen < 0) + return -1; + + var segmentStart = openParen + 1; + var nestedDepth = 0; + for (var index = openParen + 1; index <= closeParen; index++) + { + var text = lexemes[index].Text; + if (text is "(" or "[" or "<" or "{") + nestedDepth++; + else if (text is ")" or "]" or ">" or "}") + nestedDepth--; + + if ((text == "," && nestedDepth == 0) || index == closeParen) + { + var parameterIndex = FindParameterName(lexemes, kinds, segmentStart, index); + if (parameterIndex >= 0) + { + kinds[parameterIndex] = CSharpSemanticTokenKind.Parameter; + declarations[parameterIndex] = true; + declaredParameterNames.Add(NormalizeIdentifier(lexemes[parameterIndex].Text)); + } + segmentStart = index + 1; + } + } + + return closeParen; + } + + private static int FindParameterName( + IReadOnlyList lexemes, + CSharpSemanticTokenKind?[] kinds, + int start, + int end) + { + var equals = end; + for (var index = start; index < end; index++) + { + if (lexemes[index].Text == "=") + { + equals = index; + break; + } + } + + for (var index = equals - 1; index >= start; index--) + { + if (lexemes[index].Kind != LexemeKind.Identifier) + continue; + var word = NormalizeIdentifier(lexemes[index].Text); + if (!Keywords.Contains(word) && kinds[index] is not CSharpSemanticTokenKind.Modifier) + return index; + } + + return -1; + } + + private static bool TryFindInvocationOpenParen( + IReadOnlyList lexemes, + int nameIndex, + out int openParen) + { + openParen = -1; + var next = nameIndex + 1; + if (next >= lexemes.Count) + return false; + if (lexemes[next].Text == "(") + { + openParen = next; + return true; + } + if (lexemes[next].Text != "<") + return false; + + var close = FindMatching(lexemes, next, "<", ">"); + if (close < 0 || close + 1 >= lexemes.Count || lexemes[close + 1].Text != "(") + return false; + openParen = close + 1; + return true; + } + + private static bool TryFindBodyRange( + IReadOnlyList lexemes, + int closeParen, + out (int Start, int End) body) + { + body = default; + for (var index = closeParen + 1; index < lexemes.Count; index++) + { + if (lexemes[index].Text is ";" or "=>") + return false; + if (lexemes[index].Text != "{") + continue; + var close = FindMatching(lexemes, index, "{", "}"); + if (close < 0) + return false; + body = (index, close); + return true; + } + return false; + } + + private static bool ContainsPropertyAccessor(IReadOnlyList lexemes, int openBrace) + { + var closeBrace = FindMatching(lexemes, openBrace, "{", "}"); + if (closeBrace < 0) + return false; + for (var index = openBrace + 1; index < closeBrace; index++) + { + if (NormalizeIdentifier(lexemes[index].Text) is "get" or "set" or "init" or "add" or "remove") + return true; + } + return false; + } + + private static bool IsDeclarationTypeBefore( + IReadOnlyList lexemes, + CSharpSemanticTokenKind?[] kinds, + int index) + { + if (index <= 0 || lexemes[index - 1].Text is "." or "?.") + return false; + + var previous = index - 1; + if (lexemes[previous].Text is ">" or "]" or "?") + return true; + if (lexemes[previous].Kind != LexemeKind.Identifier) + return false; + + var word = NormalizeIdentifier(lexemes[previous].Text); + return BuiltInTypeKeywords.Contains(word) || + word == "var" || + kinds[previous] is CSharpSemanticTokenKind.Type or + CSharpSemanticTokenKind.Class or + CSharpSemanticTokenKind.Enum or + CSharpSemanticTokenKind.Interface or + CSharpSemanticTokenKind.Struct or + CSharpSemanticTokenKind.TypeParameter || + StartsWithUppercase(word); + } + + private static bool HasDeclarationTerminatorAfter(IReadOnlyList lexemes, int index) + { + if (index + 1 >= lexemes.Count) + return false; + return lexemes[index + 1].Text is "=" or ";" or "," or ")" or "["; + } + + private static bool IsTypeContext(IReadOnlyList lexemes, int index) + { + if (index <= 0) + return false; + var previous = NormalizeIdentifier(lexemes[index - 1].Text); + return previous is "as" or "default" or "is" or "new" or "sizeof" or "typeof"; + } + + private static int FindMatching( + IReadOnlyList lexemes, + int openIndex, + string open, + string close) + { + var depth = 0; + for (var index = openIndex; index < lexemes.Count; index++) + { + if (lexemes[index].Text == open) + depth++; + else if (lexemes[index].Text == close && --depth == 0) + return index; + } + return -1; + } + + private static bool StartsVerbatimString(string line, int index, out int prefixLength) + { + prefixLength = 0; + if (line[index] == '@' && index + 1 < line.Length && line[index + 1] == '"') + prefixLength = 2; + else if (index + 2 < line.Length && + ((line[index] == '$' && line[index + 1] == '@') || + (line[index] == '@' && line[index + 1] == '$')) && + line[index + 2] == '"') + { + prefixLength = 3; + } + return prefixLength > 0; + } + + private static int CountConsecutive(string text, int start, char value) + { + var index = start; + while (index < text.Length && text[index] == value) + index++; + return index - start; + } + + private static int FindRawStringEnd(string line, int start, int quoteCount) + { + for (var index = start; index < line.Length; index++) + { + if (line[index] == '"' && CountConsecutive(line, index, '"') >= quoteCount) + return index + quoteCount; + } + return -1; + } + + private static bool IsIdentifierStart(char value) => + char.IsLetter(value) || value is '_' or '@'; + + private static bool IsIdentifierPart(char value) + { + var category = char.GetUnicodeCategory(value); + return char.IsLetterOrDigit(value) || + value == '_' || + category is UnicodeCategory.NonSpacingMark or + UnicodeCategory.SpacingCombiningMark or + UnicodeCategory.ConnectorPunctuation or + UnicodeCategory.Format; + } + + private static bool StartsWithUppercase(string value) => + value.Length > 0 && char.IsUpper(value[0]); + + private static string NormalizeIdentifier(string value) => value.TrimStart('@'); + + private enum LexemeKind + { + Identifier, + Number, + Punctuation, + } + + private enum StringMode + { + None, + Ordinary, + Verbatim, + Raw, + } + + private readonly record struct Lexeme( + int Line, + int Start, + int Length, + string Text, + LexemeKind Kind); +} diff --git a/tests/CodeIndex.Tests/LspServerTests.cs b/tests/CodeIndex.Tests/LspServerTests.cs index 305e2fd327..6e8e0afe12 100644 --- a/tests/CodeIndex.Tests/LspServerTests.cs +++ b/tests/CodeIndex.Tests/LspServerTests.cs @@ -1027,6 +1027,82 @@ public void HandleMessage_SemanticTokens_ClassifiesCSharpKeywordsModifiersAndDec } } + [Fact] + public void HandleMessage_SemanticTokens_SharesCSharpDeclarationAndUsageKinds_Issue4852() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_lsp_semantic_tokens_4852"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + var sourcePath = Path.Combine(projectRoot, "app.cs"); + var source = string.Join('\n', + [ + "using System.Collections.Generic;", + "namespace Sample.Tools;", + "internal sealed record App(T Value)", + "{", + " private const int Count = 1;", + " public string Name { get; init; } = string.Empty;", + " public void Run(TArg input) where TArg : notnull", + " {", + " var local = new List();", + " Helper(input);", + " }", + " private void Helper(T value) { }", + "}", + ]); + File.WriteAllText(sourcePath, source); + TestProjectHelper.InsertIndexedFile(dbPath, "app.cs", "csharp", source); + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + using var server = new LspServer( + new DbReader(db), + "1.2.3", + ProgramRunner.CreateDefaultJsonOptions(), + projectRoot); + + var response = HandleInitializedMessage( + server, + CreateTextDocumentRequest("textDocument/semanticTokens/full", sourcePath, 4852)); + + Assert.NotNull(response); + var tokens = DecodeSemanticTokens(response!["result"]!["data"]!.AsArray(), source); + AssertSemanticToken(tokens, 0, "using", 15, 0); + AssertSemanticToken(tokens, 0, "System", 0, 0); + AssertSemanticToken(tokens, 0, "Collections", 0, 0); + AssertSemanticToken(tokens, 0, "Generic", 0, 0); + AssertSemanticToken(tokens, 1, "namespace", 15, 0); + AssertSemanticToken(tokens, 1, "Sample", 0, 0); + AssertSemanticToken(tokens, 2, "internal", 16, 0); + AssertSemanticToken(tokens, 2, "sealed", 16, 0); + AssertSemanticToken(tokens, 2, "record", 15, 0); + AssertSemanticToken(tokens, 2, "App", 2, 1); + AssertSemanticToken(tokens, 2, "T", 6, 1); + AssertSemanticToken(tokens, 2, "Value", 7, 1); + AssertSemanticToken(tokens, 4, "private", 16, 0); + AssertSemanticToken(tokens, 4, "Count", 23, 1); + AssertSemanticToken(tokens, 5, "Name", 9, 1); + AssertSemanticToken(tokens, 5, "get", 15, 0); + AssertSemanticToken(tokens, 5, "init", 15, 0); + AssertSemanticToken(tokens, 6, "Run", 13, 1); + AssertSemanticToken(tokens, 6, "TArg", 6, 1); + AssertSemanticToken(tokens, 6, "input", 7, 1); + AssertSemanticToken(tokens, 6, "where", 15, 0); + AssertSemanticToken(tokens, 8, "local", 8, 1); + AssertSemanticToken(tokens, 9, "Helper", 13, 0); + AssertSemanticToken(tokens, 9, "input", 7, 0); + Assert.DoesNotContain( + tokens.SelectMany((left, index) => tokens.Skip(index + 1).Select(right => (left, right))), + pair => + pair.left.Line == pair.right.Line && + pair.left.Character < pair.right.Character + pair.right.Text.Length && + pair.right.Character < pair.left.Character + pair.left.Text.Length); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void HandleMessage_InlayHint_HonorsRangeAndSuppressesExplicitTypes_Issue4418() { diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs index da471fefe6..736c79db05 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs @@ -10432,6 +10432,110 @@ public void RunExcerpt_JsonSemanticTokensIncludesCompactHint_Issue4311() } } + [Fact] + public void RunExcerpt_JsonCSharpSemanticKindsMatchLspClassifier_Issue4852() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_excerpt_semantic_kinds_4852"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + var source = string.Join('\n', + [ + "using System.Collections.Generic;", + "namespace Sample.Tools;", + "internal sealed record App(T Value)", + "{", + " private const int Count = 1;", + " public string Name { get; init; } = string.Empty;", + " public void Run(TArg input) where TArg : notnull", + " {", + " var local = new List();", + " Helper(input);", + " }", + " private void Helper(T value) { }", + "}", + ]); + TestProjectHelper.InsertIndexedFile(dbPath, "src/Sample.cs", "csharp", source); + + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunExcerpt( + ["src/Sample.cs", "--db", dbPath, "--start", "1", "--end", "13", "--json"], + _jsonOptions)); + + using var document = ParseJsonOutput(stdout); + var json = document.RootElement; + var semanticTokens = json.GetProperty("semantic_tokens").EnumerateArray().ToArray(); + var sourceLines = source.Split('\n'); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + AssertExcerptToken(1, "using", "keyword"); + AssertExcerptToken(1, "System", "namespace"); + AssertExcerptToken(2, "Sample", "namespace"); + AssertExcerptToken(3, "internal", "modifier"); + AssertExcerptToken(3, "record", "keyword"); + AssertExcerptToken(3, "App", "class", declaration: true); + AssertExcerptToken(3, "T", "typeParameter", declaration: true); + AssertExcerptToken(3, "Value", "parameter", declaration: true); + AssertExcerptToken(5, "private", "modifier"); + AssertExcerptToken(5, "Count", "field", declaration: true); + AssertExcerptToken(6, "Name", "property", declaration: true); + AssertExcerptToken(7, "Run", "method", declaration: true); + AssertExcerptToken(7, "input", "parameter", declaration: true); + AssertExcerptToken(7, "where", "keyword"); + AssertExcerptToken(9, "local", "variable", declaration: true); + AssertExcerptToken(10, "Helper", "method"); + AssertExcerptToken(10, "input", "parameter"); + + var (partialExitCode, partialStdout, partialStderr) = CaptureConsole(() => QueryCommandRunner.RunExcerpt( + ["src/Sample.cs", "--db", dbPath, "--start", "9", "--end", "10", "--json"], + _jsonOptions)); + using var partialDocument = ParseJsonOutput(partialStdout); + var partialSemanticTokens = partialDocument.RootElement + .GetProperty("semantic_tokens") + .EnumerateArray() + .ToArray(); + Assert.Equal(CommandExitCodes.Success, partialExitCode); + Assert.Equal(string.Empty, partialStderr); + AssertExcerptTokenIn(partialSemanticTokens, 9, "TArg", "typeParameter"); + AssertExcerptTokenIn(partialSemanticTokens, 10, "input", "parameter"); + + Assert.DoesNotContain( + semanticTokens.SelectMany((left, index) => semanticTokens.Skip(index + 1).Select(right => (left, right))), + pair => + pair.left.GetProperty("start_line").GetInt32() == pair.right.GetProperty("start_line").GetInt32() && + pair.left.GetProperty("start_column").GetInt32() < pair.right.GetProperty("end_column").GetInt32() && + pair.right.GetProperty("start_column").GetInt32() < pair.left.GetProperty("end_column").GetInt32()); + + void AssertExcerptToken( + int line, + string text, + string type, + bool declaration = false) + => AssertExcerptTokenIn(semanticTokens, line, text, type, declaration); + + void AssertExcerptTokenIn( + JsonElement[] actualTokens, + int line, + string text, + string type, + bool declaration = false) + { + var column = sourceLines[line - 1].IndexOf(text, StringComparison.Ordinal) + 1; + Assert.Contains(actualTokens, token => + token.GetProperty("start_line").GetInt32() == line && + token.GetProperty("start_column").GetInt32() == column && + token.GetProperty("end_column").GetInt32() == column + text.Length && + token.GetProperty("type").GetString() == type && + token.GetProperty("modifiers").EnumerateArray() + .Any(modifier => modifier.GetString() == "declaration") == declaration); + } + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void RunExcerpt_JsonMaxBytesRejectsOversizedPayload_Issue4311() { diff --git a/tests/CodeIndex.Tests/golden/excerpt.json b/tests/CodeIndex.Tests/golden/excerpt.json index 93a9bebb6d..216cc62fdf 100644 --- a/tests/CodeIndex.Tests/golden/excerpt.json +++ b/tests/CodeIndex.Tests/golden/excerpt.json @@ -68,7 +68,7 @@ "start_column": 1, "end_line": 1, "end_column": 10, - "type": "variable", + "type": "keyword", "modifiers": [] }, { @@ -76,7 +76,7 @@ "start_column": 11, "end_line": 1, "end_column": 15, - "type": "type", + "type": "namespace", "modifiers": [] }, { @@ -84,7 +84,7 @@ "start_column": 1, "end_line": 3, "end_column": 7, - "type": "variable", + "type": "modifier", "modifiers": [] }, { @@ -92,7 +92,7 @@ "start_column": 8, "end_line": 3, "end_column": 14, - "type": "variable", + "type": "modifier", "modifiers": [] }, { @@ -100,7 +100,7 @@ "start_column": 15, "end_line": 3, "end_column": 20, - "type": "variable", + "type": "keyword", "modifiers": [] }, { @@ -108,15 +108,17 @@ "start_column": 21, "end_line": 3, "end_column": 24, - "type": "type", - "modifiers": [] + "type": "class", + "modifiers": [ + "declaration" + ] }, { "start_line": 5, "start_column": 5, "end_line": 5, "end_column": 11, - "type": "variable", + "type": "modifier", "modifiers": [] }, { @@ -124,7 +126,7 @@ "start_column": 12, "end_line": 5, "end_column": 18, - "type": "variable", + "type": "modifier", "modifiers": [] }, { @@ -132,7 +134,7 @@ "start_column": 19, "end_line": 5, "end_column": 22, - "type": "variable", + "type": "keyword", "modifiers": [] }, { @@ -140,15 +142,17 @@ "start_column": 23, "end_line": 5, "end_column": 26, - "type": "type", - "modifiers": [] + "type": "method", + "modifiers": [ + "declaration" + ] }, { "start_line": 5, "start_column": 27, "end_line": 5, "end_column": 30, - "type": "variable", + "type": "keyword", "modifiers": [] }, { @@ -156,15 +160,17 @@ "start_column": 31, "end_line": 5, "end_column": 32, - "type": "variable", - "modifiers": [] + "type": "parameter", + "modifiers": [ + "declaration" + ] }, { "start_line": 5, "start_column": 34, "end_line": 5, "end_column": 37, - "type": "variable", + "type": "keyword", "modifiers": [] }, { @@ -172,15 +178,17 @@ "start_column": 38, "end_line": 5, "end_column": 39, - "type": "variable", - "modifiers": [] + "type": "parameter", + "modifiers": [ + "declaration" + ] }, { "start_line": 5, "start_column": 44, "end_line": 5, "end_column": 45, - "type": "variable", + "type": "parameter", "modifiers": [] }, { @@ -188,7 +196,7 @@ "start_column": 48, "end_line": 5, "end_column": 49, - "type": "variable", + "type": "parameter", "modifiers": [] } ], From 164d08719c84e50513ba663426c7a9ef82e1bcec Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 28 Jul 2026 12:49:47 +0900 Subject: [PATCH 2/3] Address semantic token review findings (#4852) --- DEVELOPER_GUIDE.md | 4 +- TESTING_GUIDE.md | 4 +- changelog.d/unreleased/4852.fixed.md | 4 +- .../Cli/QueryCommandRunner.Excerpt.cs | 64 +- src/CodeIndex/Lsp/LspServer.SemanticTokens.cs | 4 + .../CSharpSemanticTokenClassifier.cs | 624 ++++++++++++++++-- tests/CodeIndex.Tests/LspServerTests.cs | 109 ++- .../QueryCommandRunnerSearchTests.cs | 116 +++- 8 files changed, 815 insertions(+), 114 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 638fdf668f..65fb4bd3f8 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1651,7 +1651,7 @@ When the match line falls inside an indexed symbol range, `search --json` and MC `find --json` remains line-delimited for repeated matches and adds bounded match-span/truncation metadata to each row: `length` reports the 1-based `column` span length, `original_line_length` reports the source line length before any line-width clamp, and `snippet_truncation_context.line_count` / `char_counts` / `total_chars` / optional `reason` describe snippet clamping. `reason` is `line_width` when `--max-line-width` elides one or more snippet lines. -`excerpt --json` includes `semantic_tokens`, a lightweight range list with 1-based source start/end positions, token `type`, and `modifiers`, so IDE and LLM clients can render or post-process excerpt spans without reparsing the raw `content` string. C# excerpts and LSP `textDocument/semanticTokens/full` share the same source classifier for keywords/modifiers, namespaces and types, methods and properties, parameters, variables and fields, and declaration modifiers; excerpt classification uses the full indexed source before filtering to the visible range, so a narrow or clamped excerpt retains declaration context. Excerpt range mapping and LSP delta encoding only translate coordinates and do not choose semantic kinds. `semantic_token_coordinate_space` is `source`; when `--max-line-width` clamps returned content, `content_line_spans` maps each returned content line and visible content-column span back to the matching source line and source-column span, while clamp markers remain unmapped and are not emitted as semantic tokens. Excerpt rows also expose `requested_start_line`, `requested_end_line`, `effective_start_line`, `effective_end_line`, `content_truncation_reasons`, and optional `content_recovery` so clients can tell when `--max-line-width` caused `line_width_cap` and replay the omitted text. `content_recovery` and the matching `body_content_recovery` use `argv` as their primary machine-readable contract; the array begins with the current apphost or `dotnet` plus the running assembly, so recovery stays on the same build and database contract. Root-level paths beginning with `-` are preceded by the supported `--` end-of-options marker so the recovery CLI does not parse them as options. The legacy `command` is display-only, accompanied by `command_display_only: true` and `command_shell` (`posix-sh` or `powershell`), and is escaped for that declared shell. MCP emits the equivalent camelCase field names. Body-bearing JSON rows also use matching `body_requested_*`, `body_effective_*`, and `body_content_truncation_reasons` fields; body reasons include `body_line_cap` for snippet/body line caps and `body_byte_cap` for definition body byte caps. +`excerpt --json` includes `semantic_tokens`, a lightweight range list with 1-based source start/end positions, token `type`, and `modifiers`, so IDE and LLM clients can render or post-process excerpt spans without reparsing the raw `content` string. C# excerpts and LSP `textDocument/semanticTokens/full` share the same source classifier for keywords/modifiers, namespaces and types, methods and properties, parameters, variables and fields, and declaration modifiers. Excerpt classification uses indexed-source context, applies the output token budget only after filtering to visible source lines, and falls back to classifying the visible content when a bounded source scan cannot reach it; narrow or late-file excerpts therefore retain context when available without becoming empty because earlier tokens exhausted the output budget. Excerpt range mapping and LSP delta encoding only translate coordinates and do not choose semantic kinds. `semantic_token_coordinate_space` is `source`; when `--max-line-width` clamps returned content, `content_line_spans` maps each returned content line and visible content-column span back to the matching source line and source-column span, while clamp markers remain unmapped and are not emitted as semantic tokens. Excerpt rows also expose `requested_start_line`, `requested_end_line`, `effective_start_line`, `effective_end_line`, `content_truncation_reasons`, and optional `content_recovery` so clients can tell when `--max-line-width` caused `line_width_cap` and replay the omitted text. `content_recovery` and the matching `body_content_recovery` use `argv` as their primary machine-readable contract; the array begins with the current apphost or `dotnet` plus the running assembly, so recovery stays on the same build and database contract. Root-level paths beginning with `-` are preceded by the supported `--` end-of-options marker so the recovery CLI does not parse them as options. The legacy `command` is display-only, accompanied by `command_display_only: true` and `command_shell` (`posix-sh` or `powershell`), and is escaped for that declared shell. MCP emits the equivalent camelCase field names. Body-bearing JSON rows also use matching `body_requested_*`, `body_effective_*`, and `body_content_truncation_reasons` fields; body reasons include `body_line_cap` for snippet/body line caps and `body_byte_cap` for definition body byte caps. `inspect` and MCP `analyze_symbol` bundle the primary definition, nearby symbols from the same file, references, callers, callees, file metadata, workspace freshness/git metadata, and graph-support metadata into one response. When those bundled graph sections actually depend on SQL-backed reads, the payload also mirrors `sql_graph_contract_ready` / `sql_graph_contract_degraded_reason` (plus the existing camelCase aliases on MCP responses); mixed-language bundles that only return C# / JS / etc. graph rows omit the SQL trust signal entirely. This is intended for symbol-oriented AI workflows that would otherwise need several back-to-back calls. Call graph sections remain language-aware: for unsupported languages, clients can now distinguish "unsupported" from "no hits" via `graphSupported` / `graphSupportReason`, and should prefer `search` instead of assuming graph data will exist. @@ -4887,7 +4887,7 @@ literal-safe な `search` query は reader 層で FTS5 sanitization 前に 1000 `find --json` は繰り返し一致でも line-delimited のまま維持し、各 row に bounded な match span / truncation metadata を追加します。`length` は 1-based の `column` から始まる一致長、`original_line_length` は行幅クランプ前のソース行長、`snippet_truncation_context.line_count` / `char_counts` / `total_chars` / 任意の `reason` は snippet クランプを表します。`--max-line-width` によって snippet 行が省略された場合、`reason` は `line_width` になります。 -`excerpt --json` は 1-based の source 開始/終了位置、token `type`、`modifiers` を持つ軽量 range list の `semantic_tokens` を返すため、IDE や LLM クライアントは生の `content` 文字列を再パースせずに抜粋範囲を描画・後処理できます。C# の excerpt と LSP `textDocument/semanticTokens/full` は、keyword/modifier、namespace と type、method と property、parameter、variable と field、declaration modifier を判定する同じ source classifier を共有します。excerpt は indexed source 全体を分類してから可視範囲へ絞るため、狭い範囲やクランプされた範囲でも declaration context を維持します。excerpt の range mapping と LSP の delta encoding は座標変換だけを担当し、semantic kind を選びません。`semantic_token_coordinate_space` は `source` です。`--max-line-width` で返却内容がクランプされた場合、`content_line_spans` は返却 content 行と可視 content column span を、対応する source 行と source column span に対応付けます。clamp marker は未対応領域として扱い、semantic token には含めません。excerpt row は `requested_start_line`、`requested_end_line`、`effective_start_line`、`effective_end_line`、`content_truncation_reasons`、任意の `content_recovery` も返すため、`--max-line-width` による `line_width_cap` を検出して省略部分を再取得できます。`content_recovery` と対応する `body_content_recovery` では、`argv` が一次的な機械可読契約です。この配列は現在の apphost、または `dotnet` と実行中 assembly の組から始まるため、同じ build と database contract で復旧できます。先頭が `-` の root-level path の前には、対応済みの `--` end-of-options marker を置き、復旧 CLI が option として解釈しないようにします。従来の `command` は表示専用で、`command_display_only: true` と `command_shell`（`posix-sh` または `powershell`）を伴い、宣言した shell 向けに escape されます。MCP では同等の camelCase field 名を返します。body を持つ JSON row は対応する `body_requested_*`、`body_effective_*`、`body_content_truncation_reasons` も返します。body reason には snippet/body 行数上限の `body_line_cap` と definition body byte 上限の `body_byte_cap` があります。 +`excerpt --json` は 1-based の source 開始/終了位置、token `type`、`modifiers` を持つ軽量 range list の `semantic_tokens` を返すため、IDE や LLM クライアントは生の `content` 文字列を再パースせずに抜粋範囲を描画・後処理できます。C# の excerpt と LSP `textDocument/semanticTokens/full` は、keyword/modifier、namespace と type、method と property、parameter、variable と field、declaration modifier を判定する同じ source classifier を共有します。excerpt の分類は indexed source の context を利用し、出力 token budget を可視 source 行へ絞った後に適用します。bounded source scan が可視範囲まで到達できない場合は可視 content の分類へ fallback するため、狭い範囲では利用可能な context を維持し、file 後半の excerpt が手前の token に出力 budget を消費されて空になることも防ぎます。excerpt の range mapping と LSP の delta encoding は座標変換だけを担当し、semantic kind を選びません。`semantic_token_coordinate_space` は `source` です。`--max-line-width` で返却内容がクランプされた場合、`content_line_spans` は返却 content 行と可視 content column span を、対応する source 行と source column span に対応付けます。clamp marker は未対応領域として扱い、semantic token には含めません。excerpt row は `requested_start_line`、`requested_end_line`、`effective_start_line`、`effective_end_line`、`content_truncation_reasons`、任意の `content_recovery` も返すため、`--max-line-width` による `line_width_cap` を検出して省略部分を再取得できます。`content_recovery` と対応する `body_content_recovery` では、`argv` が一次的な機械可読契約です。この配列は現在の apphost、または `dotnet` と実行中 assembly の組から始まるため、同じ build と database contract で復旧できます。先頭が `-` の root-level path の前には、対応済みの `--` end-of-options marker を置き、復旧 CLI が option として解釈しないようにします。従来の `command` は表示専用で、`command_display_only: true` と `command_shell`（`posix-sh` または `powershell`）を伴い、宣言した shell 向けに escape されます。MCP では同等の camelCase field 名を返します。body を持つ JSON row は対応する `body_requested_*`、`body_effective_*`、`body_content_truncation_reasons` も返します。body reason には snippet/body 行数上限の `body_line_cap` と definition body byte 上限の `body_byte_cap` があります。 `inspect` と MCP の `analyze_symbol` は、主定義、同一ファイル内の近傍シンボル、参照、caller、callee、ファイルメタデータ、さらにワークスペース鮮度/git メタデータと graph 対応メタデータを1レスポンスにまとめます。bundle 内の graph 節が実際に SQL ベースの read に依存する場合だけ、`sql_graph_contract_ready` / `sql_graph_contract_degraded_reason`（MCP では既存の camelCase alias も）も返します。mixed-language bundle で C# / JS などの graph row しか返っていない場合は SQL trust signal を出さないため、無関係なクエリが stale SQL state に引きずられません。複数の連続クエリを避けたい AI ワークフロー向けです。call graph 系の節は言語差分を考慮しており、未対応言語では `graphSupported` / `graphSupportReason` によって「未対応」と「ヒットなし」を区別できます。その場合は `search` を優先して使う前提です。 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 6921fccb2e..fb2443d3cf 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -520,7 +520,7 @@ Use `docs/test-doc-maintenance-plan.md` before moving oversized suites or adding indexes one real C# fixture containing user-defined generic classes and delegates, block-comment generic trivia, repeated same-line generic names under indentation, unrelated same-name properties, ordinal-case type/property collisions, `Action` / `Stream` references, and uppercase property receivers split across partial-class and inherited-base files. Keep CLI `references`, `inspect`, and `deps` assertions together with the MCP `references`, `analyze_symbol`, and `deps` wrappers so unresolved diagnostics, generic-arity selection, case-sensitive receiver handling, false dependency-edge suppression, and preservation of real member dependencies cannot drift across surfaces. - `LspServerTests.cs` LSP JSON-RPC behavior and indexed editor semantics. Reference parity coverage must compare LSP locations with the matching CLI candidate bundle, including overload identity and both `includeDeclaration` states. Keep document/workspace identifier ranges and explicit-versus-inferred inlay hints in one source-semantics fixture so persisted-column anchoring, source confirmation, and hint suppression cannot drift independently. - C# semantic-token coverage decodes LSP delta coordinates and pairs the same keyword/modifier, namespace/type, method/property, parameter/variable, generic/record, and declaration-kind fixture with `QueryCommandRunnerTests.RunExcerpt_JsonCSharpSemanticKindsMatchLspClassifier_Issue4852`; keep both surfaces on the shared source classifier, retain ordering/non-overlap assertions, and preserve the narrow excerpt check whose parameter/type-parameter declarations are outside the returned range. + C# semantic-token coverage decodes LSP delta coordinates and pairs the same keyword/modifier, namespace/type, method/property/event/enum-member, parameter/variable/field, generic/record-struct, explicit-interface-member, interpolation, comma-separated declaration, and declaration-kind fixture with `QueryCommandRunnerTests.RunExcerpt_JsonCSharpSemanticKindsMatchLspClassifier_Issue4852`. Keep both surfaces on the shared source classifier, retain ordering/non-overlap assertions, preserve the narrow excerpt check whose parameter/type-parameter declarations are outside the returned range, and retain `RunExcerpt_JsonCSharpSemanticTokensUseVisibleRangeBudget_Issue4852` so earlier tokens cannot empty a late excerpt. `HandleMessage_SemanticTokens_FallsBackToIndexedCSharpSymbolsWhenSourceReadFails_Issue4852` keeps indexed declaration tokens available when bounded source loading fails. Document-symbol hierarchy coverage keeps positional and body record members in one same-line fixture, asserts the complete root-to-member shape and deterministic sibling order, and retains the separate same-range top-level controls so order-independent parent resolution cannot over-nest unrelated symbols. Keep a same-line duplicate-container fixture with distinct positional members so selection-column disambiguation cannot move an earlier member beneath a later same-named container. Symbol-progress coverage crosses the production chunk item limit with one sentinel, verifies the complete deterministic sequence across partial-result frames, and keeps live work-done delivery, truncation, typed request-ID cancellation, full-queue cancellation, queue-pressure notification preservation, backpressured `Server busy` response retention, emitted-count cancellation reporting, output-failure propagation, and invalid-token variants in the same protocol fixture. Assert both the item and UTF-8 JSON body budgets for every emitted chunk. The server-busy backpressure cancellation fixture must stage the cancel frame until both the active symbol request and blocked response write have been observed. Block the request on its cancellation token instead of matching fixed delays so suite load cannot convert the expected cancellation into a successful response. @@ -1416,7 +1416,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" は user-defined generic class / delegate、generic 間の block comment trivia、indent 下で同一行に反復する同名 generic、無関係な同名 property、大文字小文字だけ異なる type / property collision、`Action` / `Stream` reference、partial class の別ファイルおよび継承 base file に分かれた大文字始まりの property receiver を含む実 C# fixture を1回 index します。CLI の `references`、`inspect`、`deps` と MCP wrapper の `references`、`analyze_symbol`、`deps` を同じ fixture で検証し、unresolved 診断、generic arity による選択、case-sensitive receiver 判定、偽 dependency edge の抑制、実在する member dependency の維持が surface 間でずれないよう維持してください。 - `LspServerTests.cs` LSP の JSON-RPC 挙動と indexed editor semantics のテスト。reference parity coverage では、overload identity と `includeDeclaration` の両状態を含め、LSP location と対応する CLI candidate bundle を比較してください。document/workspace の identifier range と explicit/inferred inlay hint は 1 つの source-semantics fixture にまとめ、保存済み column の anchoring、source 上の確認、hint 抑制が別々に drift しないようにします。 - C# semantic-token coverage は LSP の delta 座標を decode し、同じ keyword/modifier、namespace/type、method/property、parameter/variable、generic/record、declaration kind の fixture を `QueryCommandRunnerTests.RunExcerpt_JsonCSharpSemanticKindsMatchLspClassifier_Issue4852` と対にします。両 surface が shared source classifier を使い続け、順序と非 overlap の assertion、および parameter / type-parameter の declaration が返却範囲外にある narrow excerpt の検証を維持してください。 + C# semantic-token coverage は LSP の delta 座標を decode し、同じ keyword/modifier、namespace/type、method/property/event/enum-member、parameter/variable/field、generic/record struct、explicit interface member、interpolation、comma-separated declaration、declaration kind の fixture を `QueryCommandRunnerTests.RunExcerpt_JsonCSharpSemanticKindsMatchLspClassifier_Issue4852` と対にします。両 surface が shared source classifier を使い続け、順序と非 overlap の assertion、parameter / type-parameter の declaration が返却範囲外にある narrow excerpt の検証、および手前の token によって file 後半の excerpt が空にならないことを保証する `RunExcerpt_JsonCSharpSemanticTokensUseVisibleRangeBudget_Issue4852` を維持してください。`HandleMessage_SemanticTokens_FallsBackToIndexedCSharpSymbolsWhenSourceReadFails_Issue4852` は bounded source loading が失敗しても indexed declaration token を利用できる状態を維持します。 document-symbol hierarchy coverage は positional member と body member を同一行の record fixture にまとめ、root から member までの完全な形状と決定的な sibling 順序を assert します。順序非依存の親解決が無関係な symbol を過剰に nest しないよう、same-range top-level の control は独立したまま維持してください。selection column による曖昧性解消で前の member が行内で後にある同名 container の配下へ移動しないよう、異なる positional member を持つ同一行の duplicate-container fixture も維持してください。 symbol-progress coverage は production の chunk item limit を sentinel 1 件だけで超え、partial-result frame 全体の決定的な完全 sequence を検証します。live work-done delivery、truncation、型付き request ID の cancellation、full-queue cancellation、queue pressure 下の notification 保持、backpressure された `Server busy` response の保持、cancellation 時の送信済み件数、output-failure propagation、invalid-token variant は同じ protocol fixture にまとめ、送信した全 chunk の item 上限と UTF-8 JSON body budget の両方を assert してください。 server-busy backpressure cancellation fixture では、active symbol request と block された response write の両方を観測するまで cancel frame を stage してください。固定 delay の一致に依存せず cancellation token 上で request を block することで、suite load により期待する cancellation が成功 response に変わる競合を防ぎます。 diff --git a/changelog.d/unreleased/4852.fixed.md b/changelog.d/unreleased/4852.fixed.md index 9a198c61ae..e2e329a8a1 100644 --- a/changelog.d/unreleased/4852.fixed.md +++ b/changelog.d/unreleased/4852.fixed.md @@ -19,8 +19,8 @@ affected: ## English -- **Restored consistent C# semantic token kinds across LSP and excerpt output (#4852)** — a shared source classifier now identifies keywords and modifiers, namespaces and types, methods and properties, parameters, variables and fields, generic/record constructs, contextual keywords, and declarations before LSP delta encoding or excerpt source-coordinate mapping. Excerpts classify the full indexed source before filtering visible spans, preserving declaration context outside narrow ranges. This fixes the direct regression of #4444 where `private` and `using` appeared as variables and method names could appear as types. +- **Restored consistent C# semantic token kinds across LSP and excerpt output (#4852)** — a shared source classifier now identifies keywords and modifiers, namespaces and types, methods, properties, events and enum members, parameters, variables and fields, generic/record constructs, contextual identifiers, interpolated expressions, explicit interface members, comma-separated declarations, and declaration modifiers before LSP delta encoding or excerpt source-coordinate mapping. Excerpts apply their token budget after filtering indexed-source classifications to visible lines and retain a bounded visible-range fallback, while LSP retains indexed declaration tokens if source loading fails. This fixes the direct regression of #4444 where `private` and `using` appeared as variables and method names could appear as types. ## 日本語 -- **LSP と excerpt 出力で一貫した C# semantic token kind を復元しました (#4852)** — shared source classifier が keyword / modifier、namespace / type、method / property、parameter、variable / field、generic / record 構文、contextual keyword、declaration を判定してから、LSP の delta encoding または excerpt の source 座標 mapping を行うようになりました。excerpt は indexed source 全体を分類してから可視 span へ絞り、狭い範囲の外にある declaration context も維持します。`private` と `using` が variable、method 名が type として表示されることがあった #4444 の直接 regression を修正します。 +- **LSP と excerpt 出力で一貫した C# semantic token kind を復元しました (#4852)** — shared source classifier が keyword / modifier、namespace / type、method / property / event / enum member、parameter、variable / field、generic / record 構文、contextual identifier、interpolation 内の式、explicit interface member、comma-separated declaration、declaration modifier を判定してから、LSP の delta encoding または excerpt の source 座標 mapping を行うようになりました。excerpt は indexed-source の分類を可視行へ絞った後に token budget を適用し、bounded な可視範囲 fallback も維持します。LSP は source loading が失敗した場合に indexed declaration token を利用します。`private` と `using` が variable、method 名が type として表示されることがあった #4444 の直接 regression を修正します。 diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Excerpt.cs b/src/CodeIndex/Cli/QueryCommandRunner.Excerpt.cs index c998aa2e68..fd806ba3c9 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Excerpt.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Excerpt.cs @@ -233,11 +233,24 @@ private static List BuildExcerptSemanticTokens( ? reader.GetIndexedSourceLinesForSemanticTokens(excerpt.Path) : []; var classifiesIndexedSource = indexedSourceLines.Count > 0; + var includedSourceLines = classifiesIndexedSource + ? spans + .Where(span => span.SourceLine > 0) + .Select(span => span.SourceLine - 1) + .ToHashSet() + : null; var classifiedCSharpTokens = isCSharp ? CSharpSemanticTokenClassifier.Classify( classifiesIndexedSource ? indexedSourceLines : lines, + CSharpSemanticTokenClassifier.DefaultExcerptTokenLimit, + includedSourceLines) + : []; + var visibleCSharpTokens = isCSharp && classifiesIndexedSource + ? CSharpSemanticTokenClassifier.Classify( + lines, CSharpSemanticTokenClassifier.DefaultExcerptTokenLimit) : []; + var csharpTokenRanges = new HashSet<(int Line, int StartColumn, int EndColumn)>(); foreach (var span in spans) { if (span.ContentLine <= 0 || span.ContentLine > lines.Length) @@ -269,15 +282,25 @@ private static List BuildExcerptSemanticTokens( var sourceStartColumn = span.SourceStartColumn + token.StartCharacter - classifiedStartColumn; - tokens.Add(new ExcerptSemanticToken + AddCSharpToken(token, sourceStartColumn); + } + + if (classifiesIndexedSource) + { + foreach (var token in visibleCSharpTokens) { - StartLine = span.SourceLine, - StartColumn = sourceStartColumn, - EndLine = span.SourceLine, - EndColumn = sourceStartColumn + token.Length, - Type = CSharpSemanticTokenClassifier.ToProtocolName(token.Kind), - Modifiers = token.IsDeclaration ? ["declaration"] : [], - }); + if (token.Line != span.ContentLine - 1 || + token.StartCharacter < startColumn || + token.StartCharacter + token.Length > endColumn) + { + continue; + } + + var sourceStartColumn = span.SourceStartColumn + + token.StartCharacter - + startColumn; + AddCSharpToken(token, sourceStartColumn); + } } continue; } @@ -308,6 +331,31 @@ private static List BuildExcerptSemanticTokens( Type = ClassifySemanticToken(tokenText), }); } + + void AddCSharpToken( + ClassifiedCSharpSemanticToken token, + int sourceStartColumn) + { + var sourceEndColumn = sourceStartColumn + token.Length; + if (tokens.Count >= CSharpSemanticTokenClassifier.DefaultExcerptTokenLimit || + !csharpTokenRanges.Add(( + span.SourceLine, + sourceStartColumn, + sourceEndColumn))) + { + return; + } + + tokens.Add(new ExcerptSemanticToken + { + StartLine = span.SourceLine, + StartColumn = sourceStartColumn, + EndLine = span.SourceLine, + EndColumn = sourceEndColumn, + Type = CSharpSemanticTokenClassifier.ToProtocolName(token.Kind), + Modifiers = token.IsDeclaration ? ["declaration"] : [], + }); + } } return tokens; diff --git a/src/CodeIndex/Lsp/LspServer.SemanticTokens.cs b/src/CodeIndex/Lsp/LspServer.SemanticTokens.cs index 4bff351107..5b27c215e6 100644 --- a/src/CodeIndex/Lsp/LspServer.SemanticTokens.cs +++ b/src/CodeIndex/Lsp/LspServer.SemanticTokens.cs @@ -22,7 +22,11 @@ internal sealed partial class LspServer : IDisposable private IEnumerable BuildCSharpSemanticTokens(IndexedDocumentContext document) { if (!TryReadAllPositionLines(document.ResolvedPath, out var sourceLines)) + { + foreach (var indexedToken in BuildIndexedSemanticTokens(document)) + yield return indexedToken; yield break; + } foreach (var token in CSharpSemanticTokenClassifier.Classify(sourceLines, MaxSemanticTokenItems)) { diff --git a/src/CodeIndex/Semantics/CSharpSemanticTokenClassifier.cs b/src/CodeIndex/Semantics/CSharpSemanticTokenClassifier.cs index cdbb4da370..e61988eefa 100644 --- a/src/CodeIndex/Semantics/CSharpSemanticTokenClassifier.cs +++ b/src/CodeIndex/Semantics/CSharpSemanticTokenClassifier.cs @@ -54,7 +54,7 @@ internal static class CSharpSemanticTokenClassifier "partial", "private", "protected", "public", "readonly", "record", "ref", "remove", "required", "return", "sbyte", "scoped", "sealed", "select", "set", "short", "sizeof", "stackalloc", "static", "string", "struct", "switch", "this", "throw", "true", "try", "typeof", "uint", - "ulong", "unchecked", "unmanaged", "unsafe", "using", "ushort", "value", "var", "virtual", + "ulong", "unchecked", "unmanaged", "unsafe", "using", "ushort", "var", "virtual", "void", "volatile", "when", "where", "while", "with", "yield", }; @@ -71,7 +71,13 @@ internal static class CSharpSemanticTokenClassifier internal static IReadOnlyList Classify( IReadOnlyList sourceLines, - int maxTokens) + int maxTokens) => + Classify(sourceLines, maxTokens, includedLines: null); + + internal static IReadOnlyList Classify( + IReadOnlyList sourceLines, + int maxTokens, + IReadOnlySet? includedLines) { if (maxTokens <= 0 || sourceLines.Count == 0) return []; @@ -87,7 +93,10 @@ internal static IReadOnlyList Classify( var declaredParameterNames = new HashSet(StringComparer.Ordinal); var declaredPropertyNames = new HashSet(StringComparer.Ordinal); var declaredFieldNames = new HashSet(StringComparer.Ordinal); + var declaredEnumMemberNames = new HashSet(StringComparer.Ordinal); + var declaredEventNames = new HashSet(StringComparer.Ordinal); var methodBodies = new List<(int Start, int End)>(); + var enumBodies = new List<(int Start, int End)>(); ClassifyKeywordsAndNumbers(lexemes, kinds); ClassifyDirectives(lexemes, kinds); @@ -97,7 +106,19 @@ internal static IReadOnlyList Classify( declarations, declaredTypeNames, declaredTypeParameterNames, - declaredParameterNames); + declaredParameterNames, + enumBodies); + ClassifyEnumMembers( + lexemes, + kinds, + declarations, + enumBodies, + declaredEnumMemberNames); + ClassifyEvents( + lexemes, + kinds, + declarations, + declaredEventNames); ClassifyProperties( lexemes, kinds, @@ -124,7 +145,9 @@ internal static IReadOnlyList Classify( declaredTypeParameterNames, declaredParameterNames, declaredPropertyNames, - declaredFieldNames); + declaredFieldNames, + declaredEnumMemberNames, + declaredEventNames); var result = new List(Math.Min(maxTokens, lexemes.Count)); for (var index = 0; index < lexemes.Count && result.Count < maxTokens; index++) @@ -133,6 +156,8 @@ internal static IReadOnlyList Classify( continue; var lexeme = lexemes[index]; + if (includedLines != null && !includedLines.Contains(lexeme.Line)) + continue; result.Add(new ClassifiedCSharpSemanticToken( lexeme.Line, lexeme.Start, @@ -238,19 +263,34 @@ private static void ClassifyTypeDeclarations( bool[] declarations, HashSet declaredTypeNames, HashSet declaredTypeParameterNames, - HashSet declaredParameterNames) + HashSet declaredParameterNames, + List<(int Start, int End)> enumBodies) { for (var index = 0; index < lexemes.Count; index++) { var keyword = NormalizeIdentifier(lexemes[index].Text); if (!TypeDeclarationKeywords.Contains(keyword)) continue; + if (keyword is "class" or "struct" && + index > 0 && + NormalizeIdentifier(lexemes[index - 1].Text) == "record") + { + continue; + } - var nameIndex = FindNextUnclassifiedIdentifier(lexemes, kinds, index + 1); + var recordKind = keyword == "record" && + index + 1 < lexemes.Count && + NormalizeIdentifier(lexemes[index + 1].Text) is "class" or "struct" + ? NormalizeIdentifier(lexemes[index + 1].Text) + : null; + var nameIndex = FindNextUnclassifiedIdentifier( + lexemes, + kinds, + index + (recordKind == null ? 1 : 2)); if (nameIndex < 0) continue; - var kind = keyword switch + var kind = (recordKind ?? keyword) switch { "enum" => CSharpSemanticTokenKind.Enum, "interface" => CSharpSemanticTokenKind.Interface, @@ -277,6 +317,108 @@ private static void ClassifyTypeDeclarations( openParen, declaredParameterNames); } + + if (kind == CSharpSemanticTokenKind.Enum && + TryFindTypeBodyRange(lexemes, nameIndex, out var enumBody)) + { + enumBodies.Add(enumBody); + } + } + } + + private static void ClassifyEnumMembers( + IReadOnlyList lexemes, + CSharpSemanticTokenKind?[] kinds, + bool[] declarations, + IReadOnlyList<(int Start, int End)> enumBodies, + HashSet declaredEnumMemberNames) + { + foreach (var body in enumBodies) + { + var segmentStart = body.Start + 1; + var parenthesisDepth = 0; + var bracketDepth = 0; + var braceDepth = 0; + for (var index = segmentStart; index <= body.End; index++) + { + var text = lexemes[index].Text; + if (text == "(") + parenthesisDepth++; + else if (text == ")" && parenthesisDepth > 0) + parenthesisDepth--; + else if (text == "[") + bracketDepth++; + else if (text == "]" && bracketDepth > 0) + bracketDepth--; + else if (text == "{") + braceDepth++; + else if (text == "}" && braceDepth > 0) + braceDepth--; + + var atSegmentEnd = index == body.End || + (text == "," && + parenthesisDepth == 0 && + bracketDepth == 0 && + braceDepth == 0); + if (!atSegmentEnd) + continue; + + var memberIndex = FindLastIdentifierBeforeEquals( + lexemes, + segmentStart, + index); + if (memberIndex >= 0 && !kinds[memberIndex].HasValue) + { + kinds[memberIndex] = CSharpSemanticTokenKind.EnumMember; + declarations[memberIndex] = true; + declaredEnumMemberNames.Add( + NormalizeIdentifier(lexemes[memberIndex].Text)); + } + segmentStart = index + 1; + } + } + } + + private static void ClassifyEvents( + IReadOnlyList lexemes, + CSharpSemanticTokenKind?[] kinds, + bool[] declarations, + HashSet declaredEventNames) + { + for (var eventIndex = 0; eventIndex < lexemes.Count; eventIndex++) + { + if (NormalizeIdentifier(lexemes[eventIndex].Text) != "event") + continue; + + var segmentStart = eventIndex + 1; + var angleDepth = 0; + for (var index = segmentStart; index < lexemes.Count; index++) + { + var text = lexemes[index].Text; + if (text == "<") + angleDepth++; + else if (text == ">" && angleDepth > 0) + angleDepth--; + + var atSegmentEnd = angleDepth == 0 && text is "," or ";" or "{"; + if (!atSegmentEnd) + continue; + + var nameIndex = FindLastIdentifierBeforeEquals( + lexemes, + segmentStart, + index); + if (nameIndex >= 0 && !kinds[nameIndex].HasValue) + { + kinds[nameIndex] = CSharpSemanticTokenKind.Event; + declarations[nameIndex] = true; + declaredEventNames.Add(NormalizeIdentifier(lexemes[nameIndex].Text)); + } + + if (text is ";" or "{") + break; + segmentStart = index + 1; + } } } @@ -290,7 +432,8 @@ private static void ClassifyProperties( { if (lexemes[index].Kind != LexemeKind.Identifier || kinds[index].HasValue || - !IsDeclarationTypeBefore(lexemes, kinds, index)) + (!IsDeclarationTypeBefore(lexemes, kinds, index) && + !IsExplicitInterfaceMemberDeclaration(lexemes, kinds, index))) { continue; } @@ -335,7 +478,8 @@ private static void ClassifyMethods( var name = NormalizeIdentifier(lexemes[index].Text); var isDeclaration = declaredTypeNames.Contains(name) || - IsDeclarationTypeBefore(lexemes, kinds, index); + IsDeclarationTypeBefore(lexemes, kinds, index) || + IsExplicitInterfaceMemberDeclaration(lexemes, kinds, index); kinds[index] = CSharpSemanticTokenKind.Method; declarations[index] = isDeclaration; @@ -368,18 +512,30 @@ private static void ClassifyVariablesAndFields( { for (var index = 0; index < lexemes.Count; index++) { - if (lexemes[index].Kind != LexemeKind.Identifier || - kinds[index].HasValue || - !IsDeclarationTypeBefore(lexemes, kinds, index) || - !HasDeclarationTerminatorAfter(lexemes, index)) + if (lexemes[index].Kind != LexemeKind.Identifier || kinds[index].HasValue) + { + continue; + } + + CSharpSemanticTokenKind kind; + if (IsDeclarationTypeBefore(lexemes, kinds, index) && + HasDeclarationTerminatorAfter(lexemes, index)) + { + var insideMethod = methodBodies.Any( + body => index > body.Start && index < body.End); + kind = insideMethod + ? CSharpSemanticTokenKind.Variable + : CSharpSemanticTokenKind.Field; + } + else if (!TryGetCommaSeparatedDeclarationKind( + lexemes, + kinds, + index, + out kind)) { continue; } - var insideMethod = methodBodies.Any(body => index > body.Start && index < body.End); - var kind = insideMethod - ? CSharpSemanticTokenKind.Variable - : CSharpSemanticTokenKind.Field; kinds[index] = kind; declarations[index] = true; if (kind == CSharpSemanticTokenKind.Field) @@ -394,7 +550,9 @@ private static void ClassifyRemainingIdentifiers( IReadOnlySet declaredTypeParameterNames, IReadOnlySet declaredParameterNames, IReadOnlySet declaredPropertyNames, - IReadOnlySet declaredFieldNames) + IReadOnlySet declaredFieldNames, + IReadOnlySet declaredEnumMemberNames, + IReadOnlySet declaredEventNames) { for (var index = 0; index < lexemes.Count; index++) { @@ -402,7 +560,16 @@ private static void ClassifyRemainingIdentifiers( continue; var word = NormalizeIdentifier(lexemes[index].Text); - if (declaredTypeParameterNames.Contains(word)) + if (IsQualifiedInstanceMemberAccess(lexemes, index)) + { + if (declaredFieldNames.Contains(word)) + kinds[index] = CSharpSemanticTokenKind.Field; + else if (declaredEventNames.Contains(word)) + kinds[index] = CSharpSemanticTokenKind.Event; + else + kinds[index] = CSharpSemanticTokenKind.Property; + } + else if (declaredTypeParameterNames.Contains(word)) { kinds[index] = CSharpSemanticTokenKind.TypeParameter; } @@ -418,6 +585,14 @@ private static void ClassifyRemainingIdentifiers( { kinds[index] = CSharpSemanticTokenKind.Field; } + else if (declaredEnumMemberNames.Contains(word)) + { + kinds[index] = CSharpSemanticTokenKind.EnumMember; + } + else if (declaredEventNames.Contains(word)) + { + kinds[index] = CSharpSemanticTokenKind.Event; + } else if (declaredTypeNames.Contains(word) || IsTypeContext(lexemes, index) || StartsWithUppercase(word)) @@ -441,8 +616,14 @@ private static List Tokenize(IReadOnlyList sourceLines, int max var result = new List(Math.Min(maxLexemes, 16_384)); var inBlockComment = false; var stringMode = StringMode.None; + var stringIsInterpolated = false; + var interpolationDepth = 0; + var interpolationBraceCount = 1; var rawQuoteCount = 0; var ordinaryQuote = '\0'; + var nestedStringMode = StringMode.None; + var nestedRawQuoteCount = 0; + var nestedOrdinaryQuote = '\0'; for (var lineIndex = 0; lineIndex < sourceLines.Count && result.Count < maxLexemes; lineIndex++) { @@ -459,43 +640,180 @@ private static List Tokenize(IReadOnlyList sourceLines, int max continue; } - if (stringMode == StringMode.Raw) + if (stringMode != StringMode.None && + (!stringIsInterpolated || interpolationDepth == 0)) { - var end = FindRawStringEnd(line, index, rawQuoteCount); - if (end < 0) - break; - stringMode = StringMode.None; - index = end; - continue; - } + if (stringMode == StringMode.Raw) + { + if (line[index] == '"' && + CountConsecutive(line, index, '"') >= rawQuoteCount) + { + index += rawQuoteCount; + stringMode = StringMode.None; + stringIsInterpolated = false; + continue; + } + if (stringIsInterpolated && + line[index] == '{' && + CountConsecutive(line, index, '{') >= interpolationBraceCount) + { + interpolationDepth = 1; + index += interpolationBraceCount; + continue; + } + index++; + continue; + } - if (stringMode == StringMode.Verbatim) - { - var end = line.IndexOf('"', index); - if (end < 0) - break; - if (end + 1 < line.Length && line[end + 1] == '"') + if (stringMode == StringMode.Verbatim) { - index = end + 2; + if (stringIsInterpolated && line[index] == '{') + { + if (index + 1 < line.Length && line[index + 1] == '{') + { + index += 2; + continue; + } + interpolationDepth = 1; + index++; + continue; + } + if (line[index] == '"' && + index + 1 < line.Length && + line[index + 1] == '"') + { + index += 2; + continue; + } + if (line[index++] == '"') + { + stringMode = StringMode.None; + stringIsInterpolated = false; + } continue; } - stringMode = StringMode.None; - index = end + 1; - continue; - } - if (stringMode == StringMode.Ordinary) - { if (line[index] == '\\') { index = Math.Min(index + 2, line.Length); continue; } + if (stringIsInterpolated && line[index] == '{') + { + if (index + 1 < line.Length && line[index + 1] == '{') + { + index += 2; + continue; + } + interpolationDepth = 1; + index++; + continue; + } if (line[index++] == ordinaryQuote) + { stringMode = StringMode.None; + stringIsInterpolated = false; + } continue; } + if (interpolationDepth > 0) + { + if (nestedStringMode == StringMode.Raw) + { + if (line[index] == '"' && + CountConsecutive(line, index, '"') >= nestedRawQuoteCount) + { + index += nestedRawQuoteCount; + nestedStringMode = StringMode.None; + } + else + { + index++; + } + continue; + } + if (nestedStringMode == StringMode.Verbatim) + { + if (line[index] == '"' && + index + 1 < line.Length && + line[index + 1] == '"') + { + index += 2; + } + else if (line[index++] == '"') + { + nestedStringMode = StringMode.None; + } + continue; + } + if (nestedStringMode == StringMode.Ordinary) + { + if (line[index] == '\\') + { + index = Math.Min(index + 2, line.Length); + } + else if (line[index++] == nestedOrdinaryQuote) + { + nestedStringMode = StringMode.None; + } + continue; + } + + if (line[index] == '{') + { + interpolationDepth++; + AddPunctuation(result, lineIndex, index, 1, line); + index++; + continue; + } + if (line[index] == '}') + { + interpolationDepth--; + if (interpolationDepth > 0) + AddPunctuation(result, lineIndex, index, 1, line); + index++; + continue; + } + + if (TryStartRawString( + line, + index, + out var nestedPrefixLength, + out nestedRawQuoteCount, + out _)) + { + nestedStringMode = StringMode.Raw; + index += nestedPrefixLength; + continue; + } + if (StartsVerbatimString( + line, + index, + out var nestedVerbatimPrefixLength, + out _)) + { + nestedStringMode = StringMode.Verbatim; + index += nestedVerbatimPrefixLength; + continue; + } + if (line[index] == '$' && + index + 1 < line.Length && + line[index + 1] == '"') + { + nestedStringMode = StringMode.Ordinary; + nestedOrdinaryQuote = '"'; + index += 2; + continue; + } + if (line[index] is '\'' or '"') + { + nestedStringMode = StringMode.Ordinary; + nestedOrdinaryQuote = line[index++]; + continue; + } + } + if (index + 1 < line.Length && line[index] == '/' && line[index + 1] == '/') break; if (index + 1 < line.Length && line[index] == '/' && line[index + 1] == '*') @@ -505,25 +823,49 @@ private static List Tokenize(IReadOnlyList sourceLines, int max continue; } - var quoteCount = CountConsecutive(line, index, '"'); - if (quoteCount >= 3) + if (TryStartRawString( + line, + index, + out var rawPrefixLength, + out rawQuoteCount, + out var rawDollarCount)) { stringMode = StringMode.Raw; - rawQuoteCount = quoteCount; - index += quoteCount; + stringIsInterpolated = rawDollarCount > 0; + interpolationBraceCount = Math.Max(rawDollarCount, 1); + index += rawPrefixLength; continue; } - if (StartsVerbatimString(line, index, out var prefixLength)) + if (StartsVerbatimString( + line, + index, + out var prefixLength, + out var verbatimIsInterpolated)) { stringMode = StringMode.Verbatim; + stringIsInterpolated = verbatimIsInterpolated; + interpolationBraceCount = 1; index += prefixLength; continue; } + if (line[index] == '$' && + index + 1 < line.Length && + line[index + 1] == '"') + { + stringMode = StringMode.Ordinary; + stringIsInterpolated = true; + interpolationBraceCount = 1; + ordinaryQuote = '"'; + index += 2; + continue; + } + if (line[index] is '\'' or '"') { stringMode = StringMode.Ordinary; + stringIsInterpolated = false; ordinaryQuote = line[index++]; continue; } @@ -569,12 +911,7 @@ private static List Tokenize(IReadOnlyList sourceLines, int max line.AsSpan(index, 2) is "=>" or "::" or "?." or "??" ? 2 : 1; - result.Add(new Lexeme( - lineIndex, - index, - punctuationLength, - line.Substring(index, punctuationLength), - LexemeKind.Punctuation)); + AddPunctuation(result, lineIndex, index, punctuationLength, line); index += punctuationLength; } } @@ -582,6 +919,21 @@ private static List Tokenize(IReadOnlyList sourceLines, int max return result; } + private static void AddPunctuation( + ICollection lexemes, + int line, + int start, + int length, + string sourceLine) + { + lexemes.Add(new Lexeme( + line, + start, + length, + sourceLine.Substring(start, length), + LexemeKind.Punctuation)); + } + private static bool IsDirectiveStart(IReadOnlyList lexemes, int index) { if (index + 1 < lexemes.Count && lexemes[index + 1].Text == "(") @@ -781,6 +1133,54 @@ private static bool TryFindBodyRange( return false; } + private static bool TryFindTypeBodyRange( + IReadOnlyList lexemes, + int nameIndex, + out (int Start, int End) body) + { + body = default; + for (var index = nameIndex + 1; index < lexemes.Count; index++) + { + if (lexemes[index].Text == ";") + return false; + if (lexemes[index].Text != "{") + continue; + + var close = FindMatching(lexemes, index, "{", "}"); + if (close < 0) + return false; + body = (index, close); + return true; + } + return false; + } + + private static int FindLastIdentifierBeforeEquals( + IReadOnlyList lexemes, + int start, + int end) + { + var equals = end; + for (var index = start; index < end; index++) + { + if (lexemes[index].Text == "=") + { + equals = index; + break; + } + } + + for (var index = equals - 1; index >= start; index--) + { + if (lexemes[index].Kind != LexemeKind.Identifier) + continue; + var word = NormalizeIdentifier(lexemes[index].Text); + if (!Keywords.Contains(word)) + return index; + } + return -1; + } + private static bool ContainsPropertyAccessor(IReadOnlyList lexemes, int openBrace) { var closeBrace = FindMatching(lexemes, openBrace, "{", "}"); @@ -820,6 +1220,74 @@ CSharpSemanticTokenKind.Struct or StartsWithUppercase(word); } + private static bool IsExplicitInterfaceMemberDeclaration( + IReadOnlyList lexemes, + CSharpSemanticTokenKind?[] kinds, + int index) + => index >= 2 && + lexemes[index - 1].Text == "." && + lexemes[index - 2].Kind == LexemeKind.Identifier && + IsDeclarationTypeBefore(lexemes, kinds, index - 2); + + private static bool TryGetCommaSeparatedDeclarationKind( + IReadOnlyList lexemes, + CSharpSemanticTokenKind?[] kinds, + int index, + out CSharpSemanticTokenKind kind) + { + kind = default; + if (index <= 0 || + lexemes[index - 1].Text != "," || + !HasDeclarationTerminatorAfter(lexemes, index)) + { + return false; + } + + var parenthesisDepth = 0; + var bracketDepth = 0; + var braceDepth = 0; + for (var previous = index - 2; previous >= 0; previous--) + { + var text = lexemes[previous].Text; + if (text == ")") + parenthesisDepth++; + else if (text == "(") + { + if (parenthesisDepth == 0) + return false; + parenthesisDepth--; + } + else if (text == "]") + bracketDepth++; + else if (text == "[") + { + if (bracketDepth == 0) + return false; + bracketDepth--; + } + else if (text == "}") + braceDepth++; + else if (text == "{") + { + if (braceDepth == 0) + return false; + braceDepth--; + } + + if (parenthesisDepth != 0 || bracketDepth != 0 || braceDepth != 0) + continue; + if (text == ";") + return false; + if (kinds[previous] is CSharpSemanticTokenKind.Variable or + CSharpSemanticTokenKind.Field) + { + kind = kinds[previous]!.Value; + return true; + } + } + return false; + } + private static bool HasDeclarationTerminatorAfter(IReadOnlyList lexemes, int index) { if (index + 1 >= lexemes.Count) @@ -835,6 +1303,13 @@ private static bool IsTypeContext(IReadOnlyList lexemes, int index) return previous is "as" or "default" or "is" or "new" or "sizeof" or "typeof"; } + private static bool IsQualifiedInstanceMemberAccess( + IReadOnlyList lexemes, + int index) + => index >= 2 && + lexemes[index - 1].Text is "." or "?." && + NormalizeIdentifier(lexemes[index - 2].Text) is "this" or "base"; + private static int FindMatching( IReadOnlyList lexemes, int openIndex, @@ -852,9 +1327,14 @@ private static int FindMatching( return -1; } - private static bool StartsVerbatimString(string line, int index, out int prefixLength) + private static bool StartsVerbatimString( + string line, + int index, + out int prefixLength, + out bool isInterpolated) { prefixLength = 0; + isInterpolated = false; if (line[index] == '@' && index + 1 < line.Length && line[index + 1] == '"') prefixLength = 2; else if (index + 2 < line.Length && @@ -863,10 +1343,40 @@ private static bool StartsVerbatimString(string line, int index, out int prefixL line[index + 2] == '"') { prefixLength = 3; + isInterpolated = true; } return prefixLength > 0; } + private static bool TryStartRawString( + string line, + int index, + out int prefixLength, + out int quoteCount, + out int dollarCount) + { + prefixLength = 0; + quoteCount = 0; + dollarCount = 0; + var quoteStart = index; + while (quoteStart < line.Length && line[quoteStart] == '$') + { + dollarCount++; + quoteStart++; + } + + quoteCount = CountConsecutive(line, quoteStart, '"'); + if (quoteCount < 3) + { + dollarCount = 0; + quoteCount = 0; + return false; + } + + prefixLength = dollarCount + quoteCount; + return true; + } + private static int CountConsecutive(string text, int start, char value) { var index = start; @@ -875,16 +1385,6 @@ private static int CountConsecutive(string text, int start, char value) return index - start; } - private static int FindRawStringEnd(string line, int start, int quoteCount) - { - for (var index = start; index < line.Length; index++) - { - if (line[index] == '"' && CountConsecutive(line, index, '"') >= quoteCount) - return index + quoteCount; - } - return -1; - } - private static bool IsIdentifierStart(char value) => char.IsLetter(value) || value is '_' or '@'; diff --git a/tests/CodeIndex.Tests/LspServerTests.cs b/tests/CodeIndex.Tests/LspServerTests.cs index 6e8e0afe12..cf7a12fb9c 100644 --- a/tests/CodeIndex.Tests/LspServerTests.cs +++ b/tests/CodeIndex.Tests/LspServerTests.cs @@ -1039,16 +1039,27 @@ public void HandleMessage_SemanticTokens_SharesCSharpDeclarationAndUsageKinds_Is [ "using System.Collections.Generic;", "namespace Sample.Tools;", - "internal sealed record App(T Value)", + "internal interface IFoo { void Execute(); string Label { get; } }", + "internal enum Color { Red, Blue = 2 }", + "internal readonly record struct Point(int X);", + "internal sealed record App(T Value) : IFoo", "{", " private const int Count = 1;", + " private int value;", " public string Name { get; init; } = string.Empty;", + " public event Action Changed;", " public void Run(TArg input) where TArg : notnull", " {", + " int first = 0, second = 1;", + " this.value = input.GetHashCode();", " var local = new List();", + " var text = $\"{input}\";", + " Changed?.Invoke();", " Helper(input);", " }", - " private void Helper(T value) { }", + " void IFoo.Execute() { }", + " string IFoo.Label => Name;", + " private void Helper(T value) { _ = $\"{value}\"; }", "}", ]); File.WriteAllText(sourcePath, source); @@ -1073,23 +1084,39 @@ public void HandleMessage_SemanticTokens_SharesCSharpDeclarationAndUsageKinds_Is AssertSemanticToken(tokens, 1, "namespace", 15, 0); AssertSemanticToken(tokens, 1, "Sample", 0, 0); AssertSemanticToken(tokens, 2, "internal", 16, 0); - AssertSemanticToken(tokens, 2, "sealed", 16, 0); - AssertSemanticToken(tokens, 2, "record", 15, 0); - AssertSemanticToken(tokens, 2, "App", 2, 1); - AssertSemanticToken(tokens, 2, "T", 6, 1); - AssertSemanticToken(tokens, 2, "Value", 7, 1); - AssertSemanticToken(tokens, 4, "private", 16, 0); - AssertSemanticToken(tokens, 4, "Count", 23, 1); - AssertSemanticToken(tokens, 5, "Name", 9, 1); - AssertSemanticToken(tokens, 5, "get", 15, 0); - AssertSemanticToken(tokens, 5, "init", 15, 0); - AssertSemanticToken(tokens, 6, "Run", 13, 1); - AssertSemanticToken(tokens, 6, "TArg", 6, 1); - AssertSemanticToken(tokens, 6, "input", 7, 1); - AssertSemanticToken(tokens, 6, "where", 15, 0); - AssertSemanticToken(tokens, 8, "local", 8, 1); - AssertSemanticToken(tokens, 9, "Helper", 13, 0); - AssertSemanticToken(tokens, 9, "input", 7, 0); + AssertSemanticToken(tokens, 2, "IFoo", 4, 1); + AssertSemanticToken(tokens, 3, "Color", 3, 1); + AssertSemanticToken(tokens, 3, "Red", 10, 1); + AssertSemanticToken(tokens, 3, "Blue", 10, 1); + AssertSemanticToken(tokens, 4, "Point", 5, 1); + AssertSemanticToken(tokens, 5, "sealed", 16, 0); + AssertSemanticToken(tokens, 5, "record", 15, 0); + AssertSemanticToken(tokens, 5, "App", 2, 1); + AssertSemanticToken(tokens, 5, "T", 6, 1); + AssertSemanticToken(tokens, 5, "Value", 7, 1); + AssertSemanticToken(tokens, 7, "private", 16, 0); + AssertSemanticToken(tokens, 7, "Count", 23, 1); + AssertSemanticToken(tokens, 8, "value", 23, 1); + AssertSemanticToken(tokens, 9, "Name", 9, 1); + AssertSemanticToken(tokens, 9, "get", 15, 0); + AssertSemanticToken(tokens, 9, "init", 15, 0); + AssertSemanticToken(tokens, 10, "Changed", 11, 1); + AssertSemanticToken(tokens, 11, "Run", 13, 1); + AssertSemanticToken(tokens, 11, "TArg", 6, 1); + AssertSemanticToken(tokens, 11, "input", 7, 1); + AssertSemanticToken(tokens, 11, "where", 15, 0); + AssertSemanticToken(tokens, 13, "first", 8, 1); + AssertSemanticToken(tokens, 13, "second", 8, 1); + AssertSemanticToken(tokens, 14, "value", 23, 0); + AssertSemanticToken(tokens, 14, "input", 7, 0); + AssertSemanticToken(tokens, 15, "local", 8, 1); + AssertSemanticToken(tokens, 16, "input", 7, 0); + AssertSemanticToken(tokens, 17, "Changed", 11, 0); + AssertSemanticToken(tokens, 18, "Helper", 13, 0); + AssertSemanticToken(tokens, 20, "Execute", 13, 1); + AssertSemanticToken(tokens, 21, "Label", 9, 1); + AssertSemanticToken(tokens, 22, "value", 7, 1); + AssertSemanticToken(tokens, 22, "value", 7, 0); Assert.DoesNotContain( tokens.SelectMany((left, index) => tokens.Skip(index + 1).Select(right => (left, right))), pair => @@ -1103,6 +1130,50 @@ public void HandleMessage_SemanticTokens_SharesCSharpDeclarationAndUsageKinds_Is } } + [Fact] + public void HandleMessage_SemanticTokens_FallsBackToIndexedCSharpSymbolsWhenSourceReadFails_Issue4852() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_lsp_semantic_tokens_fallback_4852"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + var sourcePath = Path.Combine(projectRoot, "app.cs"); + const string source = "internal sealed class App { public void Run() { } }\n"; + File.WriteAllText(sourcePath, source); + TestProjectHelper.InsertIndexedFile(dbPath, "app.cs", "csharp", source); + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + using var server = new LspServer( + new DbReader(db), + "1.2.3", + ProgramRunner.CreateDefaultJsonOptions(), + projectRoot); + + JsonObject? response; + try + { + LspServer.PositionFileLengthCheckedForTesting = _ => + throw new IOException("Injected semantic-token source read failure."); + response = HandleInitializedMessage( + server, + CreateTextDocumentRequest("textDocument/semanticTokens/full", sourcePath, 48521)); + } + finally + { + LspServer.PositionFileLengthCheckedForTesting = null; + } + + Assert.NotNull(response); + var tokens = DecodeSemanticTokens(response!["result"]!["data"]!.AsArray(), source); + AssertSemanticToken(tokens, 0, "App", 2, 1); + AssertSemanticToken(tokens, 0, "Run", 13, 1); + } + finally + { + LspServer.PositionFileLengthCheckedForTesting = null; + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void HandleMessage_InlayHint_HonorsRangeAndSuppressesExplicitTypes_Issue4418() { diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs index 736c79db05..a26257e727 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs @@ -10443,22 +10443,33 @@ public void RunExcerpt_JsonCSharpSemanticKindsMatchLspClassifier_Issue4852() [ "using System.Collections.Generic;", "namespace Sample.Tools;", - "internal sealed record App(T Value)", + "internal interface IFoo { void Execute(); string Label { get; } }", + "internal enum Color { Red, Blue = 2 }", + "internal readonly record struct Point(int X);", + "internal sealed record App(T Value) : IFoo", "{", " private const int Count = 1;", + " private int value;", " public string Name { get; init; } = string.Empty;", + " public event Action Changed;", " public void Run(TArg input) where TArg : notnull", " {", + " int first = 0, second = 1;", + " this.value = input.GetHashCode();", " var local = new List();", + " var text = $\"{input}\";", + " Changed?.Invoke();", " Helper(input);", " }", - " private void Helper(T value) { }", + " void IFoo.Execute() { }", + " string IFoo.Label => Name;", + " private void Helper(T value) { _ = $\"{value}\"; }", "}", ]); TestProjectHelper.InsertIndexedFile(dbPath, "src/Sample.cs", "csharp", source); var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunExcerpt( - ["src/Sample.cs", "--db", dbPath, "--start", "1", "--end", "13", "--json"], + ["src/Sample.cs", "--db", dbPath, "--start", "1", "--end", "24", "--json"], _jsonOptions)); using var document = ParseJsonOutput(stdout); @@ -10472,22 +10483,37 @@ public void RunExcerpt_JsonCSharpSemanticKindsMatchLspClassifier_Issue4852() AssertExcerptToken(1, "System", "namespace"); AssertExcerptToken(2, "Sample", "namespace"); AssertExcerptToken(3, "internal", "modifier"); - AssertExcerptToken(3, "record", "keyword"); - AssertExcerptToken(3, "App", "class", declaration: true); - AssertExcerptToken(3, "T", "typeParameter", declaration: true); - AssertExcerptToken(3, "Value", "parameter", declaration: true); - AssertExcerptToken(5, "private", "modifier"); - AssertExcerptToken(5, "Count", "field", declaration: true); - AssertExcerptToken(6, "Name", "property", declaration: true); - AssertExcerptToken(7, "Run", "method", declaration: true); - AssertExcerptToken(7, "input", "parameter", declaration: true); - AssertExcerptToken(7, "where", "keyword"); - AssertExcerptToken(9, "local", "variable", declaration: true); - AssertExcerptToken(10, "Helper", "method"); - AssertExcerptToken(10, "input", "parameter"); + AssertExcerptToken(3, "IFoo", "interface", declaration: true); + AssertExcerptToken(4, "Color", "enum", declaration: true); + AssertExcerptToken(4, "Red", "enumMember", declaration: true); + AssertExcerptToken(4, "Blue", "enumMember", declaration: true); + AssertExcerptToken(5, "Point", "struct", declaration: true); + AssertExcerptToken(6, "record", "keyword"); + AssertExcerptToken(6, "App", "class", declaration: true); + AssertExcerptToken(6, "T", "typeParameter", declaration: true); + AssertExcerptToken(6, "Value", "parameter", declaration: true); + AssertExcerptToken(8, "private", "modifier"); + AssertExcerptToken(8, "Count", "field", declaration: true); + AssertExcerptToken(9, "value", "field", declaration: true); + AssertExcerptToken(10, "Name", "property", declaration: true); + AssertExcerptToken(11, "Changed", "event", declaration: true); + AssertExcerptToken(12, "Run", "method", declaration: true); + AssertExcerptToken(12, "input", "parameter", declaration: true); + AssertExcerptToken(12, "where", "keyword"); + AssertExcerptToken(14, "first", "variable", declaration: true); + AssertExcerptToken(14, "second", "variable", declaration: true); + AssertExcerptToken(15, "value", "field"); + AssertExcerptToken(15, "input", "parameter"); + AssertExcerptToken(16, "local", "variable", declaration: true); + AssertExcerptToken(17, "input", "parameter"); + AssertExcerptToken(18, "Changed", "event"); + AssertExcerptToken(19, "Helper", "method"); + AssertExcerptToken(21, "Execute", "method", declaration: true); + AssertExcerptToken(22, "Label", "property", declaration: true); + AssertExcerptToken(23, "value", "parameter", declaration: true); var (partialExitCode, partialStdout, partialStderr) = CaptureConsole(() => QueryCommandRunner.RunExcerpt( - ["src/Sample.cs", "--db", dbPath, "--start", "9", "--end", "10", "--json"], + ["src/Sample.cs", "--db", dbPath, "--start", "16", "--end", "17", "--json"], _jsonOptions)); using var partialDocument = ParseJsonOutput(partialStdout); var partialSemanticTokens = partialDocument.RootElement @@ -10496,8 +10522,8 @@ public void RunExcerpt_JsonCSharpSemanticKindsMatchLspClassifier_Issue4852() .ToArray(); Assert.Equal(CommandExitCodes.Success, partialExitCode); Assert.Equal(string.Empty, partialStderr); - AssertExcerptTokenIn(partialSemanticTokens, 9, "TArg", "typeParameter"); - AssertExcerptTokenIn(partialSemanticTokens, 10, "input", "parameter"); + AssertExcerptTokenIn(partialSemanticTokens, 16, "TArg", "typeParameter"); + AssertExcerptTokenIn(partialSemanticTokens, 17, "input", "parameter"); Assert.DoesNotContain( semanticTokens.SelectMany((left, index) => semanticTokens.Skip(index + 1).Select(right => (left, right))), @@ -10536,6 +10562,58 @@ void AssertExcerptTokenIn( } } + [Fact] + public void RunExcerpt_JsonCSharpSemanticTokensUseVisibleRangeBudget_Issue4852() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_excerpt_semantic_range_budget_4852"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + var source = new StringBuilder("internal sealed class Large\n{\n"); + for (var index = 0; index < 4_000; index++) + source.Append(" private int Field").Append(index).AppendLine(";"); + const string targetLine = " private int TailMarker;"; + source.AppendLine(targetLine).AppendLine("}"); + TestProjectHelper.InsertIndexedFile( + dbPath, + "src/Large.cs", + "csharp", + source.ToString()); + + const int targetSourceLine = 4_003; + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunExcerpt( + [ + "src/Large.cs", + "--db", + dbPath, + "--start", + targetSourceLine.ToString(), + "--json", + ], + _jsonOptions)); + + using var document = ParseJsonOutput(stdout); + var semanticTokens = document.RootElement + .GetProperty("semantic_tokens") + .EnumerateArray() + .ToArray(); + var targetColumn = targetLine.IndexOf("TailMarker", StringComparison.Ordinal) + 1; + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + Assert.Contains(semanticTokens, token => + token.GetProperty("start_line").GetInt32() == targetSourceLine && + token.GetProperty("start_column").GetInt32() == targetColumn && + token.GetProperty("type").GetString() == "field" && + token.GetProperty("modifiers").EnumerateArray() + .Any(modifier => modifier.GetString() == "declaration")); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void RunExcerpt_JsonMaxBytesRejectsOversizedPayload_Issue4311() { From 2d15fdd3e1accaddd6ab7fde57c8c2ceea0690be Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 28 Jul 2026 14:39:32 +0900 Subject: [PATCH 3/3] Complete semantic token hardening (#4852) --- TESTING_GUIDE.md | 4 +- changelog.d/unreleased/4852.fixed.md | 4 +- .../Cli/QueryCommandRunner.Excerpt.cs | 5 +- .../Database/DbReader.FilesStatus.cs | 89 +++++- .../CSharpSemanticTokenClassifier.cs | 288 ++++++++++++++++-- tests/CodeIndex.Tests/LspServerTests.cs | 26 ++ .../QueryCommandRunnerSearchTests.cs | 40 ++- 7 files changed, 419 insertions(+), 37 deletions(-) diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index fb2443d3cf..684825edee 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -520,7 +520,7 @@ Use `docs/test-doc-maintenance-plan.md` before moving oversized suites or adding indexes one real C# fixture containing user-defined generic classes and delegates, block-comment generic trivia, repeated same-line generic names under indentation, unrelated same-name properties, ordinal-case type/property collisions, `Action` / `Stream` references, and uppercase property receivers split across partial-class and inherited-base files. Keep CLI `references`, `inspect`, and `deps` assertions together with the MCP `references`, `analyze_symbol`, and `deps` wrappers so unresolved diagnostics, generic-arity selection, case-sensitive receiver handling, false dependency-edge suppression, and preservation of real member dependencies cannot drift across surfaces. - `LspServerTests.cs` LSP JSON-RPC behavior and indexed editor semantics. Reference parity coverage must compare LSP locations with the matching CLI candidate bundle, including overload identity and both `includeDeclaration` states. Keep document/workspace identifier ranges and explicit-versus-inferred inlay hints in one source-semantics fixture so persisted-column anchoring, source confirmation, and hint suppression cannot drift independently. - C# semantic-token coverage decodes LSP delta coordinates and pairs the same keyword/modifier, namespace/type, method/property/event/enum-member, parameter/variable/field, generic/record-struct, explicit-interface-member, interpolation, comma-separated declaration, and declaration-kind fixture with `QueryCommandRunnerTests.RunExcerpt_JsonCSharpSemanticKindsMatchLspClassifier_Issue4852`. Keep both surfaces on the shared source classifier, retain ordering/non-overlap assertions, preserve the narrow excerpt check whose parameter/type-parameter declarations are outside the returned range, and retain `RunExcerpt_JsonCSharpSemanticTokensUseVisibleRangeBudget_Issue4852` so earlier tokens cannot empty a late excerpt. `HandleMessage_SemanticTokens_FallsBackToIndexedCSharpSymbolsWhenSourceReadFails_Issue4852` keeps indexed declaration tokens available when bounded source loading fails. + C# semantic-token coverage decodes LSP delta coordinates and pairs the same keyword/modifier, namespace/type, method/property/event/enum-member, parameter/variable/field, generic/record-struct, explicit-interface-member, attribute, using-declaration, accessor-local, numeric member-access, escaped/contextual-identifier, interpolation, comma-separated declaration, and declaration-kind fixture with `QueryCommandRunnerTests.RunExcerpt_JsonCSharpSemanticKindsMatchLspClassifier_Issue4852`. Keep both surfaces on the shared source classifier, retain ordering/non-overlap assertions, preserve the narrow excerpt check whose parameter/type-parameter declarations are outside the returned range, and retain `RunExcerpt_JsonCSharpSemanticTokensUseVisibleRangeBudget_Issue4852` so earlier tokens cannot empty a late excerpt while indexed-source reconstruction remains line/character bounded. `HandleMessage_SemanticTokens_FallsBackToIndexedCSharpSymbolsWhenSourceReadFails_Issue4852` keeps indexed declaration tokens available when bounded source loading fails. Document-symbol hierarchy coverage keeps positional and body record members in one same-line fixture, asserts the complete root-to-member shape and deterministic sibling order, and retains the separate same-range top-level controls so order-independent parent resolution cannot over-nest unrelated symbols. Keep a same-line duplicate-container fixture with distinct positional members so selection-column disambiguation cannot move an earlier member beneath a later same-named container. Symbol-progress coverage crosses the production chunk item limit with one sentinel, verifies the complete deterministic sequence across partial-result frames, and keeps live work-done delivery, truncation, typed request-ID cancellation, full-queue cancellation, queue-pressure notification preservation, backpressured `Server busy` response retention, emitted-count cancellation reporting, output-failure propagation, and invalid-token variants in the same protocol fixture. Assert both the item and UTF-8 JSON body budgets for every emitted chunk. The server-busy backpressure cancellation fixture must stage the cancel frame until both the active symbol request and blocked response write have been observed. Block the request on its cancellation token instead of matching fixed delays so suite load cannot convert the expected cancellation into a successful response. @@ -1416,7 +1416,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" は user-defined generic class / delegate、generic 間の block comment trivia、indent 下で同一行に反復する同名 generic、無関係な同名 property、大文字小文字だけ異なる type / property collision、`Action` / `Stream` reference、partial class の別ファイルおよび継承 base file に分かれた大文字始まりの property receiver を含む実 C# fixture を1回 index します。CLI の `references`、`inspect`、`deps` と MCP wrapper の `references`、`analyze_symbol`、`deps` を同じ fixture で検証し、unresolved 診断、generic arity による選択、case-sensitive receiver 判定、偽 dependency edge の抑制、実在する member dependency の維持が surface 間でずれないよう維持してください。 - `LspServerTests.cs` LSP の JSON-RPC 挙動と indexed editor semantics のテスト。reference parity coverage では、overload identity と `includeDeclaration` の両状態を含め、LSP location と対応する CLI candidate bundle を比較してください。document/workspace の identifier range と explicit/inferred inlay hint は 1 つの source-semantics fixture にまとめ、保存済み column の anchoring、source 上の確認、hint 抑制が別々に drift しないようにします。 - C# semantic-token coverage は LSP の delta 座標を decode し、同じ keyword/modifier、namespace/type、method/property/event/enum-member、parameter/variable/field、generic/record struct、explicit interface member、interpolation、comma-separated declaration、declaration kind の fixture を `QueryCommandRunnerTests.RunExcerpt_JsonCSharpSemanticKindsMatchLspClassifier_Issue4852` と対にします。両 surface が shared source classifier を使い続け、順序と非 overlap の assertion、parameter / type-parameter の declaration が返却範囲外にある narrow excerpt の検証、および手前の token によって file 後半の excerpt が空にならないことを保証する `RunExcerpt_JsonCSharpSemanticTokensUseVisibleRangeBudget_Issue4852` を維持してください。`HandleMessage_SemanticTokens_FallsBackToIndexedCSharpSymbolsWhenSourceReadFails_Issue4852` は bounded source loading が失敗しても indexed declaration token を利用できる状態を維持します。 + C# semantic-token coverage は LSP の delta 座標を decode し、同じ keyword/modifier、namespace/type、method/property/event/enum-member、parameter/variable/field、generic/record struct、explicit interface member、attribute、using declaration、accessor local、numeric member access、escaped/contextual identifier、interpolation、comma-separated declaration、declaration kind の fixture を `QueryCommandRunnerTests.RunExcerpt_JsonCSharpSemanticKindsMatchLspClassifier_Issue4852` と対にします。両 surface が shared source classifier を使い続け、順序と非 overlap の assertion、parameter / type-parameter の declaration が返却範囲外にある narrow excerpt の検証、および indexed-source reconstruction を line / character 上限内に保ちながら手前の token によって file 後半の excerpt が空にならないことを保証する `RunExcerpt_JsonCSharpSemanticTokensUseVisibleRangeBudget_Issue4852` を維持してください。`HandleMessage_SemanticTokens_FallsBackToIndexedCSharpSymbolsWhenSourceReadFails_Issue4852` は bounded source loading が失敗しても indexed declaration token を利用できる状態を維持します。 document-symbol hierarchy coverage は positional member と body member を同一行の record fixture にまとめ、root から member までの完全な形状と決定的な sibling 順序を assert します。順序非依存の親解決が無関係な symbol を過剰に nest しないよう、same-range top-level の control は独立したまま維持してください。selection column による曖昧性解消で前の member が行内で後にある同名 container の配下へ移動しないよう、異なる positional member を持つ同一行の duplicate-container fixture も維持してください。 symbol-progress coverage は production の chunk item limit を sentinel 1 件だけで超え、partial-result frame 全体の決定的な完全 sequence を検証します。live work-done delivery、truncation、型付き request ID の cancellation、full-queue cancellation、queue pressure 下の notification 保持、backpressure された `Server busy` response の保持、cancellation 時の送信済み件数、output-failure propagation、invalid-token variant は同じ protocol fixture にまとめ、送信した全 chunk の item 上限と UTF-8 JSON body budget の両方を assert してください。 server-busy backpressure cancellation fixture では、active symbol request と block された response write の両方を観測するまで cancel frame を stage してください。固定 delay の一致に依存せず cancellation token 上で request を block することで、suite load により期待する cancellation が成功 response に変わる競合を防ぎます。 diff --git a/changelog.d/unreleased/4852.fixed.md b/changelog.d/unreleased/4852.fixed.md index e2e329a8a1..5fead8bd74 100644 --- a/changelog.d/unreleased/4852.fixed.md +++ b/changelog.d/unreleased/4852.fixed.md @@ -19,8 +19,8 @@ affected: ## English -- **Restored consistent C# semantic token kinds across LSP and excerpt output (#4852)** — a shared source classifier now identifies keywords and modifiers, namespaces and types, methods, properties, events and enum members, parameters, variables and fields, generic/record constructs, contextual identifiers, interpolated expressions, explicit interface members, comma-separated declarations, and declaration modifiers before LSP delta encoding or excerpt source-coordinate mapping. Excerpts apply their token budget after filtering indexed-source classifications to visible lines and retain a bounded visible-range fallback, while LSP retains indexed declaration tokens if source loading fails. This fixes the direct regression of #4444 where `private` and `using` appeared as variables and method names could appear as types. +- **Restored consistent C# semantic token kinds across LSP and excerpt output (#4852)** — a shared source classifier now identifies keywords and modifiers, namespaces and types, methods, properties, events and enum members, parameters, variables and fields, generic/record constructs, contextual and escaped identifiers, attributes, using declarations, accessor locals, numeric member access, interpolated expressions, explicit interface members, comma-separated declarations, and declaration modifiers before LSP delta encoding or excerpt source-coordinate mapping. Excerpts apply their token budget after filtering line/character-bounded indexed-source classifications to visible lines and retain a bounded visible-range fallback, while LSP retains indexed declaration tokens if source loading fails. This fixes the direct regression of #4444 where `private` and `using` appeared as variables and method names could appear as types. ## 日本語 -- **LSP と excerpt 出力で一貫した C# semantic token kind を復元しました (#4852)** — shared source classifier が keyword / modifier、namespace / type、method / property / event / enum member、parameter、variable / field、generic / record 構文、contextual identifier、interpolation 内の式、explicit interface member、comma-separated declaration、declaration modifier を判定してから、LSP の delta encoding または excerpt の source 座標 mapping を行うようになりました。excerpt は indexed-source の分類を可視行へ絞った後に token budget を適用し、bounded な可視範囲 fallback も維持します。LSP は source loading が失敗した場合に indexed declaration token を利用します。`private` と `using` が variable、method 名が type として表示されることがあった #4444 の直接 regression を修正します。 +- **LSP と excerpt 出力で一貫した C# semantic token kind を復元しました (#4852)** — shared source classifier が keyword / modifier、namespace / type、method / property / event / enum member、parameter、variable / field、generic / record 構文、contextual / escaped identifier、attribute、using declaration、accessor local、numeric member access、interpolation 内の式、explicit interface member、comma-separated declaration、declaration modifier を判定してから、LSP の delta encoding または excerpt の source 座標 mapping を行うようになりました。excerpt は line / character 上限内で再構成した indexed-source の分類を可視行へ絞った後に token budget を適用し、bounded な可視範囲 fallback も維持します。LSP は source loading が失敗した場合に indexed declaration token を利用します。`private` と `using` が variable、method 名が type として表示されることがあった #4444 の直接 regression を修正します。 diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Excerpt.cs b/src/CodeIndex/Cli/QueryCommandRunner.Excerpt.cs index fd806ba3c9..5dbf5aaad3 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Excerpt.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Excerpt.cs @@ -230,7 +230,10 @@ private static List BuildExcerptSemanticTokens( var isCSharp = string.Equals(excerpt.Lang, "csharp", StringComparison.OrdinalIgnoreCase) || string.Equals(Path.GetExtension(excerpt.Path), ".cs", StringComparison.OrdinalIgnoreCase); var indexedSourceLines = isCSharp - ? reader.GetIndexedSourceLinesForSemanticTokens(excerpt.Path) + ? reader.GetIndexedSourceLinesForSemanticTokens( + excerpt.Path, + CSharpSemanticTokenClassifier.DefaultExcerptSourceLineLimit, + CSharpSemanticTokenClassifier.DefaultExcerptSourceCharacterLimit) : []; var classifiesIndexedSource = indexedSourceLines.Count > 0; var includedSourceLines = classifiesIndexedSource diff --git a/src/CodeIndex/Database/DbReader.FilesStatus.cs b/src/CodeIndex/Database/DbReader.FilesStatus.cs index 79fe4e97d7..94b7a3c053 100644 --- a/src/CodeIndex/Database/DbReader.FilesStatus.cs +++ b/src/CodeIndex/Database/DbReader.FilesStatus.cs @@ -744,21 +744,94 @@ FROM chunks c } /// - /// Reconstruct all indexed source lines while preserving absolute line positions for semantic classification. - /// semantic 分類向けに絶対行位置を維持しながら、indexed source 全行を再構成する。 + /// Reconstruct a bounded indexed-source prefix while preserving absolute line positions for semantic classification. + /// semantic 分類向けに絶対行位置を維持しながら、bounded な indexed-source prefix を再構成する。 /// - internal IReadOnlyList GetIndexedSourceLinesForSemanticTokens(string path) + internal IReadOnlyList GetIndexedSourceLinesForSemanticTokens( + string path, + int maxLines, + int maxCharacters) { - if (!TryLoadIndexedFileLines(path, out _, out var totalLines, out var lineMap) || totalLines <= 0) + if (string.IsNullOrWhiteSpace(path) || maxLines <= 0 || maxCharacters <= 0) return []; - var sourceLines = new string?[totalLines]; - foreach (var (line, content) in lineMap) + var totalLines = 0; + using (var fileCmd = _conn.CreateCommand()) { - if (line > 0 && line <= sourceLines.Length) - sourceLines[line - 1] = content; + fileCmd.CommandText = "SELECT lines FROM files WHERE path = @path"; + SqliteCommandPolicy.Add(fileCmd, "@path", path); + using var fileReader = fileCmd.ExecuteTrackedReader(); + if (!fileReader.TrackedRead()) + return []; + totalLines = fileReader.GetInt32(0); } + if (totalLines <= 0) + return []; + var sourceLines = new string?[Math.Min(totalLines, maxLines)]; + using var chunkCmd = _conn.CreateCommand(); + chunkCmd.CommandText = """ + SELECT c.start_line, c.end_line, substr(c.content, 1, @maxChunkCharacters) + FROM chunks c + JOIN files f ON c.file_id = f.id + WHERE f.path = @path + AND c.start_line <= @maxLines + ORDER BY c.start_line, c.chunk_index + """; + SqliteCommandPolicy.Add(chunkCmd, "@path", path); + SqliteCommandPolicy.Add(chunkCmd, "@maxLines", sourceLines.Length); + SqliteCommandPolicy.Add( + chunkCmd, + "@maxChunkCharacters", + checked(maxCharacters + 1)); + + var remainingCharacters = maxCharacters; + var lastContiguousLine = 0; + var budgetExhausted = false; + using var chunkReader = chunkCmd.ExecuteTrackedReader(); + while (!budgetExhausted && chunkReader.TrackedRead()) + { + var chunkStartLine = chunkReader.GetInt32(0); + var chunkEndLine = chunkReader.GetInt32(1); + var chunkLines = chunkReader.GetString(2).Split('\n'); + var lineCount = Math.Min( + chunkLines.Length, + chunkEndLine - chunkStartLine + 1); + for (var index = 0; index < lineCount; index++) + { + var absoluteLine = chunkStartLine + index; + if (absoluteLine <= 0) + continue; + if (absoluteLine > sourceLines.Length) + { + budgetExhausted = true; + break; + } + if (sourceLines[absoluteLine - 1] != null) + continue; + + var content = chunkLines[index]; + var requiredCharacters = checked(content.Length + 1); + if (requiredCharacters > remainingCharacters) + { + budgetExhausted = true; + break; + } + + sourceLines[absoluteLine - 1] = content; + remainingCharacters -= requiredCharacters; + while (lastContiguousLine < sourceLines.Length && + sourceLines[lastContiguousLine] != null) + { + lastContiguousLine++; + } + } + } + + if (lastContiguousLine == 0) + return []; + if (lastContiguousLine < sourceLines.Length) + Array.Resize(ref sourceLines, lastContiguousLine); return sourceLines; } diff --git a/src/CodeIndex/Semantics/CSharpSemanticTokenClassifier.cs b/src/CodeIndex/Semantics/CSharpSemanticTokenClassifier.cs index e61988eefa..4e592fec70 100644 --- a/src/CodeIndex/Semantics/CSharpSemanticTokenClassifier.cs +++ b/src/CodeIndex/Semantics/CSharpSemanticTokenClassifier.cs @@ -34,6 +34,8 @@ internal readonly record struct ClassifiedCSharpSemanticToken( internal static class CSharpSemanticTokenClassifier { internal const int DefaultExcerptTokenLimit = 10_000; + internal const int DefaultExcerptSourceLineLimit = 100_000; + internal const int DefaultExcerptSourceCharacterLimit = 4 * 1024 * 1024; private static readonly HashSet Modifiers = new(StringComparer.Ordinal) { @@ -42,6 +44,20 @@ internal static class CSharpSemanticTokenClassifier "unsafe", "virtual", "volatile", }; + private static readonly HashSet ContextualModifiers = new(StringComparer.Ordinal) + { + "async", "file", "partial", "required", + }; + + private static readonly HashSet ContextualKeywords = new(StringComparer.Ordinal) + { + "add", "alias", "and", "ascending", "async", "await", "by", "descending", "dynamic", + "equals", "file", "from", "get", "global", "group", "init", "into", "join", "let", + "managed", "nameof", "not", "notnull", "on", "or", "orderby", "partial", "record", + "remove", "required", "scoped", "select", "set", "unmanaged", "var", "when", "where", + "with", "yield", + }; + private static readonly HashSet Keywords = new(StringComparer.Ordinal) { "add", "alias", "and", "as", "ascending", "async", "await", "base", "bool", "break", @@ -97,6 +113,7 @@ internal static IReadOnlyList Classify( var declaredEventNames = new HashSet(StringComparer.Ordinal); var methodBodies = new List<(int Start, int End)>(); var enumBodies = new List<(int Start, int End)>(); + var typeBodies = new List<(int Start, int End)>(); ClassifyKeywordsAndNumbers(lexemes, kinds); ClassifyDirectives(lexemes, kinds); @@ -107,7 +124,8 @@ internal static IReadOnlyList Classify( declaredTypeNames, declaredTypeParameterNames, declaredParameterNames, - enumBodies); + enumBodies, + typeBodies); ClassifyEnumMembers( lexemes, kinds, @@ -136,7 +154,7 @@ internal static IReadOnlyList Classify( lexemes, kinds, declarations, - methodBodies, + typeBodies, declaredFieldNames); ClassifyRemainingIdentifiers( lexemes, @@ -232,9 +250,11 @@ private static void ClassifyKeywordsAndNumbers( continue; var word = NormalizeIdentifier(lexeme.Text); - if (Modifiers.Contains(word)) + if (lexeme.Text[0] == '@') + continue; + if (Modifiers.Contains(word) && IsModifierAt(lexemes, index, word)) kinds[index] = CSharpSemanticTokenKind.Modifier; - else if (Keywords.Contains(word)) + else if (Keywords.Contains(word) && IsKeywordAt(lexemes, index, word)) kinds[index] = CSharpSemanticTokenKind.Keyword; } } @@ -246,6 +266,8 @@ private static void ClassifyDirectives( for (var index = 0; index < lexemes.Count; index++) { var word = NormalizeIdentifier(lexemes[index].Text); + if (kinds[index] != CSharpSemanticTokenKind.Keyword) + continue; if (word == "namespace") { MarkQualifiedName(lexemes, kinds, index + 1, stopAtEquals: false); @@ -264,12 +286,14 @@ private static void ClassifyTypeDeclarations( HashSet declaredTypeNames, HashSet declaredTypeParameterNames, HashSet declaredParameterNames, - List<(int Start, int End)> enumBodies) + List<(int Start, int End)> enumBodies, + List<(int Start, int End)> typeBodies) { for (var index = 0; index < lexemes.Count; index++) { var keyword = NormalizeIdentifier(lexemes[index].Text); - if (!TypeDeclarationKeywords.Contains(keyword)) + if (!TypeDeclarationKeywords.Contains(keyword) || + kinds[index] != CSharpSemanticTokenKind.Keyword) continue; if (keyword is "class" or "struct" && index > 0 && @@ -318,10 +342,11 @@ private static void ClassifyTypeDeclarations( declaredParameterNames); } - if (kind == CSharpSemanticTokenKind.Enum && - TryFindTypeBodyRange(lexemes, nameIndex, out var enumBody)) + if (TryFindTypeBodyRange(lexemes, nameIndex, out var typeBody)) { - enumBodies.Add(enumBody); + typeBodies.Add(typeBody); + if (kind == CSharpSemanticTokenKind.Enum) + enumBodies.Add(typeBody); } } } @@ -387,7 +412,8 @@ private static void ClassifyEvents( { for (var eventIndex = 0; eventIndex < lexemes.Count; eventIndex++) { - if (NormalizeIdentifier(lexemes[eventIndex].Text) != "event") + if (NormalizeIdentifier(lexemes[eventIndex].Text) != "event" || + kinds[eventIndex] != CSharpSemanticTokenKind.Keyword) continue; var segmentStart = eventIndex + 1; @@ -469,6 +495,12 @@ private static void ClassifyMethods( if (!TryFindInvocationOpenParen(lexemes, index, out var openParen)) continue; + if (IsInsideAttributeList(lexemes, index)) + { + kinds[index] = CSharpSemanticTokenKind.Type; + continue; + } + var previous = index - 1; if (previous >= 0 && NormalizeIdentifier(lexemes[previous].Text) == "new") { @@ -507,7 +539,7 @@ private static void ClassifyVariablesAndFields( IReadOnlyList lexemes, CSharpSemanticTokenKind?[] kinds, bool[] declarations, - IReadOnlyList<(int Start, int End)> methodBodies, + IReadOnlyList<(int Start, int End)> typeBodies, HashSet declaredFieldNames) { for (var index = 0; index < lexemes.Count; index++) @@ -521,11 +553,9 @@ private static void ClassifyVariablesAndFields( if (IsDeclarationTypeBefore(lexemes, kinds, index) && HasDeclarationTerminatorAfter(lexemes, index)) { - var insideMethod = methodBodies.Any( - body => index > body.Start && index < body.End); - kind = insideMethod - ? CSharpSemanticTokenKind.Variable - : CSharpSemanticTokenKind.Field; + kind = IsDirectlyInsideTypeBody(lexemes, index, typeBodies) + ? CSharpSemanticTokenKind.Field + : CSharpSemanticTokenKind.Variable; } else if (!TryGetCommaSeparatedDeclarationKind( lexemes, @@ -887,10 +917,32 @@ private static List Tokenize(IReadOnlyList sourceLines, int max if (char.IsDigit(line[index])) { var start = index++; - while (index < line.Length && - (char.IsLetterOrDigit(line[index]) || line[index] is '_' or '.')) + var hasDecimalPoint = false; + while (index < line.Length) { - index++; + var value = line[index]; + if (char.IsLetterOrDigit(value) || value == '_') + { + index++; + continue; + } + if (!hasDecimalPoint && + value == '.' && + index + 1 < line.Length && + char.IsDigit(line[index + 1])) + { + hasDecimalPoint = true; + index++; + continue; + } + if (value is '+' or '-' && + index > start && + line[index - 1] is 'e' or 'E') + { + index++; + continue; + } + break; } result.Add(new Lexeme( lineIndex, @@ -934,6 +986,111 @@ private static void AddPunctuation( LexemeKind.Punctuation)); } + private static bool IsModifierAt( + IReadOnlyList lexemes, + int index, + string word) + { + if (!Modifiers.Contains(word)) + return false; + if (!ContextualModifiers.Contains(word)) + return true; + if (LooksLikeIdentifierDeclaration(lexemes, index)) + return false; + if (index + 1 >= lexemes.Count) + return false; + + var next = lexemes[index + 1]; + if (next.Kind == LexemeKind.Identifier) + return true; + if (word != "async" || next.Text != "(") + return false; + + var closeParen = FindMatching(lexemes, index + 1, "(", ")"); + return closeParen >= 0 && + closeParen + 1 < lexemes.Count && + lexemes[closeParen + 1].Text == "=>"; + } + + private static bool IsKeywordAt( + IReadOnlyList lexemes, + int index, + string word) + { + if (!Keywords.Contains(word) || + lexemes[index].Text.StartsWith('@')) + { + return false; + } + if (!ContextualKeywords.Contains(word)) + return true; + if (LooksLikeIdentifierDeclaration(lexemes, index)) + return false; + + var previous = index > 0 ? lexemes[index - 1].Text : null; + var next = index + 1 < lexemes.Count ? lexemes[index + 1].Text : null; + if (word is "get" or "set" or "init" or "add" or "remove") + { + return previous is "{" or "}" or ";" && + next is "{" or ";" or "=>"; + } + if (word == "record") + { + return next is "class" or "struct" || + (index + 1 < lexemes.Count && + lexemes[index + 1].Kind == LexemeKind.Identifier && + next is not "=" and not "," and not ";" and not ")"); + } + if (word is "notnull" or "unmanaged" or "managed") + return previous is ":" or ","; + if (word == "var") + { + return index + 1 < lexemes.Count && + lexemes[index + 1].Kind == LexemeKind.Identifier; + } + if (word == "nameof") + return next == "("; + if (word == "global") + return next is "using" or "::"; + if (word == "with") + return next == "{"; + if (word == "yield") + return next is "return" or "break"; + if (word == "when") + return next == "(" || + (index + 1 < lexemes.Count && + lexemes[index + 1].Kind == LexemeKind.Identifier); + if (word is "and" or "or" or "not" or "ascending" or "by" or + "descending" or "equals" or "from" or "group" or "into" or + "join" or "let" or "on" or "orderby" or "select" or "where") + { + return next is not null and not "." and not "?." and not "(" and + not "=" and not "," and not ";" and not ")"; + } + + return index + 1 < lexemes.Count && + lexemes[index + 1].Kind == LexemeKind.Identifier; + } + + private static bool LooksLikeIdentifierDeclaration( + IReadOnlyList lexemes, + int index) + { + if (index <= 0 || lexemes[index - 1].Text is "." or "?.") + return false; + + var previous = lexemes[index - 1]; + if (previous.Text is ">" or "]" or "?") + return true; + if (previous.Kind != LexemeKind.Identifier) + return false; + + var previousWord = NormalizeIdentifier(previous.Text); + return BuiltInTypeKeywords.Contains(previousWord) || + previousWord == "var" || + StartsWithUppercase(previousWord); + } + private static bool IsDirectiveStart(IReadOnlyList lexemes, int index) { if (index + 1 < lexemes.Count && lexemes[index + 1].Text == "(") @@ -947,6 +1104,32 @@ private static bool IsDirectiveStart(IReadOnlyList lexemes, int index) return false; } + var equals = -1; + var identifierCount = 0; + for (var next = index + 1; next < lexemes.Count; next++) + { + if (lexemes[next].Text is ";" or "{") + break; + if (lexemes[next].Text == "=") + { + equals = next; + break; + } + if (lexemes[next].Kind == LexemeKind.Identifier && + NormalizeIdentifier(lexemes[next].Text) is not "static" and not "unsafe") + { + identifierCount++; + } + } + + if (equals >= 0 && identifierCount > 1) + return false; + if (index + 1 < lexemes.Count && + NormalizeIdentifier(lexemes[index + 1].Text) == "var") + { + return false; + } + return true; } @@ -986,7 +1169,10 @@ private static int FindNextUnclassifiedIdentifier( if (lexemes[index].Text is ";" or "{" or "}") return -1; if (lexemes[index].Kind == LexemeKind.Identifier && - !Keywords.Contains(NormalizeIdentifier(lexemes[index].Text)) && + !IsKeywordAt( + lexemes, + index, + NormalizeIdentifier(lexemes[index].Text)) && !kinds[index].HasValue) { return index; @@ -1013,7 +1199,10 @@ private static void MarkTypeParametersAfter( for (var index = open + 1; index < close; index++) { if (lexemes[index].Kind != LexemeKind.Identifier || - Keywords.Contains(NormalizeIdentifier(lexemes[index].Text))) + IsKeywordAt( + lexemes, + index, + NormalizeIdentifier(lexemes[index].Text))) { continue; } @@ -1081,7 +1270,8 @@ private static int FindParameterName( if (lexemes[index].Kind != LexemeKind.Identifier) continue; var word = NormalizeIdentifier(lexemes[index].Text); - if (!Keywords.Contains(word) && kinds[index] is not CSharpSemanticTokenKind.Modifier) + if (!IsKeywordAt(lexemes, index, word) && + kinds[index] is not CSharpSemanticTokenKind.Modifier) return index; } @@ -1175,7 +1365,7 @@ private static int FindLastIdentifierBeforeEquals( if (lexemes[index].Kind != LexemeKind.Identifier) continue; var word = NormalizeIdentifier(lexemes[index].Text); - if (!Keywords.Contains(word)) + if (!IsKeywordAt(lexemes, index, word)) return index; } return -1; @@ -1194,6 +1384,58 @@ private static bool ContainsPropertyAccessor(IReadOnlyList lexemes, int return false; } + private static bool IsInsideAttributeList( + IReadOnlyList lexemes, + int index) + { + var bracketDepth = 0; + for (var previous = index - 1; previous >= 0; previous--) + { + if (lexemes[previous].Text == "]") + { + bracketDepth++; + continue; + } + if (lexemes[previous].Text != "[") + continue; + if (bracketDepth > 0) + { + bracketDepth--; + continue; + } + + if (previous == 0) + return true; + return lexemes[previous - 1].Text is + "{" or "}" or ";" or "(" or "," or ":" or "<" or "["; + } + return false; + } + + private static bool IsDirectlyInsideTypeBody( + IReadOnlyList lexemes, + int index, + IReadOnlyList<(int Start, int End)> typeBodies) + { + foreach (var body in typeBodies) + { + if (index <= body.Start || index >= body.End) + continue; + + var braceDepth = 0; + for (var current = body.Start + 1; current < index; current++) + { + if (lexemes[current].Text == "{") + braceDepth++; + else if (lexemes[current].Text == "}" && braceDepth > 0) + braceDepth--; + } + if (braceDepth == 0) + return true; + } + return false; + } + private static bool IsDeclarationTypeBefore( IReadOnlyList lexemes, CSharpSemanticTokenKind?[] kinds, diff --git a/tests/CodeIndex.Tests/LspServerTests.cs b/tests/CodeIndex.Tests/LspServerTests.cs index cf7a12fb9c..c3bc73f565 100644 --- a/tests/CodeIndex.Tests/LspServerTests.cs +++ b/tests/CodeIndex.Tests/LspServerTests.cs @@ -1060,6 +1060,18 @@ public void HandleMessage_SemanticTokens_SharesCSharpDeclarationAndUsageKinds_Is " void IFoo.Execute() { }", " string IFoo.Label => Name;", " private void Helper(T value) { _ = $\"{value}\"; }", + " [Obsolete()]", + " private void ReviewCases()", + " {", + " int record = 0, @class = 1;", + " using var stream = Open();", + " var text = 1.ToString();", + " _ = @class;", + " async();", + " }", + " private IDisposable Open() => null!;", + " public int Size { get { int nested = 1; return nested; } }", + " private void async() { }", "}", ]); File.WriteAllText(sourcePath, source); @@ -1117,6 +1129,20 @@ public void HandleMessage_SemanticTokens_SharesCSharpDeclarationAndUsageKinds_Is AssertSemanticToken(tokens, 21, "Label", 9, 1); AssertSemanticToken(tokens, 22, "value", 7, 1); AssertSemanticToken(tokens, 22, "value", 7, 0); + AssertSemanticToken(tokens, 23, "Obsolete", 1, 0); + AssertSemanticToken(tokens, 24, "ReviewCases", 13, 1); + AssertSemanticToken(tokens, 26, "record", 8, 1); + AssertSemanticToken(tokens, 26, "@class", 8, 1); + AssertSemanticToken(tokens, 27, "stream", 8, 1); + AssertSemanticToken(tokens, 27, "Open", 13, 0); + AssertSemanticToken(tokens, 28, "1", 19, 0); + AssertSemanticToken(tokens, 28, "ToString", 13, 0); + AssertSemanticToken(tokens, 29, "@class", 8, 0); + AssertSemanticToken(tokens, 30, "async", 13, 0); + AssertSemanticToken(tokens, 32, "Open", 13, 1); + AssertSemanticToken(tokens, 33, "nested", 8, 1); + AssertSemanticToken(tokens, 33, "nested", 8, 0); + AssertSemanticToken(tokens, 34, "async", 13, 1); Assert.DoesNotContain( tokens.SelectMany((left, index) => tokens.Skip(index + 1).Select(right => (left, right))), pair => diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs index a26257e727..f8ff79ed98 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs @@ -10464,12 +10464,24 @@ public void RunExcerpt_JsonCSharpSemanticKindsMatchLspClassifier_Issue4852() " void IFoo.Execute() { }", " string IFoo.Label => Name;", " private void Helper(T value) { _ = $\"{value}\"; }", + " [Obsolete()]", + " private void ReviewCases()", + " {", + " int record = 0, @class = 1;", + " using var stream = Open();", + " var text = 1.ToString();", + " _ = @class;", + " async();", + " }", + " private IDisposable Open() => null!;", + " public int Size { get { int nested = 1; return nested; } }", + " private void async() { }", "}", ]); TestProjectHelper.InsertIndexedFile(dbPath, "src/Sample.cs", "csharp", source); var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunExcerpt( - ["src/Sample.cs", "--db", dbPath, "--start", "1", "--end", "24", "--json"], + ["src/Sample.cs", "--db", dbPath, "--start", "1", "--end", "36", "--json"], _jsonOptions)); using var document = ParseJsonOutput(stdout); @@ -10511,6 +10523,19 @@ public void RunExcerpt_JsonCSharpSemanticKindsMatchLspClassifier_Issue4852() AssertExcerptToken(21, "Execute", "method", declaration: true); AssertExcerptToken(22, "Label", "property", declaration: true); AssertExcerptToken(23, "value", "parameter", declaration: true); + AssertExcerptToken(24, "Obsolete", "type"); + AssertExcerptToken(25, "ReviewCases", "method", declaration: true); + AssertExcerptToken(27, "record", "variable", declaration: true); + AssertExcerptToken(27, "@class", "variable", declaration: true); + AssertExcerptToken(28, "stream", "variable", declaration: true); + AssertExcerptToken(28, "Open", "method"); + AssertExcerptToken(29, "1", "number"); + AssertExcerptToken(29, "ToString", "method"); + AssertExcerptToken(30, "@class", "variable"); + AssertExcerptToken(31, "async", "method"); + AssertExcerptToken(33, "Open", "method", declaration: true); + AssertExcerptToken(34, "nested", "variable", declaration: true); + AssertExcerptToken(35, "async", "method", declaration: true); var (partialExitCode, partialStdout, partialStderr) = CaptureConsole(() => QueryCommandRunner.RunExcerpt( ["src/Sample.cs", "--db", dbPath, "--start", "16", "--end", "17", "--json"], @@ -10579,6 +10604,19 @@ public void RunExcerpt_JsonCSharpSemanticTokensUseVisibleRangeBudget_Issue4852() "src/Large.cs", "csharp", source.ToString()); + using (var db = new DbContext(DbOpenIntent.WriteIndex, dbPath)) + { + var boundedSource = new DbReader(db) + .GetIndexedSourceLinesForSemanticTokens( + "src/Large.cs", + maxLines: 10, + maxCharacters: 96); + Assert.InRange(boundedSource.Count, 1, 10); + Assert.InRange( + boundedSource.Sum(line => (line?.Length ?? 0) + 1), + 1, + 96); + } const int targetSourceLine = 4_003; var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunExcerpt(