From dc47a986cddc09a9b493fcaa1861aeddc1b7ab42 Mon Sep 17 00:00:00 2001 From: Steve Ramage Date: Sun, 21 Jun 2026 11:36:37 -0700 Subject: [PATCH 1/2] feat: grammar-based value completion behind the experimental flag (#467 #343) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cashes in the frontier/expected-set: at the caret, suggest the literal/choice tokens the grammar could accept next. - Combinator.nextTokenChoices(prefix): pure FIRST-set computation — the enumerable choices (LiteralChoiceTerminal / FlexibleLiteralChoiceTerminal) expected at the end of `prefix`, read from the Stuck values at that offset. No IntelliJ types; bounded by maxSteps. - UnitFileValueCompletionContributor: when useGrammarParseEngine is on and the option is a GrammarOptionValue, compute completions from the value text before the caret. It scans split points (largest first) for the tightest word where the grammar expects an enumerable token matching what's typed, then uses resultSet.withPrefixMatcher(word) so the platform filters correctly — otherwise a partial token like "~AF_IN" would be the prefix and match nothing. Falls back to the original getAutoCompleteOptions path otherwise. Polls ProgressManager.checkCanceled between split attempts. Tests: NextTokenChoicesTest (FIRST set incl. context — after "~" offers families not "none"; after "AF_INET " offers another family); GrammarValueCompletionTest drives real completion with the flag on (partial family, and a subsequent family in a list). Full suite green. Refs #467 #343 Co-Authored-By: Claude Opus 4.8 (1M context) --- .../UnitFileValueCompletionContributor.kt | 43 +++++++++++++++++++ .../optionvalues/grammar/Completion.kt | 34 +++++++++++++++ .../completion/GrammarValueCompletionTest.kt | 42 ++++++++++++++++++ .../grammar/NextTokenChoicesTest.kt | 35 +++++++++++++++ 4 files changed, 154 insertions(+) create mode 100644 src/main/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/semanticdata/optionvalues/grammar/Completion.kt create mode 100644 src/test/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/completion/GrammarValueCompletionTest.kt create mode 100644 src/test/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/semanticdata/optionvalues/grammar/NextTokenChoicesTest.kt diff --git a/src/main/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/completion/UnitFileValueCompletionContributor.kt b/src/main/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/completion/UnitFileValueCompletionContributor.kt index 7b29e5fa..25c976d2 100644 --- a/src/main/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/completion/UnitFileValueCompletionContributor.kt +++ b/src/main/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/completion/UnitFileValueCompletionContributor.kt @@ -2,6 +2,7 @@ package net.sjrx.intellij.plugins.systemdunitfiles.completion import com.intellij.codeInsight.completion.* import com.intellij.codeInsight.lookup.LookupElementBuilder +import com.intellij.openapi.progress.ProgressManager import com.intellij.patterns.PlatformPatterns import com.intellij.psi.util.PsiTreeUtil import com.intellij.util.ProcessingContext @@ -11,6 +12,9 @@ import net.sjrx.intellij.plugins.systemdunitfiles.psi.UnitFileProperty import net.sjrx.intellij.plugins.systemdunitfiles.psi.UnitFileSectionGroups import net.sjrx.intellij.plugins.systemdunitfiles.semanticdata.SemanticDataRepository import net.sjrx.intellij.plugins.systemdunitfiles.semanticdata.fileClass +import net.sjrx.intellij.plugins.systemdunitfiles.semanticdata.optionvalues.grammar.GrammarOptionValue +import net.sjrx.intellij.plugins.systemdunitfiles.semanticdata.optionvalues.grammar.nextTokenChoices +import net.sjrx.intellij.plugins.systemdunitfiles.settings.ExperimentalSettings import java.util.function.Supplier import java.util.stream.Collectors @@ -48,6 +52,14 @@ class UnitFileValueCompletionContributor : CompletionContributor() { val fileClass = section.containingFile.fileClass() val validator = sdr.getOptionValidator(fileClass, sectionName, keyName) + + if (ExperimentalSettings.getInstance(property.project).state.useGrammarParseEngine && + validator is GrammarOptionValue + ) { + addGrammarCompletions(parameters, validator, property, resultSet) + return + } + resultSet.addAllElements( validator.getAutoCompleteOptions(property.project) .stream() @@ -59,4 +71,35 @@ class UnitFileValueCompletionContributor : CompletionContributor() { } ) } + + /** + * Experimental grammar-based completion (#467 / #343): suggest the literal/choice tokens the + * grammar could accept at the caret. We read the value text up to the caret, find the tightest + * split where the grammar expects an enumerable token whose choices match what's already typed, + * and set that as the prefix so the platform filters correctly (otherwise a partial token like + * "~AF_IN" would be the prefix and match nothing). + */ + private fun addGrammarCompletions( + parameters: CompletionParameters, + validator: GrammarOptionValue, + property: UnitFileProperty, + resultSet: CompletionResultSet, + ) { + val valueStart = property.valueNode?.psi?.textRange?.startOffset ?: return + val caret = parameters.offset + if (caret < valueStart) return + val pre = parameters.position.containingFile.text.substring(valueStart, caret) + + // Largest split first => tightest (shortest) word being completed. + for (split in pre.length downTo 0) { + ProgressManager.checkCanceled() + val choices = validator.combinator.nextTokenChoices(pre.substring(0, split)) + if (choices.isEmpty()) continue + val word = pre.substring(split) + if (choices.any { it.startsWith(word) }) { + resultSet.withPrefixMatcher(word).addAllElements(choices.map { LookupElementBuilder.create(it) }) + return + } + } + } } diff --git a/src/main/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/semanticdata/optionvalues/grammar/Completion.kt b/src/main/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/semanticdata/optionvalues/grammar/Completion.kt new file mode 100644 index 00000000..76c070bd --- /dev/null +++ b/src/main/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/semanticdata/optionvalues/grammar/Completion.kt @@ -0,0 +1,34 @@ +package net.sjrx.intellij.plugins.systemdunitfiles.semanticdata.optionvalues.grammar + +/* + * Grammar-based completion support (GitHub #467 / #343). + * + * The frontier we built for error localization is exactly what completion needs: "what was expected + * at this offset?" is the same question as "what could come next here?". [nextTokenChoices] answers + * it for the position at the end of [prefix]. + * + * It reads the FIRST set at the end of [prefix] from the parse frontier — the Stuck values whose + * offset is the end of [prefix] — and collects the enumerable choices of the terminals expected + * there (literal choices and flexible-literal choices). Non-enumerable terminals (numbers, regexes, + * whitespace, EOF) contribute nothing to suggest. + * + * Pure Kotlin, no IntelliJ types. The matcher is exhaustive, so [maxSteps] bounds the work; callers + * running on a UI/highlighting thread should also poll cancellation between calls. + */ +fun Combinator.nextTokenChoices(prefix: String, maxSteps: Int = 100_000): Set { + val choices = linkedSetOf() + var steps = 0 + for (step in parse(prefix, 0)) { + if (++steps > maxSteps) break + if (step is Stuck && step.offset == prefix.length) { + for (matcher in step.expected) { + when (matcher) { + is LiteralChoiceTerminal -> choices += matcher.choices + is FlexibleLiteralChoiceTerminal -> choices += matcher.choices + else -> {} // not enumerable — nothing concrete to suggest + } + } + } + } + return choices +} diff --git a/src/test/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/completion/GrammarValueCompletionTest.kt b/src/test/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/completion/GrammarValueCompletionTest.kt new file mode 100644 index 00000000..1dcfc2d0 --- /dev/null +++ b/src/test/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/completion/GrammarValueCompletionTest.kt @@ -0,0 +1,42 @@ +package net.sjrx.intellij.plugins.systemdunitfiles.completion + +import net.sjrx.intellij.plugins.systemdunitfiles.AbstractUnitFileTest +import net.sjrx.intellij.plugins.systemdunitfiles.settings.ExperimentalSettings +import org.junit.Test + +/** + * End-to-end grammar-based completion (#467 / #343), behind the experimental flag. + */ +class GrammarValueCompletionTest : AbstractUnitFileTest() { + + // The light-test project is shared across classes; don't leak the opt-in. + override fun tearDown() { + try { + ExperimentalSettings.getInstance(project).state.useGrammarParseEngine = false + } finally { + super.tearDown() + } + } + + private fun enableNewEngine() { + ExperimentalSettings.getInstance(project).state.useGrammarParseEngine = true + } + + @Test + fun testCompletesPartialAddressFamily() { + enableNewEngine() + setupFileInEditor("file.service", "[Service]\nRestrictAddressFamilies=AF_IN${COMPLETION_POSITION}") + + val results = basicCompletionResultStrings + assertContainsElements(results, "AF_INET", "AF_INET6") + } + + @Test + fun testCompletesSubsequentFamilyInList() { + enableNewEngine() + setupFileInEditor("file.service", "[Service]\nRestrictAddressFamilies=AF_UNIX AF_IN${COMPLETION_POSITION}") + + val results = basicCompletionResultStrings + assertContainsElements(results, "AF_INET", "AF_INET6") + } +} diff --git a/src/test/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/semanticdata/optionvalues/grammar/NextTokenChoicesTest.kt b/src/test/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/semanticdata/optionvalues/grammar/NextTokenChoicesTest.kt new file mode 100644 index 00000000..392feb70 --- /dev/null +++ b/src/test/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/semanticdata/optionvalues/grammar/NextTokenChoicesTest.kt @@ -0,0 +1,35 @@ +package net.sjrx.intellij.plugins.systemdunitfiles.semanticdata.optionvalues.grammar + +import net.sjrx.intellij.plugins.systemdunitfiles.semanticdata.optionvalues.ai.ConfigParseAddressFamiliesOptionValue +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +/** Unit tests for the grammar-completion FIRST-set computation (no IntelliJ fixture needed). */ +class NextTokenChoicesTest { + + private val addressFamilies = ConfigParseAddressFamiliesOptionValue().combinator + + @Test + fun testEmptyValueOffersStartTokens() { + val choices = addressFamilies.nextTokenChoices("") + assertTrue(choices.contains("none")) + assertTrue(choices.contains("~")) + assertTrue(choices.contains("AF_INET")) + } + + @Test + fun testAfterInversionOffersFamiliesNotNone() { + // After "~" the grammar expects an address family, not "none" or another "~". + val choices = addressFamilies.nextTokenChoices("~") + assertTrue(choices.contains("AF_INET")) + assertFalse(choices.contains("none")) + assertFalse(choices.contains("~")) + } + + @Test + fun testAfterFamilyAndSpaceOffersAnotherFamily() { + // Context awareness: after a family and a separator, another family is expected. + assertTrue(addressFamilies.nextTokenChoices("AF_INET ").contains("AF_INET6")) + } +} From 8be705415c9ee17c4b0359a372733027c358ff1e Mon Sep 17 00:00:00 2001 From: Steve Ramage Date: Sun, 21 Jun 2026 11:53:11 -0700 Subject: [PATCH 2/2] fix: complete partial tokens instead of advancing (grammar completion) The first cut used "largest split wins", so for a lenient terminal a partial token like RootImagePolicy=h was treated as a finished partition identifier and completion offered "=" instead of "home" (and AF_IN-style cases only worked by luck). Rework addGrammarCompletions to prefer completing a genuine partial: the longest non-empty trailing word for which the grammar expects an enumerable choice that STRICTLY extends it (something left to type). Only if there's no such partial do we fall back to advancing at a fresh boundary (empty value, or after a complete token like "~" or "root=") with an empty prefix. This keeps the previously-working cases (empty value, after "partition=", "AF_IN") and fixes partial-identifier completion. Adds ImagePolicyCompletionTest covering the empty boundary, partial-identifier completion ("r" -> root/root-verity/...), and the post-"root=" policy-flag position. Refs #467 #343 Co-Authored-By: Claude Opus 4.8 (1M context) --- .../UnitFileValueCompletionContributor.kt | 21 ++++++-- .../completion/ImagePolicyCompletionTest.kt | 48 +++++++++++++++++++ 2 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 src/test/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/completion/ImagePolicyCompletionTest.kt diff --git a/src/main/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/completion/UnitFileValueCompletionContributor.kt b/src/main/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/completion/UnitFileValueCompletionContributor.kt index 25c976d2..ce23ee5e 100644 --- a/src/main/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/completion/UnitFileValueCompletionContributor.kt +++ b/src/main/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/completion/UnitFileValueCompletionContributor.kt @@ -89,17 +89,28 @@ class UnitFileValueCompletionContributor : CompletionContributor() { val caret = parameters.offset if (caret < valueStart) return val pre = parameters.position.containingFile.text.substring(valueStart, caret) + val combinator = validator.combinator - // Largest split first => tightest (shortest) word being completed. - for (split in pre.length downTo 0) { + // Case 1 — completing a partial token. Find the token start: the longest non-empty trailing + // word for which the grammar expects an enumerable choice that STRICTLY extends it (i.e. there's + // more to type). This beats just advancing, so "h" completes to "home" rather than offering "=" + // (the lenient terminal would otherwise treat "h" as a finished identifier). + for (split in 0 until pre.length) { ProgressManager.checkCanceled() - val choices = validator.combinator.nextTokenChoices(pre.substring(0, split)) - if (choices.isEmpty()) continue val word = pre.substring(split) - if (choices.any { it.startsWith(word) }) { + val choices = combinator.nextTokenChoices(pre.substring(0, split)) + if (choices.any { it.length > word.length && it.startsWith(word) }) { resultSet.withPrefixMatcher(word).addAllElements(choices.map { LookupElementBuilder.create(it) }) return } } + + // Case 2 — at a fresh token boundary (e.g. empty value, or after a complete token like "~" or + // "root="). Offer whatever can come next, matched against an empty prefix. + ProgressManager.checkCanceled() + val choices = combinator.nextTokenChoices(pre) + if (choices.isNotEmpty()) { + resultSet.withPrefixMatcher("").addAllElements(choices.map { LookupElementBuilder.create(it) }) + } } } diff --git a/src/test/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/completion/ImagePolicyCompletionTest.kt b/src/test/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/completion/ImagePolicyCompletionTest.kt new file mode 100644 index 00000000..0b0004b9 --- /dev/null +++ b/src/test/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/completion/ImagePolicyCompletionTest.kt @@ -0,0 +1,48 @@ +package net.sjrx.intellij.plugins.systemdunitfiles.completion + +import net.sjrx.intellij.plugins.systemdunitfiles.AbstractUnitFileTest +import net.sjrx.intellij.plugins.systemdunitfiles.settings.ExperimentalSettings +import org.junit.Test + +/** + * Grammar completion for the more complex RootImagePolicy= grammar (#467 / #343), behind the flag. + * Exercises a fresh boundary (empty), completing a partial partition identifier, and the position + * after "partition=" where policy flags are expected. + */ +class ImagePolicyCompletionTest : AbstractUnitFileTest() { + + override fun tearDown() { + try { + ExperimentalSettings.getInstance(project).state.useGrammarParseEngine = false + } finally { + super.tearDown() + } + } + + private fun enableNewEngine() { + ExperimentalSettings.getInstance(project).state.useGrammarParseEngine = true + } + + @Test + fun testEmptyValueOffersPartitionsAndDefault() { + enableNewEngine() + setupFileInEditor("file.service", "[Service]\nRootImagePolicy=${COMPLETION_POSITION}") + assertContainsElements(basicCompletionResultStrings, "root", "usr", "swap", "+") + } + + @Test + fun testPartialPartitionIdentifierIsCompleted() { + // Regression for the bug: typing "r" used to offer "=" (the lenient terminal treated "r" as a + // finished identifier); it should complete the identifier instead. + enableNewEngine() + setupFileInEditor("file.service", "[Service]\nRootImagePolicy=r${COMPLETION_POSITION}") + assertContainsElements(basicCompletionResultStrings, "root", "root-verity", "root-verity-sig") + } + + @Test + fun testAfterPartitionEqualsOffersPolicyFlags() { + enableNewEngine() + setupFileInEditor("file.service", "[Service]\nRootImagePolicy=root=${COMPLETION_POSITION}") + assertContainsElements(basicCompletionResultStrings, "verity", "signed", "encrypted") + } +}