diff --git a/nix/opencode.nix b/nix/opencode.nix index a22f7d3d2473..e0537230f274 100644 --- a/nix/opencode.nix +++ b/nix/opencode.nix @@ -1,5 +1,6 @@ { lib, + pkgs, stdenvNoCC, callPackage, bun, @@ -13,97 +14,152 @@ writableTmpDirAsHomeHook, node_modules ? callPackage ./node-modules.nix { }, }: -stdenvNoCC.mkDerivation (finalAttrs: { - pname = "opencode"; - inherit (node_modules) version src; - inherit node_modules; - - nativeBuildInputs = [ - bun - nodejs # for patchShebangs node_modules - installShellFiles - makeBinaryWrapper - models-dev - writableTmpDirAsHomeHook - ]; - - postPatch = '' - # NOTE: Relax Bun version check to be a warning instead of an error - substituteInPlace packages/script/src/index.ts \ - --replace-fail 'throw new Error(`This script requires bun@''${expectedBunVersionRange}' \ - 'console.warn(`Warning: This script requires bun@''${expectedBunVersionRange}' - ''; - - configurePhase = '' - runHook preConfigure - - cp -R ${finalAttrs.node_modules}/. . - patchShebangs node_modules - patchShebangs packages/*/node_modules - - runHook postConfigure - ''; - - env.MODELS_DEV_API_JSON = "${models-dev}/dist/_api.json"; - env.OPENCODE_DISABLE_MODELS_FETCH = true; - env.OPENCODE_VERSION = finalAttrs.version; - env.OPENCODE_CHANNEL = "prod"; - - buildPhase = '' - runHook preBuild - - cd ./packages/opencode - bun --bun ./script/build.ts --single --skip-install - bun --bun ./script/schema.ts schema.json - - runHook postBuild - ''; - - installPhase = '' - runHook preInstall - - install -Dm755 dist/opencode-*/bin/opencode $out/bin/opencode - install -Dm644 schema.json $out/share/opencode/schema.json - - wrapProgram $out/bin/opencode \ - --prefix PATH : ${ - lib.makeBinPath ( - [ - ripgrep - ] - # bun runs sysctl to detect if running on rosetta2 - ++ lib.optional stdenvNoCC.hostPlatform.isDarwin sysctl +stdenvNoCC.mkDerivation ( + finalAttrs: + let + # Grammar assets fetched content-addressed by Nix. build.ts falls back to the + # network when OPENTUI_BUNDLED_GRAMMARS_DIR is absent. + bundledGrammarAssets = builtins.fromJSON ( + builtins.readFile ../packages/opencode/script/bundled-grammars.json + ); + + # Group assets by language prefix so each grammar becomes its own + # derivation. + bundledGrammarGroups = lib.groupBy ( + asset: lib.head (lib.splitString "-" asset.file) + ) bundledGrammarAssets; + + individualGrammars = lib.mapAttrs ( + lang: assets: + pkgs.runCommand "opencode-assets-grammar-${lang}" + { + passthru = { + grammar = lang; + grammarAssets = assets; + }; + } + ( + "mkdir -p $out\n" + + lib.concatMapStringsSep "\n" ( + asset: + "cp ${ + pkgs.fetchurl { + url = asset.url; + sha256 = asset.sha256; + } + } $out/${asset.file}" + ) assets ) - } - - runHook postInstall - ''; - - postInstall = lib.optionalString (stdenvNoCC.buildPlatform.canExecute stdenvNoCC.hostPlatform) '' - # trick yargs into also generating zsh completions - installShellCompletion --cmd opencode \ - --bash <($out/bin/opencode completion) \ - --zsh <(SHELL=/bin/zsh $out/bin/opencode completion) - ''; - - nativeInstallCheckInputs = [ - versionCheckHook - writableTmpDirAsHomeHook - ]; - doInstallCheck = true; - versionCheckKeepEnvironment = [ "HOME" "OPENCODE_DISABLE_MODELS_FETCH" ]; - versionCheckProgramArg = "--version"; - - passthru = { - jsonschema = "${placeholder "out"}/share/opencode/schema.json"; - env = finalAttrs.env; - }; - - meta = { - description = "The open source coding agent"; - homepage = "https://opencode.ai"; - license = lib.licenses.mit; - mainProgram = "opencode"; - inherit (node_modules.meta) platforms; - }; -}) + ) bundledGrammarGroups; + + grammarDerivations = lib.attrValues individualGrammars; + + treeSitterGrammars = + pkgs.runCommand "opencode-tree-sitter-grammars" + { passthru = { inherit bundledGrammarAssets individualGrammars; }; } + ( + "mkdir -p $out\n" + + lib.concatMapStringsSep "\n" (grammar: "cp -r ${grammar}/. $out/") grammarDerivations + ); + + in + { + pname = "opencode"; + inherit (node_modules) version src; + inherit node_modules; + + nativeBuildInputs = [ + bun + nodejs # for patchShebangs node_modules + installShellFiles + makeBinaryWrapper + models-dev + writableTmpDirAsHomeHook + ]; + + postPatch = '' + # NOTE: Relax Bun version check to be a warning instead of an error + substituteInPlace packages/script/src/index.ts \ + --replace-fail 'throw new Error(`This script requires bun@''${expectedBunVersionRange}' \ + 'console.warn(`Warning: This script requires bun@''${expectedBunVersionRange}' + ''; + + configurePhase = '' + runHook preConfigure + + cp -R ${finalAttrs.node_modules}/. . + patchShebangs node_modules + patchShebangs packages/*/node_modules + + runHook postConfigure + ''; + + env.MODELS_DEV_API_JSON = "${models-dev}/dist/_api.json"; + env.OPENCODE_DISABLE_MODELS_FETCH = true; + env.OPENTUI_BUNDLED_GRAMMARS_DIR = "${treeSitterGrammars}"; + env.OPENCODE_VERSION = finalAttrs.version; + env.OPENCODE_CHANNEL = "prod"; + + buildPhase = '' + runHook preBuild + + cd ./packages/opencode + bun --bun ./script/build.ts --single --skip-install + bun --bun ./script/schema.ts schema.json + + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + + install -Dm755 dist/opencode-*/bin/opencode $out/bin/opencode + install -Dm644 schema.json $out/share/opencode/schema.json + + wrapProgram $out/bin/opencode \ + --prefix PATH : ${ + lib.makeBinPath ( + [ + ripgrep + ] + # bun runs sysctl to detect if running on rosetta2 + ++ lib.optional stdenvNoCC.hostPlatform.isDarwin sysctl + ) + } + + runHook postInstall + ''; + + postInstall = lib.optionalString (stdenvNoCC.buildPlatform.canExecute stdenvNoCC.hostPlatform) '' + # trick yargs into also generating zsh completions + installShellCompletion --cmd opencode \ + --bash <($out/bin/opencode completion) \ + --zsh <(SHELL=/bin/zsh $out/bin/opencode completion) + ''; + + nativeInstallCheckInputs = [ + versionCheckHook + writableTmpDirAsHomeHook + ]; + doInstallCheck = true; + versionCheckKeepEnvironment = [ + "HOME" + "OPENCODE_DISABLE_MODELS_FETCH" + ]; + versionCheckProgramArg = "--version"; + + passthru = { + jsonschema = "${placeholder "out"}/share/opencode/schema.json"; + env = finalAttrs.env; + inherit treeSitterGrammars; + }; + + meta = { + description = "The open source coding agent"; + homepage = "https://opencode.ai"; + license = lib.licenses.mit; + mainProgram = "opencode"; + inherit (node_modules.meta) platforms; + }; + } +) diff --git a/packages/opencode/script/build.ts b/packages/opencode/script/build.ts index ce45f770f04e..1619a10a7c00 100755 --- a/packages/opencode/script/build.ts +++ b/packages/opencode/script/build.ts @@ -1,6 +1,7 @@ #!/usr/bin/env bun import { $ } from "bun" +import fs from "fs" import path from "path" import { fileURLToPath } from "url" import { createSolidTransformPlugin } from "@opentui/solid/bun-plugin" @@ -50,6 +51,103 @@ const createEmbeddedWebUIBundle = async () => { const embeddedFileMap = skipEmbedWebUi ? null : await createEmbeddedWebUIBundle() const treeSitterWorker = await Bun.file(fileURLToPath(import.meta.resolve("@opentui/core/parser.worker"))).text() +// Resolve grammar assets from Nix or network fetch +import { bundledGrammars } from "./bundled-grammars" +import { existsSync } from "node:fs" + +// Verify known hashes for assets +async function assertSha256(bytes: Uint8Array, expected: string, url: string) { + const digest = await crypto.subtle.digest("SHA-256", bytes) + const actual = Buffer.from(digest).toString("hex") + if (actual !== expected) { + throw new Error(`Checksum mismatch for ${url}: expected ${expected}, got ${actual}`) + } +} + +async function resolveGrammarAsset(spec: { + url: string + sha256: string + file: string +}): Promise> { + const localDir = process.env.OPENTUI_BUNDLED_GRAMMARS_DIR + if (localDir) { + const local = path.join(localDir, spec.file) + if (existsSync(local)) { + const bytes = new Uint8Array((await Bun.file(local).arrayBuffer()) as ArrayBuffer) + await assertSha256(bytes, spec.sha256, spec.url) + return bytes + } + } + const res = await fetch(spec.url) + if (!res.ok) throw new Error(`Failed to fetch bundled grammar asset ${spec.url}: ${res.status}`) + const bytes = new Uint8Array((await res.arrayBuffer()) as ArrayBuffer) + await assertSha256(bytes, spec.sha256, spec.url) + return bytes +} + +const grammarBytes: Record< + string, + { wasm: Uint8Array; queries: Record[]> } +> = Object.fromEntries( + await Promise.all( + Object.entries(bundledGrammars).map(async ([name, grammar]) => [ + name, + { + wasm: await resolveGrammarAsset(grammar.wasm), + queries: Object.fromEntries( + await Promise.all( + Object.entries(grammar.queries).map(async ([qkey, assets]) => [ + qkey, + await Promise.all(assets.map((asset) => resolveGrammarAsset(asset))), + ]), + ), + ), + }, + ]), + ), +) + +// Write grammar assets to a gitignored dir and generate the TUI resolver. +const grammarDir = path.join(dir, "gen", "grammars") +fs.rmSync(grammarDir, { recursive: true, force: true }) +fs.mkdirSync(grammarDir, { recursive: true }) +for (const [name, grammar] of Object.entries(bundledGrammars)) { + await Bun.write(path.join(grammarDir, grammar.wasm.file), grammarBytes[name]!.wasm) + for (const [qkey, assets] of Object.entries(grammar.queries)) { + for (let i = 0; i < assets.length; i++) { + await Bun.write(path.join(grammarDir, assets[i]!.file), grammarBytes[name]!.queries[qkey]![i]!) + } + } +} + +const resolverPath = path.join(dir, "../tui/src/bundled-grammars.gen.ts") +const grammarRel = (file: string) => + path.relative(path.dirname(resolverPath), path.join(grammarDir, file)).replaceAll("\\", "/") +const importLine = (binding: string, file: string) => + `import ${binding} from ${JSON.stringify(grammarRel(file))} with { type: "file" };` +const resolverImports = Object.entries(bundledGrammars).flatMap(([name, grammar]) => [ + importLine(`wasm_${name}`, grammar.wasm.file), + ...Object.entries(grammar.queries).flatMap(([qkey, assets]) => + assets.map((asset, i) => importLine(`${qkey}_${name}_${i}`, asset.file)), + ), +]) +const resolverEntries = Object.entries(bundledGrammars).map( + ([name, grammar]) => + ` ${JSON.stringify(name)}: { wasm: wasm_${name}, queries: { ${Object.entries(grammar.queries) + .map(([qkey, assets]) => `${qkey}: [${assets.map((_, i) => `${qkey}_${name}_${i}`).join(", ")}]`) + .join(", ")} } },`, +) +const resolverContent = [ + "// Generated by packages/opencode/script/build.ts. Do not edit.", + ...resolverImports, + "", + "export default {", + ...resolverEntries, + "}", + "", +].join("\n") +const committedStub = await Bun.file(resolverPath).text() + const allTargets: { os: string arch: "arm64" | "x64" @@ -142,94 +240,102 @@ if (!skipInstall) { await $`bun install --os="*" --cpu="*" @parcel/watcher@${pkg.dependencies["@parcel/watcher"]}` await $`bun install --os="*" --cpu="*" @ff-labs/fff-bun@${pkg.dependencies["@ff-labs/fff-bun"]}` } -for (const item of targets) { - const name = [ - pkg.name, - // changing to win32 flags npm for some reason - item.os === "win32" ? "windows" : item.os, - item.arch, - item.avx2 === false ? "baseline" : undefined, - item.abi === undefined ? undefined : item.abi, - ] - .filter(Boolean) - .join("-") - console.log(`building ${name}`) - await $`mkdir -p dist/${name}/bin` - - const workerPath = "./src/cli/tui/worker.ts" - const treeSitterWorkerPath = "opentui-tree-sitter-worker.js" - const bunfsRoot = item.os === "win32" ? "B:/~BUN/root/" : "/$bunfs/root/" - - await Bun.build({ - conditions: ["bun", "node"], - tsconfig: "./tsconfig.json", - plugins: [plugin], - external: ["node-gyp"], - format: "esm", - minify: true, - sourcemap: sourcemapsFlag ? "linked" : "none", - splitting: true, - compile: { - autoloadBunfig: false, - autoloadDotenv: false, - autoloadTsconfig: true, - autoloadPackageJson: true, - target: name.replace(pkg.name, "bun") as any, - outfile: `dist/${name}/bin/opencode`, - execArgv: [`--user-agent=opencode/${Script.version}`, "--use-system-ca", "--"], - windows: {}, - }, - files: { - [treeSitterWorkerPath]: treeSitterWorker, - ...(embeddedFileMap ? { "opencode-web-ui.gen.ts": embeddedFileMap } : {}), - }, - entrypoints: [ - "./src/index.ts", - workerPath, - treeSitterWorkerPath, - ...(embeddedFileMap ? ["opencode-web-ui.gen.ts"] : []), - ], - define: { - FFF_LIBC: JSON.stringify(item.abi === "musl" ? "musl" : "gnu"), - OPENCODE_VERSION: `'${Script.version}'`, - OPENCODE_MODELS_DEV: generated.modelsData, - OTUI_TREE_SITTER_WORKER_PATH: bunfsRoot + treeSitterWorkerPath, - OPENCODE_WORKER_PATH: workerPath, - OPENCODE_CHANNEL: `'${Script.channel}'`, - OPENCODE_LIBC: item.os === "linux" ? `'${item.abi ?? "glibc"}'` : "", - ...(item.os === "linux" ? { "process.env.OPENTUI_LIBC": JSON.stringify(item.abi ?? "glibc") } : {}), - }, - }) +const resolverEntrypoint = path.relative(dir, resolverPath).replaceAll("\\", "/") +try { + await Bun.write(resolverPath, resolverContent) + for (const item of targets) { + const name = [ + pkg.name, + // changing to win32 flags npm for some reason + item.os === "win32" ? "windows" : item.os, + item.arch, + item.avx2 === false ? "baseline" : undefined, + item.abi === undefined ? undefined : item.abi, + ] + .filter(Boolean) + .join("-") + console.log(`building ${name}`) + await $`mkdir -p dist/${name}/bin` - // Smoke test: only run if binary is for current platform - if (item.os === process.platform && item.arch === process.arch && !item.abi) { - const binaryPath = `dist/${name}/bin/opencode` - console.log(`Running smoke test: ${binaryPath} --version`) - try { - const versionOutput = await $`${binaryPath} --version`.text() - console.log(`Smoke test passed: ${versionOutput.trim()}`) - } catch (e) { - console.error(`Smoke test failed for ${name}:`, e) - process.exit(1) - } - } + const workerPath = "./src/cli/tui/worker.ts" + const treeSitterWorkerPath = "opentui-tree-sitter-worker.js" + const bunfsRoot = item.os === "win32" ? "B:/~BUN/root/" : "/$bunfs/root/" - await $`rm -rf ./dist/${name}/bin/tui` - await Bun.file(`dist/${name}/package.json`).write( - JSON.stringify( - { - name, - version: Script.version, - preferUnplugged: true, - os: [item.os], - cpu: [item.arch], - ...(item.abi ? { libc: [item.abi] } : {}), + await Bun.build({ + conditions: ["bun", "node"], + tsconfig: "./tsconfig.json", + plugins: [plugin], + external: ["node-gyp"], + format: "esm", + minify: true, + sourcemap: sourcemapsFlag ? "linked" : "none", + splitting: true, + compile: { + autoloadBunfig: false, + autoloadDotenv: false, + autoloadTsconfig: true, + autoloadPackageJson: true, + target: name.replace(pkg.name, "bun") as any, + outfile: `dist/${name}/bin/opencode`, + execArgv: [`--user-agent=opencode/${Script.version}`, "--use-system-ca", "--"], + windows: {}, + }, + files: { + [treeSitterWorkerPath]: treeSitterWorker, + ...(embeddedFileMap ? { "opencode-web-ui.gen.ts": embeddedFileMap } : {}), }, - null, - 2, - ), - ) - binaries[name] = Script.version + entrypoints: [ + "./src/index.ts", + workerPath, + treeSitterWorkerPath, + ...(embeddedFileMap ? ["opencode-web-ui.gen.ts"] : []), + resolverEntrypoint, + ], + define: { + FFF_LIBC: JSON.stringify(item.abi === "musl" ? "musl" : "gnu"), + OPENCODE_VERSION: `'${Script.version}'`, + OPENCODE_MODELS_DEV: generated.modelsData, + OTUI_TREE_SITTER_WORKER_PATH: bunfsRoot + treeSitterWorkerPath, + OPENCODE_WORKER_PATH: workerPath, + OPENCODE_CHANNEL: `'${Script.channel}'`, + OPENCODE_LIBC: item.os === "linux" ? `'${item.abi ?? "glibc"}'` : "", + ...(item.os === "linux" ? { "process.env.OPENTUI_LIBC": JSON.stringify(item.abi ?? "glibc") } : {}), + }, + }) + + // Smoke test: only run if binary is for current platform + if (item.os === process.platform && item.arch === process.arch && !item.abi) { + const binaryPath = `dist/${name}/bin/opencode` + console.log(`Running smoke test: ${binaryPath} --version`) + try { + const versionOutput = await $`${binaryPath} --version`.text() + console.log(`Smoke test passed: ${versionOutput.trim()}`) + } catch (e) { + console.error(`Smoke test failed for ${name}:`, e) + process.exit(1) + } + } + + await $`rm -rf ./dist/${name}/bin/tui` + await Bun.file(`dist/${name}/package.json`).write( + JSON.stringify( + { + name, + version: Script.version, + preferUnplugged: true, + os: [item.os], + cpu: [item.arch], + ...(item.abi ? { libc: [item.abi] } : {}), + }, + null, + 2, + ), + ) + binaries[name] = Script.version + } +} finally { + // Restore the committed stub so the working tree stays clean. + await Bun.write(resolverPath, committedStub) } if (Script.release) { diff --git a/packages/opencode/script/bundled-grammars.json b/packages/opencode/script/bundled-grammars.json new file mode 100644 index 000000000000..9b3602bc126e --- /dev/null +++ b/packages/opencode/script/bundled-grammars.json @@ -0,0 +1,427 @@ +[ + { + "url": "https://github.com/tree-sitter/tree-sitter-agda/releases/download/v1.3.3/tree-sitter-agda.wasm", + "sha256": "8d88926c064b19b756814b6633090e0a63674675e6663ff81b6cf16ec7c60d6e", + "file": "agda-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/agda/highlights.scm", + "sha256": "124f3246955696b0a7d33c750be29977efe256faf9702ed292b1c5680736cf6b", + "file": "agda-highlights.scm" + }, + { + "url": "https://github.com/tree-sitter/tree-sitter-bash/releases/download/v0.25.0/tree-sitter-bash.wasm", + "sha256": "364f0a2cd385c792239423026ef442dbd073d34c396b7bc9e5932426b8e4aa5d", + "file": "bash-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/bash/highlights.scm", + "sha256": "a4e5e1afa7656c3275629467170362997a7b36f51405b0fc0e6c0be080693acd", + "file": "bash-highlights.scm" + }, + { + "url": "https://github.com/tree-sitter/tree-sitter-c/releases/download/v0.24.1/tree-sitter-c.wasm", + "sha256": "c852c2a85ebf2beb636aa3b0ef7f7e70458684d74f6741b20dcb296885bed9f9", + "file": "c-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/c/highlights.scm", + "sha256": "2c473f14247cb168380627ebafe911ebc07052eb71c52950946380156be964f7", + "file": "c-highlights.scm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/c/locals.scm", + "sha256": "b3ecf04dadb49555af03644686fb68c3ce3ccfd98af33f003371beb5a37652b0", + "file": "c-locals.scm" + }, + { + "url": "https://github.com/anomalyco/tree-sitter-clojure/releases/download/v0.0.1/tree-sitter-clojure.wasm", + "sha256": "90f8866b74d04e1643ed73cce25720c732df2aaa38b45da9dcc773bae8181c27", + "file": "clojure-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/clojure/highlights.scm", + "sha256": "73c7916d54213dcd734de3ad5d890ddea08237074b65120472c86a733c032c76", + "file": "clojure-highlights.scm" + }, + { + "url": "https://github.com/tree-sitter/tree-sitter-cpp/releases/download/v0.23.4/tree-sitter-cpp.wasm", + "sha256": "174eb0deb75b2ec7881bcacda9f995648d8e683956e5c2267e69ab6dc503fcbf", + "file": "cpp-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/cpp/highlights.scm", + "sha256": "a10230678cc32f0c45eab0d3109f5bbef9fa65210efb9f0576d0a3ebacc752a8", + "file": "cpp-highlights.scm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/cpp/locals.scm", + "sha256": "7047179aee75f5e85b0fadd1da4a1856b32d68d6694952738a257ca13a5e4a52", + "file": "cpp-locals.scm" + }, + { + "url": "https://github.com/tree-sitter/tree-sitter-c-sharp/releases/download/v0.23.1/tree-sitter-c_sharp.wasm", + "sha256": "d7fe0bf8e8b8d26b2b253f375e3c43386b332d23f7b7b3e15d49fff180fe46e3", + "file": "csharp-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/c_sharp/highlights.scm", + "sha256": "6cb8448f7cacca8986f44336f3a386a7a84088e667b5ca16406b59e1cc43e4fd", + "file": "csharp-highlights.scm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/c_sharp/locals.scm", + "sha256": "7dccbd815d6033de319708ef27d82350af5acafefeb5f4314e5b8a525cb85e80", + "file": "csharp-locals.scm" + }, + { + "url": "https://github.com/tree-sitter/tree-sitter-css/releases/download/v0.25.0/tree-sitter-css.wasm", + "sha256": "8a23977fe271357cce6f254ef88c9bebf3854602d8046605aef6a45c02135c59", + "file": "css-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/css/highlights.scm", + "sha256": "0001d93c2787981b132f3316a6c18c63ffb82b582c80ea5b025613161cbd75c3", + "file": "css-highlights.scm" + }, + { + "url": "https://github.com/tree-sitter-grammars/tree-sitter-diff/releases/download/v0.1.0/tree-sitter-diff.wasm", + "sha256": "60409c8a17dd5a804e5119887c5b1322f0e780f1976e27d81a1e251dc92e2761", + "file": "diff-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/tree-sitter-grammars/tree-sitter-diff/v0.2.0/queries/highlights.scm", + "sha256": "6e7ac233333886d61fa0cd4186c36b7ab34fa95a86aea618ed87b60f9eaa4e4b", + "file": "diff-highlights.scm" + }, + { + "url": "https://github.com/elixir-lang/tree-sitter-elixir/releases/download/v0.3.5/tree-sitter-elixir.wasm", + "sha256": "1818ae0d7343f880abb0b7932c9a815f09896ca67b3e126096a2dd7677d1477b", + "file": "elixir-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/elixir/highlights.scm", + "sha256": "a5ba25c44d681d11af4d611a8e229463f91219ced15a3ca240a6c215df1f869a", + "file": "elixir-highlights.scm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/elixir/locals.scm", + "sha256": "e5cf2d79b4872bec580e66be8ec4b5ab559efdbe2d0bcb633786ce8ffeb0e95b", + "file": "elixir-locals.scm" + }, + { + "url": "https://github.com/ionide/tree-sitter-fsharp/releases/download/0.3.0/tree-sitter-fsharp.wasm", + "sha256": "f7c54b09d3bcff0fd44dcd605d989eb2cdd107555cec09271eb085f1617a28d4", + "file": "fsharp-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/fsharp/highlights.scm", + "sha256": "6c7f8ddf4b4852663dedcf1e6007cb9cced49279c86fe36d02de863311430c46", + "file": "fsharp-highlights.scm" + }, + { + "url": "https://github.com/tree-sitter/tree-sitter-go/releases/download/v0.25.0/tree-sitter-go.wasm", + "sha256": "9504573f352b20be7f2f1911754d710622aedc15afff16d5ed8fb5645681aee7", + "file": "go-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/go/highlights.scm", + "sha256": "81698c638abbf4c3a3ec8ccc3c6110be2cc725ad740bf87e6fa9ac685ecad267", + "file": "go-highlights.scm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/go/locals.scm", + "sha256": "06e596ade8e1650c453be232c1dab5974f387a554b032b5a28c2a9ea69685d1e", + "file": "go-locals.scm" + }, + { + "url": "https://github.com/tree-sitter/tree-sitter-haskell/releases/download/v0.23.1/tree-sitter-haskell.wasm", + "sha256": "37a6b07b1a838d02ffb4f4c2a06863637a8efe48432d60a275f50f1d08f1092c", + "file": "haskell-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/haskell/highlights.scm", + "sha256": "80a7d4729cd9786faa7ebe83eb1030eb9174993cfd981975d258fcb422f972f5", + "file": "haskell-highlights.scm" + }, + { + "url": "https://github.com/tree-sitter-grammars/tree-sitter-hcl/releases/download/v1.2.0/tree-sitter-hcl.wasm", + "sha256": "2f9acf63e7c263215f283e2cf0f4ebbb9bfa77f58e18fa42b91094be201372a7", + "file": "hcl-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/hcl/highlights.scm", + "sha256": "553290666870af8165121a20914d5d3b549f5e3ba250773f06a0f331dcf9fcff", + "file": "hcl-highlights.scm" + }, + { + "url": "https://github.com/tree-sitter/tree-sitter-html/releases/download/v0.23.2/tree-sitter-html.wasm", + "sha256": "c48fcd82c7ea8bf943180088ba7f28c48b2bb5287874179168bf9d31e394cf85", + "file": "html-tree-sitter.wasm" + }, + { + "url": "https://github.com/tree-sitter/tree-sitter-html/raw/refs/tags/v0.23.2/queries/highlights.scm", + "sha256": "1ebb3811a8cdc054385b847a3aac6fbf7079faefd5b3dfab5bbad256cd5afdcf", + "file": "html-highlights.scm" + }, + { + "url": "https://github.com/tree-sitter/tree-sitter-java/releases/download/v0.23.5/tree-sitter-java.wasm", + "sha256": "4fdeac4ca6ca089f06c6f7e562abcac1733cd465728cc7031ebb73c2019122c4", + "file": "java-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/java/highlights.scm", + "sha256": "93860577090333f2ae2ceac1896617c03efd3d8c2e55ab5f65b556afc5f5dc19", + "file": "java-highlights.scm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/java/locals.scm", + "sha256": "8e0d7f8f8c26f42cffcae485ec308aae9ad980a646837759381a2b8ad85df3ca", + "file": "java-locals.scm" + }, + { + "url": "https://github.com/tree-sitter/tree-sitter-json/releases/download/v0.24.8/tree-sitter-json.wasm", + "sha256": "d2119fb98d5912719b13f9458574f8608d2d29dfbe45f6be1f860ea1fe2a2405", + "file": "json-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/json/highlights.scm", + "sha256": "a7860933f0da33d282b46e697fd2ea8102fae34d44e1a2e6bfdc39b910975bd1", + "file": "json-highlights.scm" + }, + { + "url": "https://github.com/tree-sitter/tree-sitter-julia/releases/download/v0.23.1/tree-sitter-julia.wasm", + "sha256": "e0f52c36eadf0299e46fccd6715c760d35eaa3f09721bec38633da551ac9e781", + "file": "julia-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/julia/highlights.scm", + "sha256": "7bb322f3c048a7e951b64e4c8a0bc97e782034f95dff15385b946eded569d94d", + "file": "julia-highlights.scm" + }, + { + "url": "https://github.com/fwcd/tree-sitter-kotlin/releases/download/0.3.8/tree-sitter-kotlin.wasm", + "sha256": "c624e7443b371c28adc5d81674e73067564c12555ebe3ed96a6c8db814b7602d", + "file": "kotlin-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/fwcd/tree-sitter-kotlin/0.3.8/queries/highlights.scm", + "sha256": "2d74bb308bc589c2dadd9a599613f750f51470a8974190e09e24ec4786bab987", + "file": "kotlin-highlights.scm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/kotlin/locals.scm", + "sha256": "c97567b90fc0d306f594d9820bf3e89bd5db19fab4efcae32f31831602c4786e", + "file": "kotlin-locals.scm" + }, + { + "url": "https://github.com/tree-sitter-grammars/tree-sitter-lua/releases/download/v0.5.0/tree-sitter-lua.wasm", + "sha256": "df08a1704e504c70b8dba4a3e6f8e0c99a4fb94e1b1693d2969f53141d09f0d4", + "file": "lua-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/tree-sitter-grammars/tree-sitter-lua/v0.5.0/queries/highlights.scm", + "sha256": "d2625fb712357f86386878602d18845f9f3e90fee81f02a5ecd87be6512918d1", + "file": "lua-highlights.scm" + }, + { + "url": "https://raw.githubusercontent.com/tree-sitter-grammars/tree-sitter-lua/v0.5.0/queries/locals.scm", + "sha256": "0a72d6880e9d8d6ef7715914d02ece1f08fb148eac715b195191030da8aedd2f", + "file": "lua-locals.scm" + }, + { + "url": "https://github.com/tree-sitter-grammars/tree-sitter-make/releases/download/v1.1.1/tree-sitter-make.wasm", + "sha256": "1254d1e5787be8bb944d55d1f6552b6b2b7908272ed498013bfdc07bd7d042dd", + "file": "make-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/make/highlights.scm", + "sha256": "d286db89e1d5675911e01afa614546590991b7478601573da19ce1890ef42322", + "file": "make-highlights.scm" + }, + { + "url": "https://github.com/ast-grep/ast-grep.github.io/raw/40b84530640aa83a0d34a20a2b0623d7b8e5ea97/website/public/parsers/tree-sitter-nix.wasm", + "sha256": "10a66e15b62be1a59cd256abf4b48fdeac07e50c0a8e50ceb64296fcec019930", + "file": "nix-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/nix/highlights.scm", + "sha256": "56fd3a06df812415ea583b363624539d625dd7b2dfbc37522a1653eb07ac9fef", + "file": "nix-highlights.scm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/nix/locals.scm", + "sha256": "7534473cf89cfcc51d590b8fe2b88a9ce94b64ee6c0104f550f0ff6c35e5a900", + "file": "nix-locals.scm" + }, + { + "url": "https://github.com/tree-sitter/tree-sitter-ocaml/releases/download/v0.24.2/tree-sitter-ocaml.wasm", + "sha256": "761a78a804931cfac1fa0c6238989b4b0e86cc70db461b1315d743de923f8246", + "file": "ocaml-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/ocaml/highlights.scm", + "sha256": "6a079d18ce60c3af404df0e7ef1f2aca9c5f332b4962c0d80a190b574ccaf36c", + "file": "ocaml-highlights.scm" + }, + { + "url": "https://github.com/tree-sitter/tree-sitter-php/releases/download/v0.24.2/tree-sitter-php.wasm", + "sha256": "d4df6a6ff08c87c3ec4f9cbb785fe09998a0cb570e03f57d7b19b3acfb146aa7", + "file": "php-tree-sitter.wasm" + }, + { + "url": "https://github.com/tree-sitter/tree-sitter-php/raw/refs/tags/v0.24.2/queries/highlights.scm", + "sha256": "a2a7367659cff3b4be09961c8117d69a9b8703bfc266f8092e089b1760f197e9", + "file": "php-highlights.scm" + }, + { + "url": "https://github.com/tree-sitter/tree-sitter-python/releases/download/v0.23.6/tree-sitter-python.wasm", + "sha256": "8c93692fb368e288a5824cee55773c9b3602804f513bda48c97661e52e9c2da2", + "file": "python-tree-sitter.wasm" + }, + { + "url": "https://github.com/tree-sitter/tree-sitter-python/raw/refs/tags/v0.25.0/queries/highlights.scm", + "sha256": "a6708f209381618e2b398972c8f1ccd892f0c064eab35a2a3f911c3e22e79a7e", + "file": "python-highlights.scm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/python/locals.scm", + "sha256": "370635358588e6d66e6065dd9fb1e21d864e8eb125abbf4728e573bc06ef4921", + "file": "python-locals.scm" + }, + { + "url": "https://github.com/r-lib/tree-sitter-r/releases/download/v1.2.0/tree-sitter-r.wasm", + "sha256": "2a8f5acd1c53d91e0ec5c01a6830d8ac7f5a7f96f0ac4b3768c016c8e9d07711", + "file": "r-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/r/highlights.scm", + "sha256": "e230ee7d1732694dd805143417865442170717c6c1e7a776d45288bbd2fbb207", + "file": "r-highlights.scm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/r/locals.scm", + "sha256": "f492d0f06b7716a5e6ecd606ef5b09bf7362481ad46e01dfedb4637a5b232e3c", + "file": "r-locals.scm" + }, + { + "url": "https://github.com/tree-sitter/tree-sitter-ruby/releases/download/v0.23.1/tree-sitter-ruby.wasm", + "sha256": "09a96427d7c72f0613ed470cd9812223fc4a91d6a9c025c0235cc6bd59ff96f4", + "file": "ruby-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/ruby/highlights.scm", + "sha256": "361117cdee5e2b6a3e262791c49e7e6e0a3c604e4596fce56d6aabe28d8b1294", + "file": "ruby-highlights.scm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/ruby/locals.scm", + "sha256": "d507cbb78f380559f314429ae9d2d654cea7abbe3f261c40ae71c3b97ea270e1", + "file": "ruby-locals.scm" + }, + { + "url": "https://github.com/tree-sitter/tree-sitter-rust/releases/download/v0.24.0/tree-sitter-rust.wasm", + "sha256": "44b8d1a2e307ee8933d7c0bcb6b0f30d56bc999999fd1d2d9608a7dcf6e8ce56", + "file": "rust-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/rust/highlights.scm", + "sha256": "4e0f5994f30ee545fc63435054a0b029c7d0222632c5c787c4e9c249dc0ef20f", + "file": "rust-highlights.scm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/rust/locals.scm", + "sha256": "a31540dbd462ba6699a5e494764aee2bbfdaea560915d708f6da4d4df4ad3ec5", + "file": "rust-locals.scm" + }, + { + "url": "https://github.com/tree-sitter/tree-sitter-scala/releases/download/v0.24.0/tree-sitter-scala.wasm", + "sha256": "6cd77d9152bb634b1690d8e9ce8c0b23aa0d84c018e113260f7b11ce54446d7b", + "file": "scala-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/scala/highlights.scm", + "sha256": "f739d4ce8c58b5e325a7614359df2f5da3cf7ed77188daac75d259c7ebda9193", + "file": "scala-highlights.scm" + }, + { + "url": "https://github.com/alex-pinkus/tree-sitter-swift/releases/download/0.7.1/tree-sitter-swift.wasm", + "sha256": "e179468b0f3c30c29d8dd8e57c5a79e67812623046cc18fa593f8563795c0b77", + "file": "swift-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/alex-pinkus/tree-sitter-swift/0.7.3/queries/highlights.scm", + "sha256": "b0bbeb918904b252e8bd361f4afb4756647232391110a67deecbf7227464aff8", + "file": "swift-highlights.scm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/swift/locals.scm", + "sha256": "25a2cc839769cdd69791e9db4832fd232844b7e69a86231a34d046c55d883e52", + "file": "swift-locals.scm" + }, + { + "url": "https://github.com/tree-sitter-grammars/tree-sitter-toml/releases/download/v0.7.0/tree-sitter-toml.wasm", + "sha256": "1ac6a83826c35a68857f8325e00c78f6bcbef4eb6db3931a6cf3041f76e5e09f", + "file": "toml-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/toml/highlights.scm", + "sha256": "059618709a4c7e6b287643a68282257e197c5a05cfb9d82b6ac517d32230cb12", + "file": "toml-highlights.scm" + }, + { + "url": "https://github.com/tree-sitter-grammars/tree-sitter-vim/releases/download/v0.8.1/tree-sitter-vim.wasm", + "sha256": "93cc332cbd093d163ba3d3b7ff223eacc023e6f5135088018a1c5e86ca627bec", + "file": "vim-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/vim/highlights.scm", + "sha256": "baed811290e2e15639d6dbbd15acad7c4d76eee24dd3d59a5ffa9e3e36dfde0a", + "file": "vim-highlights.scm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/vim/locals.scm", + "sha256": "d0a6fba18d8434bd4820aeb687bb587ec1a6bcf3e2b9e89d96537a7fe26a22db", + "file": "vim-locals.scm" + }, + { + "url": "https://github.com/anomalyco/tree-sitter-vue/releases/download/v0.1.2/tree-sitter-vue.wasm", + "sha256": "94d9292677797c4d0299009885dfca5453513f44b79405efbf9d7dd3e1a1b5a2", + "file": "vue-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/anomalyco/tree-sitter-vue/v0.1.2/queries/html_tags/highlights.scm", + "sha256": "2b64708e3cea1527adca4841247e28b90229c367722471c1d250e8925d1cc6a4", + "file": "vue-highlights.scm" + }, + { + "url": "https://raw.githubusercontent.com/anomalyco/tree-sitter-vue/v0.1.2/queries/vue/highlights.scm", + "sha256": "43cae0c256b1236a051e856eb4dec18a157d70a00e9d41304549b858c534cefc", + "file": "vue-highlights-2.scm" + }, + { + "url": "https://github.com/tree-sitter-grammars/tree-sitter-xml/releases/download/v0.7.0/tree-sitter-xml.wasm", + "sha256": "9950327f8542a3d563fe4192aa05986a53e6fd6adca518cbb1a1699cbc8970ad", + "file": "xml-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/xml/highlights.scm", + "sha256": "e3fb22e8840e0991b698cd87756ee28dcbb322c9657f0043070a94b9cc93cfc9", + "file": "xml-highlights.scm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/xml/locals.scm", + "sha256": "8c08a6775731ea6ccecfdcc5bdd5c59a8349c00c2b3f9d9f69ceb481712a7305", + "file": "xml-locals.scm" + }, + { + "url": "https://github.com/tree-sitter-grammars/tree-sitter-yaml/releases/download/v0.7.2/tree-sitter-yaml.wasm", + "sha256": "0a0e5ebcfeb0b2bf272d071396eeed107b2c4b2617b69d5959b13f78b357a4d6", + "file": "yaml-tree-sitter.wasm" + }, + { + "url": "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/074aa4422bf029908338e855d0c0f71470a971bb/runtime/queries/yaml/highlights.scm", + "sha256": "13352b80d2ce28c3328ccb2f321e5144c3365ff6055078e00e7cff118d1488b5", + "file": "yaml-highlights.scm" + } +] diff --git a/packages/opencode/script/bundled-grammars.ts b/packages/opencode/script/bundled-grammars.ts new file mode 100644 index 000000000000..4fd5899a78d2 --- /dev/null +++ b/packages/opencode/script/bundled-grammars.ts @@ -0,0 +1,46 @@ +// Grammar manifest for offline syntax highlighting. `build.ts` resolves +// assets from Nix or the network fetch. The flat asset list lives in +// bundled-grammars.json and is grouped here by language prefix. +import bundledGrammarAssets from "./bundled-grammars.json" + +export interface BundledGrammarAsset { + url: string + sha256: string + /** basename used when embedding into $bunfs and when Nix provides the file */ + file: string +} + +export interface BundledGrammar { + wasm: BundledGrammarAsset + queries: Record +} + +function languagePrefix(file: string): string { + return file.slice(0, file.indexOf("-")) +} + +function groupByLanguage(assets: BundledGrammarAsset[]): Record { + const groups: Record = {} + for (const asset of assets) { + const lang = languagePrefix(asset.file) + ;(groups[lang] ??= []).push(asset) + } + return groups +} + +function buildGrammar(assets: BundledGrammarAsset[]): BundledGrammar { + const wasm = assets.find((a) => a.file.endsWith(".wasm"))! + const queries: Record = {} + for (const asset of assets) { + if (asset === wasm) continue + // e.g. "python-highlights.scm" -> "highlights", "vue-highlights-2.scm" -> "highlights" + const rest = asset.file.slice(languagePrefix(asset.file).length + 1) + const key = rest.replace(/-?\d+\.scm$/, ".scm").replace(/\.scm$/, "") + ;(queries[key] ??= []).push(asset) + } + return { wasm, queries } +} + +export const bundledGrammars: Record = Object.fromEntries( + Object.entries(groupByLanguage(bundledGrammarAssets)).map(([lang, assets]) => [lang, buildGrammar(assets)]), +) diff --git a/packages/opencode/test/bundled-grammars.test.ts b/packages/opencode/test/bundled-grammars.test.ts new file mode 100644 index 000000000000..34149574a44d --- /dev/null +++ b/packages/opencode/test/bundled-grammars.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, test } from "bun:test" +import { bundledGrammars } from "../script/bundled-grammars" + +describe("bundled tree-sitter grammars", () => { + test("embeds every grammar under unique basenames", () => { + const basenames = Object.values(bundledGrammars).flatMap((g) => [ + g.wasm.file, + ...Object.values(g.queries).flatMap((assets) => assets.map((a) => a.file)), + ]) + expect(basenames.length).toBeGreaterThan(0) + expect(new Set(basenames).size).toBe(basenames.length) + }) + + test("pins a sha256 for every asset", () => { + for (const grammar of Object.values(bundledGrammars)) { + expect(grammar.wasm.sha256).toMatch(/^[a-f0-9]{64}$/) + for (const assets of Object.values(grammar.queries)) { + for (const asset of assets) expect(asset.sha256).toMatch(/^[a-f0-9]{64}$/) + } + } + }) +}) diff --git a/packages/tui/src/bundled-grammars.gen.ts b/packages/tui/src/bundled-grammars.gen.ts new file mode 100644 index 000000000000..4aa2c08a0533 --- /dev/null +++ b/packages/tui/src/bundled-grammars.gen.ts @@ -0,0 +1,3 @@ +// build.ts replaces this with `import { type: "file" }` paths during the compile step, +// then restores the stub so `bun typecheck` and `bun dev` work without generated files. +export default {} as Record }> diff --git a/packages/tui/src/parsers-config.ts b/packages/tui/src/parsers-config.ts index 0450c4d2eab2..68059c598fe2 100644 --- a/packages/tui/src/parsers-config.ts +++ b/packages/tui/src/parsers-config.ts @@ -1,3 +1,13 @@ +// The committed stub exports `{}`, so dev falls back to pinned URLs. During a build, +// `build.ts` replaces it with embedded `$bunfs` paths via `import { type: "file" }`. +import bundled from "./bundled-grammars.gen" +import { bundledGrammars } from "../../opencode/script/bundled-grammars" + +const bundledAsset = (name: string, kind: string) => { + const paths = bundled[name]?.queries?.[kind] + return paths?.length ? paths : (bundledGrammars[name]?.queries[kind]?.map((asset) => asset.url) ?? []) +} + export default { // NOTE: FOR markdown, javascript and typescript, we use the opentui built-in parsers // Warn: when taking queries from the nvim-treesitter repo, make sure to include the query dependencies as well @@ -6,150 +16,110 @@ export default { parsers: [ { filetype: "python", - wasm: "https://github.com/tree-sitter/tree-sitter-python/releases/download/v0.23.6/tree-sitter-python.wasm", + wasm: bundled["python"]?.wasm ?? bundledGrammars.python.wasm.url, queries: { - highlights: [ - // NOTE: This nvim-treesitter query is currently broken, because the parser is not compatible with the query apparently. - // it is using "except" nodes that the parser is complaining about, but it has been in the query for 3+ years. - // Unclear. - // "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/python/highlights.scm", - "https://github.com/tree-sitter/tree-sitter-python/raw/refs/heads/master/queries/highlights.scm", - ], - locals: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/python/locals.scm", - ], + // NOTE: This nvim-treesitter query is currently broken, because the parser is not compatible with the query apparently. + // it is using "except" nodes that the parser is complaining about, but it has been in the query for 3+ years. + // Unclear. + // "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/python/highlights.scm", + highlights: bundledAsset("python", "highlights"), + locals: bundledAsset("python", "locals"), }, }, { filetype: "rust", - wasm: "https://github.com/tree-sitter/tree-sitter-rust/releases/download/v0.24.0/tree-sitter-rust.wasm", + wasm: bundled["rust"]?.wasm ?? bundledGrammars.rust.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/rust/highlights.scm", - ], - locals: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/rust/locals.scm", - ], + highlights: bundledAsset("rust", "highlights"), + locals: bundledAsset("rust", "locals"), }, }, { filetype: "go", - wasm: "https://github.com/tree-sitter/tree-sitter-go/releases/download/v0.25.0/tree-sitter-go.wasm", + wasm: bundled["go"]?.wasm ?? bundledGrammars.go.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/go/highlights.scm", - ], - locals: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/go/locals.scm", - ], + highlights: bundledAsset("go", "highlights"), + locals: bundledAsset("go", "locals"), }, }, { filetype: "cpp", - wasm: "https://github.com/tree-sitter/tree-sitter-cpp/releases/download/v0.23.4/tree-sitter-cpp.wasm", + wasm: bundled["cpp"]?.wasm ?? bundledGrammars.cpp.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/cpp/highlights.scm", - ], - locals: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/cpp/locals.scm", - ], + highlights: bundledAsset("cpp", "highlights"), + locals: bundledAsset("cpp", "locals"), }, }, { filetype: "csharp", - wasm: "https://github.com/tree-sitter/tree-sitter-c-sharp/releases/download/v0.23.1/tree-sitter-c_sharp.wasm", + wasm: bundled["csharp"]?.wasm ?? bundledGrammars.csharp.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/c_sharp/highlights.scm", - ], - locals: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/c_sharp/locals.scm", - ], + highlights: bundledAsset("csharp", "highlights"), + locals: bundledAsset("csharp", "locals"), }, }, { filetype: "bash", - wasm: "https://github.com/tree-sitter/tree-sitter-bash/releases/download/v0.25.0/tree-sitter-bash.wasm", + wasm: bundled["bash"]?.wasm ?? bundledGrammars.bash.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/bash/highlights.scm", - ], + highlights: bundledAsset("bash", "highlights"), }, }, { filetype: "c", - wasm: "https://github.com/tree-sitter/tree-sitter-c/releases/download/v0.24.1/tree-sitter-c.wasm", + wasm: bundled["c"]?.wasm ?? bundledGrammars.c.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/c/highlights.scm", - ], - locals: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/c/locals.scm", - ], + highlights: bundledAsset("c", "highlights"), + locals: bundledAsset("c", "locals"), }, }, { filetype: "java", - wasm: "https://github.com/tree-sitter/tree-sitter-java/releases/download/v0.23.5/tree-sitter-java.wasm", + wasm: bundled["java"]?.wasm ?? bundledGrammars.java.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/java/highlights.scm", - ], - locals: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/java/locals.scm", - ], + highlights: bundledAsset("java", "highlights"), + locals: bundledAsset("java", "locals"), }, }, { filetype: "kotlin", - wasm: "https://github.com/fwcd/tree-sitter-kotlin/releases/download/0.3.8/tree-sitter-kotlin.wasm", + wasm: bundled["kotlin"]?.wasm ?? bundledGrammars.kotlin.wasm.url, queries: { - highlights: ["https://raw.githubusercontent.com/fwcd/tree-sitter-kotlin/0.3.8/queries/highlights.scm"], - locals: ["https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/master/queries/kotlin/locals.scm"], + highlights: bundledAsset("kotlin", "highlights"), + locals: bundledAsset("kotlin", "locals"), }, }, { filetype: "ruby", - wasm: "https://github.com/tree-sitter/tree-sitter-ruby/releases/download/v0.23.1/tree-sitter-ruby.wasm", + wasm: bundled["ruby"]?.wasm ?? bundledGrammars.ruby.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/ruby/highlights.scm", - ], - locals: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/ruby/locals.scm", - ], + highlights: bundledAsset("ruby", "highlights"), + locals: bundledAsset("ruby", "locals"), }, }, { filetype: "php", - wasm: "https://github.com/tree-sitter/tree-sitter-php/releases/download/v0.24.2/tree-sitter-php.wasm", + wasm: bundled["php"]?.wasm ?? bundledGrammars.php.wasm.url, queries: { - highlights: [ - // NOTE: This nvim-treesitter query is currently broken, because the parser is not compatible with the query apparently. - // "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/php/highlights.scm", - "https://github.com/tree-sitter/tree-sitter-php/raw/refs/heads/master/queries/highlights.scm", - ], + // NOTE: This nvim-treesitter query is currently broken, because the parser is not compatible with the query apparently. + // "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/php/highlights.scm", + highlights: bundledAsset("php", "highlights"), }, }, { filetype: "scala", - wasm: "https://github.com/tree-sitter/tree-sitter-scala/releases/download/v0.24.0/tree-sitter-scala.wasm", + wasm: bundled["scala"]?.wasm ?? bundledGrammars.scala.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/scala/highlights.scm", - ], + highlights: bundledAsset("scala", "highlights"), }, }, { filetype: "html", - wasm: "https://github.com/tree-sitter/tree-sitter-html/releases/download/v0.23.2/tree-sitter-html.wasm", + wasm: bundled["html"]?.wasm ?? bundledGrammars.html.wasm.url, queries: { - highlights: [ - // NOTE: This nvim-treesitter query is currently broken, because the parser is not compatible with the query apparently. - // "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/html/highlights.scm", - "https://github.com/tree-sitter/tree-sitter-html/raw/refs/heads/master/queries/highlights.scm", - ], + // NOTE: This nvim-treesitter query is currently broken, because the parser is not compatible with the query apparently. + // "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/html/highlights.scm", + highlights: bundledAsset("html", "highlights"), // TODO: Injections not working for some reason // injections: [ // "https://github.com/tree-sitter/tree-sitter-html/raw/refs/heads/master/queries/injections.scm", @@ -168,218 +138,163 @@ export default { }, { filetype: "vue", - wasm: "https://github.com/anomalyco/tree-sitter-vue/releases/download/v0.1.2/tree-sitter-vue.wasm", + wasm: bundled["vue"]?.wasm ?? bundledGrammars.vue.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/anomalyco/tree-sitter-vue/v0.1.2/queries/html_tags/highlights.scm", - "https://raw.githubusercontent.com/anomalyco/tree-sitter-vue/v0.1.2/queries/vue/highlights.scm", - ], + highlights: bundledAsset("vue", "highlights"), }, }, { filetype: "hcl", - wasm: "https://github.com/tree-sitter-grammars/tree-sitter-hcl/releases/download/v1.2.0/tree-sitter-hcl.wasm", + wasm: bundled["hcl"]?.wasm ?? bundledGrammars.hcl.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/master/queries/hcl/highlights.scm", - ], + highlights: bundledAsset("hcl", "highlights"), }, }, { filetype: "json", - wasm: "https://github.com/tree-sitter/tree-sitter-json/releases/download/v0.24.8/tree-sitter-json.wasm", + wasm: bundled["json"]?.wasm ?? bundledGrammars.json.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/json/highlights.scm", - ], + highlights: bundledAsset("json", "highlights"), }, }, { filetype: "yaml", - wasm: "https://github.com/tree-sitter-grammars/tree-sitter-yaml/releases/download/v0.7.2/tree-sitter-yaml.wasm", + wasm: bundled["yaml"]?.wasm ?? bundledGrammars.yaml.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/yaml/highlights.scm", - ], + highlights: bundledAsset("yaml", "highlights"), }, }, { filetype: "haskell", - wasm: "https://github.com/tree-sitter/tree-sitter-haskell/releases/download/v0.23.1/tree-sitter-haskell.wasm", + wasm: bundled["haskell"]?.wasm ?? bundledGrammars.haskell.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/haskell/highlights.scm", - ], + highlights: bundledAsset("haskell", "highlights"), }, }, { filetype: "css", - wasm: "https://github.com/tree-sitter/tree-sitter-css/releases/download/v0.25.0/tree-sitter-css.wasm", + wasm: bundled["css"]?.wasm ?? bundledGrammars.css.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/css/highlights.scm", - ], + highlights: bundledAsset("css", "highlights"), }, }, { filetype: "julia", - wasm: "https://github.com/tree-sitter/tree-sitter-julia/releases/download/v0.23.1/tree-sitter-julia.wasm", + wasm: bundled["julia"]?.wasm ?? bundledGrammars.julia.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/julia/highlights.scm", - ], + highlights: bundledAsset("julia", "highlights"), }, }, { filetype: "lua", - wasm: "https://github.com/tree-sitter-grammars/tree-sitter-lua/releases/download/v0.5.0/tree-sitter-lua.wasm", + wasm: bundled["lua"]?.wasm ?? bundledGrammars.lua.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/tree-sitter-grammars/tree-sitter-lua/v0.5.0/queries/highlights.scm", - ], - locals: ["https://raw.githubusercontent.com/tree-sitter-grammars/tree-sitter-lua/v0.5.0/queries/locals.scm"], + highlights: bundledAsset("lua", "highlights"), + locals: bundledAsset("lua", "locals"), }, }, { filetype: "ocaml", - wasm: "https://github.com/tree-sitter/tree-sitter-ocaml/releases/download/v0.24.2/tree-sitter-ocaml.wasm", + wasm: bundled["ocaml"]?.wasm ?? bundledGrammars.ocaml.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/ocaml/highlights.scm", - ], + highlights: bundledAsset("ocaml", "highlights"), }, }, { filetype: "clojure", // temporarily using fork to fix issues - wasm: "https://github.com/anomalyco/tree-sitter-clojure/releases/download/v0.0.1/tree-sitter-clojure.wasm", + wasm: bundled["clojure"]?.wasm ?? bundledGrammars.clojure.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/clojure/highlights.scm", - ], + highlights: bundledAsset("clojure", "highlights"), }, }, { filetype: "swift", - wasm: "https://github.com/alex-pinkus/tree-sitter-swift/releases/download/0.7.1/tree-sitter-swift.wasm", + wasm: bundled["swift"]?.wasm ?? bundledGrammars.swift.wasm.url, queries: { - highlights: [ - // NOTE: Using parser repo queries instead of nvim-treesitter due to incompatible #lua-match? predicates - // "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/highlights.scm - "https://raw.githubusercontent.com/alex-pinkus/tree-sitter-swift/main/queries/highlights.scm", - ], - locals: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/swift/locals.scm", - ], + // NOTE: Using parser repo queries instead of nvim-treesitter due to incompatible #lua-match? predicates + // "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/highlights.scm + highlights: bundledAsset("swift", "highlights"), + locals: bundledAsset("swift", "locals"), }, }, { filetype: "toml", - wasm: "https://github.com/tree-sitter-grammars/tree-sitter-toml/releases/download/v0.7.0/tree-sitter-toml.wasm", + wasm: bundled["toml"]?.wasm ?? bundledGrammars.toml.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/master/queries/toml/highlights.scm", - ], + highlights: bundledAsset("toml", "highlights"), }, }, { filetype: "nix", // TODO: Replace with official tree-sitter-nix WASM when published // See: https://github.com/nix-community/tree-sitter-nix/issues/66 - wasm: "https://github.com/ast-grep/ast-grep.github.io/raw/40b84530640aa83a0d34a20a2b0623d7b8e5ea97/website/public/parsers/tree-sitter-nix.wasm", + wasm: bundled["nix"]?.wasm ?? bundledGrammars.nix.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/nix/highlights.scm", - ], - locals: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/nix/locals.scm", - ], + highlights: bundledAsset("nix", "highlights"), + locals: bundledAsset("nix", "locals"), }, }, { filetype: "diff", aliases: ["udiff", "patch"], - wasm: "https://github.com/tree-sitter-grammars/tree-sitter-diff/releases/download/v0.1.0/tree-sitter-diff.wasm", + wasm: bundled["diff"]?.wasm ?? bundledGrammars.diff.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/tree-sitter-grammars/tree-sitter-diff/master/queries/highlights.scm", - ], + highlights: bundledAsset("diff", "highlights"), }, }, { filetype: "elixir", - wasm: "https://github.com/elixir-lang/tree-sitter-elixir/releases/download/v0.3.5/tree-sitter-elixir.wasm", + wasm: bundled["elixir"]?.wasm ?? bundledGrammars.elixir.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/elixir/highlights.scm", - ], - locals: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/elixir/locals.scm", - ], + highlights: bundledAsset("elixir", "highlights"), + locals: bundledAsset("elixir", "locals"), }, }, { filetype: "fsharp", - wasm: "https://github.com/ionide/tree-sitter-fsharp/releases/download/0.3.0/tree-sitter-fsharp.wasm", + wasm: bundled["fsharp"]?.wasm ?? bundledGrammars.fsharp.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/fsharp/highlights.scm", - ], + highlights: bundledAsset("fsharp", "highlights"), }, }, { filetype: "r", - wasm: "https://github.com/r-lib/tree-sitter-r/releases/download/v1.2.0/tree-sitter-r.wasm", + wasm: bundled["r"]?.wasm ?? bundledGrammars.r.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/r/highlights.scm", - ], - locals: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/r/locals.scm", - ], + highlights: bundledAsset("r", "highlights"), + locals: bundledAsset("r", "locals"), }, }, { filetype: "make", aliases: ["makefile"], - wasm: "https://github.com/tree-sitter-grammars/tree-sitter-make/releases/download/v1.1.1/tree-sitter-make.wasm", + wasm: bundled["make"]?.wasm ?? bundledGrammars.make.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/make/highlights.scm", - ], + highlights: bundledAsset("make", "highlights"), }, }, { filetype: "vim", - wasm: "https://github.com/tree-sitter-grammars/tree-sitter-vim/releases/download/v0.8.1/tree-sitter-vim.wasm", + wasm: bundled["vim"]?.wasm ?? bundledGrammars.vim.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/vim/highlights.scm", - ], - locals: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/vim/locals.scm", - ], + highlights: bundledAsset("vim", "highlights"), + locals: bundledAsset("vim", "locals"), }, }, { filetype: "xml", - wasm: "https://github.com/tree-sitter-grammars/tree-sitter-xml/releases/download/v0.7.0/tree-sitter-xml.wasm", + wasm: bundled["xml"]?.wasm ?? bundledGrammars.xml.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/xml/highlights.scm", - ], - locals: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/xml/locals.scm", - ], + highlights: bundledAsset("xml", "highlights"), + locals: bundledAsset("xml", "locals"), }, }, { filetype: "agda", - wasm: "https://github.com/tree-sitter/tree-sitter-agda/releases/download/v1.3.3/tree-sitter-agda.wasm", + wasm: bundled["agda"]?.wasm ?? bundledGrammars.agda.wasm.url, queries: { - highlights: [ - "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/refs/heads/master/queries/agda/highlights.scm", - ], + highlights: bundledAsset("agda", "highlights"), }, }, ],