diff --git a/MODULE.bazel b/MODULE.bazel index 818cff34732..f14274944b7 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -150,7 +150,7 @@ crate.from_cargo( ) # audiopus_sys builds its vendored opus via the `cmake` crate. cmake + ninja -# come from @cmake_tools (pinned release binaries, first PATH entry below), so +# come from @cmake_tools (pinned release binaries, first PATH entries below), so # the build no longer cares what the runner image ships — which is also what # lets it run on BuildBuddy RBE workers. The host directories stay behind them # as a fallback. The policy override is what its old CMakeLists needs under @@ -182,9 +182,11 @@ crate.annotation( # the UBSan runtime. Opus shipped without sanitizers under # cargo-zigbuild too. "CFLAGS": "-fno-sanitize=undefined", - # Canonical repo path of @cmake_tools first — keep in sync with the - # cmake tools section below. - "PATH": "$${pwd}/external/+cmake_tools_repository+cmake_tools/bin:/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin:/run/current-system/sw/bin", + # Canonical repo paths of @cmake_tools first — keep in sync with the + # cmake tools section below. cmake and ninja sit in separate directories + # because bin/ is the notarized CMake.app bundle on macOS and macOS + # deletes that bundle if anything is written into it; see tools.bzl. + "PATH": "$${pwd}/external/+cmake_tools_repository+cmake_tools/bin:$${pwd}/external/+cmake_tools_repository+cmake_tools/ninja-bin:/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin:/run/current-system/sw/bin", }, # Stages the pinned cmake/ninja binaries into the build-script action's # sandbox (the PATH entry alone would not make them inputs). Canonical diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 87c14904e74..0cf2a78c62a 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -479,7 +479,7 @@ "@@rules_rust+//crate_universe:extensions.bzl%crate": { "general": { "bzlTransitiveDigest": "4e0uNkFgykXeLgcLZkqWKJbzzSSYqEPlPhfMFR3V4Mw=", - "usagesDigest": "r6mxD9H3+FqXEPGOME5Q27/DtZ4REjqZ/e7ffAsBOwQ=", + "usagesDigest": "tmq/TwdVWJcu4RKDTlL1A9fAMgKASS5pPCIXs0xyF98=", "recordedInputs": [ "ENV:CARGO_BAZEL_DEBUG \\0", "ENV:CARGO_BAZEL_GENERATOR_SHA256 \\0", @@ -1548,7 +1548,7 @@ "https://static.crates.io/crates/audiopus_sys/0.2.2/download" ], "strip_prefix": "audiopus_sys-0.2.2", - "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'oh-my-pi'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"audiopus_sys\",\n deps = [\n \"@crates__audiopus_sys-0.2.2//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"static\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=audiopus_sys\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@@//bazel/triples:aarch64-apple-darwin\": [],\n \"@@//bazel/triples:aarch64-unknown-linux-gnu\": [],\n \"@@//bazel/triples:aarch64-unknown-linux-musl\": [],\n \"@@//bazel/triples:x86_64-apple-darwin\": [],\n \"@@//bazel/triples:x86_64-pc-windows-msvc\": [],\n \"@@//bazel/triples:x86_64-unknown-linux-gnu\": [],\n \"@@//bazel/triples:x86_64-unknown-linux-musl\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.2\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n build_script_env = {\n \"CFLAGS\": \"-fno-sanitize=undefined\",\n \"CMAKE_GENERATOR\": \"Ninja\",\n \"CMAKE_POLICY_VERSION_MINIMUM\": \"3.5\",\n \"CMAKE_TOOLCHAIN_FILE_x86_64_pc_windows_msvc\": \"$${pwd}/external/+msvc_cc_repository+msvc_cc/toolchain.cmake\",\n \"OPUS_NO_PKG\": \"1\",\n \"PATH\": \"$${pwd}/external/+cmake_tools_repository+cmake_tools/bin:/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin:/run/current-system/sw/bin\",\n },\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"static\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n deps = [\n \"@crates__cmake-0.1.58//:cmake\",\n \"@crates__log-0.4.33//:log\",\n \"@crates__pkg-config-0.3.33//:pkg_config\",\n ],\n emit_warnings = False,\n edition = \"2018\",\n pkg_name = \"audiopus_sys\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=audiopus_sys\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n tools = [\n \"@@+cmake_tools_repository+cmake_tools//:tools\",\n ],\n version = \"0.2.2\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" + "build_file_content": "###############################################################################\n# @generated\n# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To \n# regenerate this file, run the following:\n#\n# bazel mod show_repo 'oh-my-pi'\n###############################################################################\n\nload(\n \"@rules_rust//cargo:defs.bzl\",\n \"cargo_build_script\",\n \"cargo_toml_env_vars\",\n)\n\nload(\"@rules_rust//rust:defs.bzl\", \"rust_library\")\n\n# buildifier: disable=bzl-visibility\nload(\"@rules_rust//crate_universe/private:selects.bzl\", \"selects\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\ncargo_toml_env_vars(\n name = \"cargo_toml_env_vars\",\n src = \"Cargo.toml\",\n)\n\nrust_library(\n name = \"audiopus_sys\",\n deps = [\n \"@crates__audiopus_sys-0.2.2//:build_script_build\",\n ],\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"static\",\n ],\n crate_root = \"src/lib.rs\",\n edition = \"2018\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=audiopus_sys\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n target_compatible_with = select({\n \"@@//bazel/triples:aarch64-apple-darwin\": [],\n \"@@//bazel/triples:aarch64-unknown-linux-gnu\": [],\n \"@@//bazel/triples:aarch64-unknown-linux-musl\": [],\n \"@@//bazel/triples:x86_64-apple-darwin\": [],\n \"@@//bazel/triples:x86_64-pc-windows-msvc\": [],\n \"@@//bazel/triples:x86_64-unknown-linux-gnu\": [],\n \"@@//bazel/triples:x86_64-unknown-linux-musl\": [],\n \"//conditions:default\": [\"@platforms//:incompatible\"],\n }),\n version = \"0.2.2\",\n)\n\ncargo_build_script(\n name = \"_bs\",\n build_script_env = {\n \"CFLAGS\": \"-fno-sanitize=undefined\",\n \"CMAKE_GENERATOR\": \"Ninja\",\n \"CMAKE_POLICY_VERSION_MINIMUM\": \"3.5\",\n \"CMAKE_TOOLCHAIN_FILE_x86_64_pc_windows_msvc\": \"$${pwd}/external/+msvc_cc_repository+msvc_cc/toolchain.cmake\",\n \"OPUS_NO_PKG\": \"1\",\n \"PATH\": \"$${pwd}/external/+cmake_tools_repository+cmake_tools/bin:$${pwd}/external/+cmake_tools_repository+cmake_tools/ninja-bin:/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin:/run/current-system/sw/bin\",\n },\n compile_data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \"**/*.rs\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n crate_features = [\n \"default\",\n \"static\",\n ],\n crate_name = \"build_script_build\",\n crate_root = \"build.rs\",\n data = glob(\n allow_empty = True,\n include = [\"**\"],\n exclude = [\n \"**/* *\",\n \".tmp_git_root/**/*\",\n \"BUILD\",\n \"BUILD.bazel\",\n \"WORKSPACE\",\n \"WORKSPACE.bazel\",\n ],\n ),\n deps = [\n \"@crates__cmake-0.1.58//:cmake\",\n \"@crates__log-0.4.33//:log\",\n \"@crates__pkg-config-0.3.33//:pkg_config\",\n ],\n emit_warnings = False,\n edition = \"2018\",\n pkg_name = \"audiopus_sys\",\n rustc_env_files = [\n \":cargo_toml_env_vars\",\n ],\n rustc_flags = [\n \"--cap-lints=allow\",\n ],\n srcs = glob(\n allow_empty = True,\n include = [\"**/*.rs\"],\n ),\n tags = [\n \"cargo-bazel\",\n \"crate-name=audiopus_sys\",\n \"manual\",\n \"noclippy\",\n \"norustfmt\",\n ],\n tools = [\n \"@@+cmake_tools_repository+cmake_tools//:tools\",\n ],\n version = \"0.2.2\",\n visibility = [\"//visibility:private\"],\n)\n\nalias(\n name = \"build_script_build\",\n actual = \":_bs\",\n tags = [\"manual\"],\n)\n" } }, "crates__autocfg-1.5.1": { diff --git a/bazel/toolchains/cmake/tools.bzl b/bazel/toolchains/cmake/tools.bzl index ca04d85caf4..d89205724e4 100644 --- a/bazel/toolchains/cmake/tools.bzl +++ b/bazel/toolchains/cmake/tools.bzl @@ -6,8 +6,13 @@ script's PATH. Historically that PATH pointed at host directories, which made the build depend on whatever the runner image shipped — and broke remote execution outright (BuildBuddy workers carry no cmake). This repo pins the official CMake and Ninja release binaries for the exec host and exposes them -under a single `bin/` directory that the audiopus_sys annotation prepends to -the build-script PATH (see MODULE.bazel). +under `bin/` (cmake) and `ninja-bin/` (ninja), both of which the audiopus_sys +annotation prepends to the build-script PATH (see MODULE.bazel). + +The two directories are not cosmetic. On macOS `bin/` is a symlink into the +Apple-notarized CMake.app, and writing ninja into that bundle makes macOS treat +it as tampered — the write fails and the bundle is deleted, leaving cmake to die +with SIGKILL ("damaged"). ninja therefore gets a directory of its own. Like @llvm_msvc_tools, the archive choice keys off the *host* running the repository rule: with remote execution the exec platform is linux-x64, so @@ -42,7 +47,7 @@ package(default_visibility = ["//visibility:public"]) filegroup( name = "tools", - srcs = glob(["bin/**", "share/**", "CMake.app/**"], allow_empty = True), + srcs = glob(["bin/**", "ninja-bin/**", "share/**", "CMake.app/**"], allow_empty = True), ) """ @@ -77,23 +82,37 @@ def _cmake_tools_impl(rctx): # The macOS archive is an app bundle; surface its bin/ (cmake, and the # share/ data it locates relative to the binary) at the repo top level so # the PATH entry is the same on every host. + # + # CMake.app is Apple-notarized and its resources are sealed, so nothing may + # ever be written *into* it: adding an executable to a signed bundle reads + # as malware persistence, and macOS answers by refusing the write (EPERM) + # and deleting the bundle outright. The symlinked bin/ then dangles, and the + # `cmake` the audiopus_sys build script resolves from PATH either fails + # signature validation — killed with SIGKILL, reported as "damaged" — or is + # simply gone. That is why ninja lands in its own ninja-bin/ below rather + # than joining cmake in bin/, and why doc/man are only pruned when they are + # real top-level directories (linux); on macOS they live inside the bundle. if key.startswith("macos-"): rctx.symlink("CMake.app/Contents/bin", "bin") rctx.symlink("CMake.app/Contents/share", "share") # Drop the bundled GUI/docs where present; only bin/ + share/cmake-* (the # Modules/ tree cmake refuses to run without) matter to build scripts. - for name in ("doc", "man"): - entry = rctx.path(name) - if entry.exists: - rctx.delete(entry) - + if not key.startswith("macos-"): + for name in ("doc", "man"): + entry = rctx.path(name) + if entry.exists: + rctx.delete(entry) + + # ninja-bin/ on every host, not bin/: on macOS bin/ is the notarized bundle + # (see above), and a uniform layout keeps the PATH annotation in MODULE.bazel + # host-independent. ninja_asset, ninja_sha256 = _NINJA_DISTS[key] rctx.report_progress("Downloading Ninja {} ({})".format(_NINJA_VERSION, ninja_asset)) rctx.download_and_extract( url = "https://github.com/ninja-build/ninja/releases/download/v{}/{}".format(_NINJA_VERSION, ninja_asset), sha256 = ninja_sha256, - output = "bin", + output = "ninja-bin", ) rctx.file("BUILD.bazel", _BUILD.format( diff --git a/bun.lock b/bun.lock index d18f94d3bcc..3c929c32f11 100644 --- a/bun.lock +++ b/bun.lock @@ -376,9 +376,9 @@ }, }, "patchedDependencies": { - "puppeteer-core@25.3.0": "patches/puppeteer-core@25.3.0.patch", - "@ark/schema@0.56.2": "patches/@ark%2Fschema@0.56.2.patch", "@agentclientprotocol/sdk@1.2.1": "patches/@agentclientprotocol%2Fsdk@1.2.1.patch", + "@ark/schema@0.56.2": "patches/@ark%2Fschema@0.56.2.patch", + "puppeteer-core@25.3.0": "patches/puppeteer-core@25.3.0.patch", }, "overrides": { "@ark/schema": "0.56.2", diff --git a/docs/aura/FORK.md b/docs/aura/FORK.md index 32434e72088..d0607b5cf4c 100644 --- a/docs/aura/FORK.md +++ b/docs/aura/FORK.md @@ -191,10 +191,11 @@ changes the template sources. - `packages/coding-agent/src/cli/version-identity.ts` — `--version` identity line (`/` + runtime protocol version) - `scripts/build-relocatable-runtime-bundle.ts`, - `scripts/build-relocatable-runtime-bundle.test.ts` — fork-owned Linux x64/glibc - packager and behavioral contract tests for a relocatable standalone Aura binary, + `scripts/build-relocatable-runtime-bundle.test.ts` — fork-owned host packager + (Linux x64/glibc and macOS arm64) and behavioral contract tests for a relocatable + standalone Aura binary, complete Elide distribution, embedded-library sidecars, runtime overlay, launcher, - archive, checksum, and post-extraction verification; the launcher resolves relative, absolute, and chained installation symlinks before deriving its bundle root; the release compiler is the Bun pinned by root `packageManager`, taken from the running interpreter when it already matches and otherwise installed under `$XDG_CACHE_HOME/aura/pinned-bun/` with `bun` trusted so its postinstall materializes a real binary + archive, checksum, and post-extraction verification; the launcher resolves relative, absolute, and chained installation symlinks before deriving its bundle root; the release compiler is the Bun pinned by root `packageManager`, taken from the running interpreter when it already matches and otherwise installed under `$XDG_CACHE_HOME/aura/pinned-bun/` with `bun` trusted so its postinstall materializes a real binary; the host decides the bundle name, library suffix, and linkage probe — `ldd` on Linux, `otool -l` on macOS, where `@rpath` expansion must resolve inside the bundle and archives fall back from GNU tar to bsdtar's narrower reproducibility flags - `packages/metaharness/src/inherent-capability-benchmark.ts`, `src/inherent-capability-benchmark.test.ts`, and `packages/coding-agent/scripts/runtime-telemetry-preflight.ts` — focused current diff --git a/scripts/build-relocatable-runtime-bundle.test.ts b/scripts/build-relocatable-runtime-bundle.test.ts index 33b365cd608..fb0199f58ce 100644 --- a/scripts/build-relocatable-runtime-bundle.test.ts +++ b/scripts/build-relocatable-runtime-bundle.test.ts @@ -3,15 +3,24 @@ import * as fs from "node:fs/promises"; import * as os from "node:os"; import * as path from "node:path"; import { + bsdtarCreateFlags, + embeddedLibraryNames, + hostBundleTarget, + machOLinkagePlan, + parseMachOLoadCommands, pinnedBunManifest, pinnedBunPackage, pinnedBunVersion, + requiredRuntimeArtifacts, resolvePinnedBun, } from "./build-relocatable-runtime-bundle"; const repoRoot = path.join(import.meta.dir, ".."); const scriptPath = path.join(repoRoot, "scripts", "build-relocatable-runtime-bundle.ts"); -const bundleName = "aura-elide-linux-x64"; +// The bundler always targets its host, so the contract tests must follow the host's naming too. +const hostTarget = hostBundleTarget(); +const bundleName = hostTarget.bundleName; +const libraries = embeddedLibraryNames(hostTarget); const tempDirs: string[] = []; afterEach(async () => { @@ -19,7 +28,9 @@ afterEach(async () => { }); async function temporaryDirectory(): Promise { - const directory = await fs.mkdtemp(path.join(os.tmpdir(), "aura-runtime-bundle-")); + // The launcher resolves its own root with `cd -P`, so tests have to compare against real paths: + // macOS hands out /var/folders/... temp dirs that are symlinks into /private/var/folders/.... + const directory = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "aura-runtime-bundle-"))); tempDirs.push(directory); return directory; } @@ -35,8 +46,8 @@ async function createInputs(root: string): Promise<{ runtimeDist: string; auraBi await fs.mkdir(path.join(runtimeDist, "lib"), { recursive: true }); await fs.mkdir(path.join(runtimeDist, "share"), { recursive: true }); await writeExecutable(path.join(runtimeDist, "bin", "elide"), "#!/bin/sh\nexit 0\n"); - await Bun.write(path.join(runtimeDist, "lib", "libelide_embed.so"), "facade"); - await Bun.write(path.join(runtimeDist, "lib", "libelide_embed_engine.so"), "engine"); + await Bun.write(path.join(runtimeDist, "lib", libraries.facade), "facade"); + await Bun.write(path.join(runtimeDist, "lib", libraries.engine), "engine"); await Bun.write(path.join(runtimeDist, "share", "runtime-marker.txt"), "complete distribution"); const auraBinary = path.join(root, "fake-aura"); @@ -117,8 +128,8 @@ describe("relocatable Aura runtime bundle", () => { const bundle = path.join(outputDir, bundleName); expect(await Bun.file(path.join(bundle, "share", "runtime-marker.txt")).text()).toBe("complete distribution"); - expect(await Bun.file(path.join(bundle, "lib", "libelide_embed.so")).text()).toBe("facade"); - expect(await Bun.file(path.join(bundle, "lib", "libelide_embed_engine.so")).text()).toBe("engine"); + expect(await Bun.file(path.join(bundle, "lib", libraries.facade)).text()).toBe("facade"); + expect(await Bun.file(path.join(bundle, "lib", libraries.engine)).text()).toBe("engine"); expect(await Bun.file(path.join(bundle, "etc", "aura-bundle.yml")).text()).toBe( "runtime:\n enabled: true\n adapter: auto\n autoDownload: false\n", ); @@ -137,7 +148,7 @@ describe("relocatable Aura runtime bundle", () => { ]); expect(launchCode, stderr).toBe(0); expect(stdout).toContain(`runtime=${path.join(bundle, "bin", "elide")}`); - expect(stdout).toContain(`embedded=${path.join(bundle, "lib", "libelide_embed.so")}`); + expect(stdout).toContain(`embedded=${path.join(bundle, "lib", libraries.facade)}`); expect(stdout).toContain(`config=${path.join(bundle, "etc", "aura-bundle.yml")}:/user/override.yml`); expect(stdout).toContain("args=hello world"); }); @@ -178,17 +189,13 @@ describe("relocatable Aura runtime bundle", () => { ]); expect(launchCode, stderr).toBe(0); expect(stdout).toContain(`runtime=${path.join(bundle, "bin", "elide")}`); - expect(stdout).toContain(`embedded=${path.join(bundle, "lib", "libelide_embed.so")}`); + expect(stdout).toContain(`embedded=${path.join(bundle, "lib", libraries.facade)}`); expect(stdout).toContain(`config=${path.join(bundle, "etc", "aura-bundle.yml")}:/user/override.yml`); expect(stdout).toContain("args=hello world"); }); test("rejects distributions missing a required runtime artifact", async () => { - for (const relativePath of [ - path.join("bin", "elide"), - path.join("lib", "libelide_embed.so"), - path.join("lib", "libelide_embed_engine.so"), - ]) { + for (const relativePath of requiredRuntimeArtifacts(hostTarget)) { const root = await temporaryDirectory(); const { runtimeDist, auraBinary } = await createInputs(root); await fs.rm(path.join(runtimeDist, relativePath)); @@ -221,9 +228,26 @@ describe("relocatable Aura runtime bundle", () => { exit 0 `, ); + // The fake libraries are not real objects, so the linkage probe has to be stubbed. Each stub + // still reports linkage that the real checker must accept for the smoke run to pass. const fakeBin = path.join(root, "fake-bin"); await fs.mkdir(fakeBin); await writeExecutable(path.join(fakeBin, "ldd"), "#!/bin/sh\necho 'all dependencies resolved'\n"); + await writeExecutable( + path.join(fakeBin, "otool"), + `#!/bin/sh +cat <<'LOADCOMMANDS' + cmd LC_ID_DYLIB + name @rpath/${libraries.facade} (offset 24) + cmd LC_LOAD_DYLIB + name @rpath/${libraries.engine} (offset 24) + cmd LC_LOAD_DYLIB + name /usr/lib/libSystem.B.dylib (offset 24) + cmd LC_RPATH + path @loader_path (offset 12) +LOADCOMMANDS +`, + ); const result = await runBuilder( [ @@ -281,7 +305,7 @@ exit 0 const archive = path.join(outputDir, `${bundleName}.tar.gz`); expect(await Bun.file(archive).exists()).toBe(true); expect((await Bun.file(`${archive}.sha256`).text()).trim()).toMatch( - /^[a-f0-9]{64} {2}aura-elide-linux-x64\.tar\.gz$/, + new RegExp(`^[a-f0-9]{64} {2}${bundleName.replaceAll(".", "\\.")}\\.tar\\.gz$`), ); const listing = Bun.spawn(["tar", "-tzf", archive], { stdout: "pipe", stderr: "pipe" }); @@ -292,6 +316,102 @@ exit 0 ]); expect(listingCode, stderr).toBe(0); expect(stdout).toContain(`${bundleName}/bin/aura\n`); - expect(stdout).toContain(`${bundleName}/lib/libelide_embed_engine.so\n`); + expect(stdout).toContain(`${bundleName}/lib/${libraries.engine}\n`); + }); +}); + +describe("host bundle targets", () => { + test("names the bundle, Aura binary, and embedded libraries per host", () => { + const linux = hostBundleTarget("linux", "x64"); + expect(linux.bundleName).toBe("aura-elide-linux-x64"); + expect(linux.auraBinaryName).toBe("aura-linux-x64"); + expect(embeddedLibraryNames(linux)).toEqual({ + facade: "libelide_embed.so", + engine: "libelide_embed_engine.so", + }); + + const darwin = hostBundleTarget("darwin", "arm64"); + expect(darwin.bundleName).toBe("aura-elide-darwin-arm64"); + expect(darwin.auraBinaryName).toBe("aura-darwin-arm64"); + expect(embeddedLibraryNames(darwin)).toEqual({ + facade: "libelide_embed.dylib", + engine: "libelide_embed_engine.dylib", + }); + }); + + test("rejects hosts with no verified embedded runtime", () => { + expect(() => hostBundleTarget("darwin", "x64")).toThrow( + "Relocatable embedded bundles support linux/x64, darwin/arm64; received darwin/x64.", + ); + expect(() => hostBundleTarget("win32", "x64")).toThrow("received win32/x64"); + }); + + test("omits bsdtar's macOS metadata flag off darwin", () => { + expect(bsdtarCreateFlags("darwin")).toContain("--no-mac-metadata"); + expect(bsdtarCreateFlags("linux")).not.toContain("--no-mac-metadata"); + }); +}); + +describe("Mach-O linkage", () => { + const loadCommands = `Load command 8 + cmd LC_ID_DYLIB + cmdsize 56 + name @rpath/libelide_embed.dylib (offset 24) + time stamp 1 Wed Dec 31 16:00:01 1969 +Load command 9 + cmd LC_LOAD_DYLIB + cmdsize 64 + name @rpath/libelide_embed_engine.dylib (offset 24) + time stamp 2 Wed Dec 31 16:00:02 1969 +Load command 10 + cmd LC_LOAD_DYLIB + cmdsize 56 + name /usr/lib/libSystem.B.dylib (offset 24) +Load command 11 + cmd LC_RPATH + cmdsize 32 + path @loader_path (offset 12) +`; + + test("reads dependencies and rpaths without mistaking the install name for a dependency", () => { + expect(parseMachOLoadCommands(loadCommands)).toEqual({ + rpaths: ["@loader_path"], + dependencies: ["@rpath/libelide_embed_engine.dylib", "/usr/lib/libSystem.B.dylib"], + }); + }); + + test("expands @rpath against the staged bundle and ignores the dyld shared cache", () => { + const plan = machOLinkagePlan( + { libraryDirectory: "/bundle/lib", executableDirectory: "/bundle/bin" }, + parseMachOLoadCommands(loadCommands), + ); + expect(plan.issues).toEqual([]); + // /usr/lib/libSystem.B.dylib has no file behind it, so it must not become a required path. + expect(plan.dependencies).toEqual([ + { + name: "@rpath/libelide_embed_engine.dylib", + candidates: ["/bundle/lib/libelide_embed_engine.dylib"], + }, + ]); + }); + + test("flags linkage that would break once the bundle moves", () => { + const plan = machOLinkagePlan( + { libraryDirectory: "/bundle/lib", executableDirectory: "/bundle/bin" }, + { + rpaths: ["/build/host/lib", "@executable_path/../lib"], + dependencies: ["/opt/homebrew/lib/libz.dylib", "@loader_path/libelide_embed_engine.dylib"], + }, + ); + expect(plan.issues).toEqual([ + "LC_RPATH /build/host/lib is not relative to @loader_path or @executable_path.", + "Dependency /opt/homebrew/lib/libz.dylib points outside the bundle and the system library cache.", + ]); + expect(plan.dependencies).toEqual([ + { + name: "@loader_path/libelide_embed_engine.dylib", + candidates: ["/bundle/lib/libelide_embed_engine.dylib"], + }, + ]); }); }); diff --git a/scripts/build-relocatable-runtime-bundle.ts b/scripts/build-relocatable-runtime-bundle.ts index 19ce30d4d22..d8a1d2c00aa 100755 --- a/scripts/build-relocatable-runtime-bundle.ts +++ b/scripts/build-relocatable-runtime-bundle.ts @@ -4,14 +4,70 @@ import * as fs from "node:fs/promises"; import * as os from "node:os"; import * as path from "node:path"; -const BUNDLE_NAME = "aura-elide-linux-x64"; -const REQUIRED_RUNTIME_ARTIFACTS = [ - path.join("bin", "elide"), - path.join("lib", "libelide_embed.so"), - path.join("lib", "libelide_embed_engine.so"), -] as const; +/** + * A host the bundler can build for. The bundle embeds native WHIPLASH libraries and is smoke + * tested by executing it, so the host and the target are always the same machine — there is no + * cross-building here. + */ +export interface BundleTarget { + /** Release-target id shared with `scripts/ci-release-build-binaries.ts`. */ + id: string; + platform: NodeJS.Platform; + arch: string; + /** Shared-library suffix WHIPLASH emits for this host. */ + librarySuffix: string; + bundleName: string; + auraBinaryName: string; +} + +const BUNDLE_TARGETS: readonly BundleTarget[] = [ + { + id: "linux-x64", + platform: "linux", + arch: "x64", + librarySuffix: ".so", + bundleName: "aura-elide-linux-x64", + auraBinaryName: "aura-linux-x64", + }, + { + id: "darwin-arm64", + platform: "darwin", + arch: "arm64", + librarySuffix: ".dylib", + bundleName: "aura-elide-darwin-arm64", + auraBinaryName: "aura-darwin-arm64", + }, +]; + +export function hostBundleTarget( + platform: NodeJS.Platform = process.platform, + arch: string = process.arch, +): BundleTarget { + const target = BUNDLE_TARGETS.find(candidate => candidate.platform === platform && candidate.arch === arch); + if (!target) { + const supported = BUNDLE_TARGETS.map(candidate => `${candidate.platform}/${candidate.arch}`).join(", "); + throw new Error(`Relocatable embedded bundles support ${supported}; received ${platform}/${arch}.`); + } + return target; +} + +/** The embedded facade the launcher points Aura at, and the engine it dlopens in turn. */ +export function embeddedLibraryNames(target: BundleTarget): { facade: string; engine: string } { + return { + facade: `libelide_embed${target.librarySuffix}`, + engine: `libelide_embed_engine${target.librarySuffix}`, + }; +} + +export function requiredRuntimeArtifacts(target: BundleTarget): string[] { + const libraries = embeddedLibraryNames(target); + return [path.join("bin", "elide"), path.join("lib", libraries.facade), path.join("lib", libraries.engine)]; +} + const BUNDLE_CONFIG = "runtime:\n enabled: true\n adapter: auto\n autoDownload: false\n"; -const BUNDLE_LAUNCHER = `#!/bin/sh + +function bundleLauncher(target: BundleTarget): string { + return `#!/bin/sh set -eu LAUNCHER=$0 @@ -26,7 +82,7 @@ done ROOT=$(CDPATH= cd -P -- "$(dirname -- "$LAUNCHER")/.." && pwd) export AURA_RUNTIME_BIN="$ROOT/bin/elide" -export AURA_RUNTIME_EMBEDDED_LIB="$ROOT/lib/libelide_embed.so" +export AURA_RUNTIME_EMBEDDED_LIB="$ROOT/lib/${embeddedLibraryNames(target).facade}" if [ -n "\${PI_CONFIG_FILES:-}" ]; then export PI_CONFIG_FILES="$ROOT/etc/aura-bundle.yml:$PI_CONFIG_FILES" @@ -36,6 +92,7 @@ fi exec "$ROOT/bin/aura.bin" "$@" `; +} interface BundleOptions { runtimeDist: string; @@ -81,6 +138,8 @@ function pinnedBunCacheRoot(): string { function usage(): string { return `Usage: bun scripts/build-relocatable-runtime-bundle.ts --runtime-dist [options] +Builds a bundle for the host platform (${BUNDLE_TARGETS.map(target => target.id).join(", ")}). + Options: --runtime-dist Complete WHIPLASH distribution (or AURA_RUNTIME_DIST) --output-dir Output parent directory (default: /out) @@ -143,6 +202,10 @@ function parseOptions(argv: string[], cwd = process.cwd()): BundleOptions { }; } +async function isFile(file: string): Promise { + return (await fs.stat(file).catch(() => null))?.isFile() ?? false; +} + async function requireFile(file: string, displayPath: string, executable = false): Promise { const stat = await fs.stat(file).catch(() => null); if (!stat) throw new Error(`Required artifact is missing: ${displayPath}`); @@ -150,8 +213,8 @@ async function requireFile(file: string, displayPath: string, executable = false if (executable && (stat.mode & 0o111) === 0) throw new Error(`Required artifact is not executable: ${displayPath}`); } -async function validateRuntimeDistribution(runtimeDist: string): Promise { - for (const relativePath of REQUIRED_RUNTIME_ARTIFACTS) { +async function validateRuntimeDistribution(runtimeDist: string, target: BundleTarget): Promise { + for (const relativePath of requiredRuntimeArtifacts(target)) { await requireFile(path.join(runtimeDist, relativePath), relativePath, relativePath === path.join("bin", "elide")); } } @@ -214,7 +277,133 @@ export async function resolvePinnedBun(packageManager: string, cacheRoot: string return executable; } -async function resolveAuraBinary(options: BundleOptions, repoRoot: string): Promise { +export interface MachOLoadCommands { + /** `LC_RPATH` entries, in load order — the search list `@rpath` expands against. */ + rpaths: string[]; + /** `LC_LOAD_DYLIB`-style dependencies, excluding the library's own `LC_ID_DYLIB` install name. */ + dependencies: string[]; +} + +const DYLIB_LOAD_COMMANDS = new Set(["LC_LOAD_DYLIB", "LC_LOAD_WEAK_DYLIB", "LC_REEXPORT_DYLIB"]); +/** Absolute dependencies below these live in the dyld shared cache, so no file backs them on disk. */ +const DYLD_SHARED_CACHE_PREFIXES = ["/usr/lib/", "/System/"]; + +/** Parses `otool -l` output into the load commands that decide whether a dylib survives relocation. */ +export function parseMachOLoadCommands(output: string): MachOLoadCommands { + const rpaths: string[] = []; + const dependencies: string[] = []; + let command = ""; + for (const line of output.split("\n")) { + const trimmed = line.trim(); + const commandMatch = /^cmd (LC_\w+)$/.exec(trimmed); + if (commandMatch?.[1]) { + command = commandMatch[1]; + continue; + } + // Both `name` and `path` carry a trailing `(offset N)` that is not part of the value. + const valueMatch = /^(name|path) (.+) \(offset \d+\)$/.exec(trimmed); + if (!valueMatch?.[2]) continue; + if (valueMatch[1] === "path" && command === "LC_RPATH") rpaths.push(valueMatch[2]); + if (valueMatch[1] === "name" && DYLIB_LOAD_COMMANDS.has(command)) dependencies.push(valueMatch[2]); + } + return { rpaths, dependencies }; +} + +export interface MachOLinkagePlan { + /** Linkage that cannot survive relocation, regardless of what exists on this machine. */ + issues: string[]; + /** Bundle-internal dependencies; each must be satisfied by at least one candidate path. */ + dependencies: { name: string; candidates: string[] }[]; +} + +/** + * Resolves a dylib's load commands against the bundle it was staged into. A relocatable bundle + * only references `@rpath`/`@loader_path`/`@executable_path` and the dyld shared cache; anything + * pointing at an absolute build-machine path breaks as soon as the bundle is extracted elsewhere. + */ +export function machOLinkagePlan( + locations: { libraryDirectory: string; executableDirectory: string }, + commands: MachOLoadCommands, +): MachOLinkagePlan { + const issues: string[] = []; + const expand = (reference: string): string | null => { + if (reference.startsWith("@loader_path")) + return path.resolve(locations.libraryDirectory, `.${reference.slice("@loader_path".length)}`); + if (reference.startsWith("@executable_path")) + return path.resolve(locations.executableDirectory, `.${reference.slice("@executable_path".length)}`); + return null; + }; + + const searchPaths: string[] = []; + for (const rpath of commands.rpaths) { + const expanded = expand(rpath); + if (expanded) searchPaths.push(expanded); + else issues.push(`LC_RPATH ${rpath} is not relative to @loader_path or @executable_path.`); + } + + const dependencies: { name: string; candidates: string[] }[] = []; + for (const dependency of commands.dependencies) { + if (dependency.startsWith("@rpath/")) { + const relative = dependency.slice("@rpath/".length); + dependencies.push({ name: dependency, candidates: searchPaths.map(base => path.resolve(base, relative)) }); + continue; + } + const expanded = expand(dependency); + if (expanded) { + dependencies.push({ name: dependency, candidates: [expanded] }); + continue; + } + if (!dependency.startsWith("/")) continue; + if (DYLD_SHARED_CACHE_PREFIXES.some(prefix => dependency.startsWith(prefix))) continue; + issues.push(`Dependency ${dependency} points outside the bundle and the system library cache.`); + } + return { issues, dependencies }; +} + +/** + * Walks the embedded runtime's dependency graph the way dyld will, since `otool` — unlike `ldd` + * — reports one object's direct dependencies and nothing below them. `@loader_path` is relative + * to whichever library is doing the loading, so each object is resolved from its own directory. + */ +async function darwinLinkageIssues(bundle: string, library: string): Promise { + const issues: string[] = []; + const visited = new Set(); + const pending = [library]; + while (pending.length > 0) { + const current = pending.pop(); + if (!current || visited.has(current)) continue; + visited.add(current); + const object = path.relative(bundle, current); + const plan = machOLinkagePlan( + { libraryDirectory: path.dirname(current), executableDirectory: path.join(bundle, "bin") }, + parseMachOLoadCommands( + await runCommand(["otool", "-l", current], { description: `Linkage probe for ${object}` }), + ), + ); + issues.push(...plan.issues.map(issue => `${object}: ${issue}`)); + for (const dependency of plan.dependencies) { + const present = await Promise.all(dependency.candidates.map(candidate => isFile(candidate))); + const resolved = dependency.candidates.find((_, index) => present[index]); + if (resolved) pending.push(resolved); + else issues.push(`${object}: dependency ${dependency.name} is missing from the bundle.`); + } + } + return issues; +} + +/** Rejects an embedded runtime whose dependencies would not resolve after the bundle moves. */ +async function verifyEmbeddedLinkage(bundle: string, target: BundleTarget): Promise { + const library = path.join(bundle, "lib", embeddedLibraryNames(target).facade); + if (target.platform === "darwin") { + const issues = await darwinLinkageIssues(bundle, library); + if (issues.length > 0) throw new Error(`Embedded runtime has unresolved dependencies.\n${issues.join("\n")}`); + return; + } + const linkage = await runCommand(["ldd", library], { description: "Embedded runtime linkage probe" }); + if (linkage.includes("not found")) throw new Error(`Embedded runtime has unresolved dependencies.\n${linkage}`); +} + +async function resolveAuraBinary(options: BundleOptions, repoRoot: string, target: BundleTarget): Promise { if (options.auraBinary) { await requireFile(options.auraBinary, options.auraBinary, true); return options.auraBinary; @@ -225,19 +414,19 @@ async function resolveAuraBinary(options: BundleOptions, repoRoot: string): Prom ? pinnedBunPackage(manifest.packageManager) : pinnedBunPackage(undefined); const bun = await resolvePinnedBun(packageManager, pinnedBunCacheRoot()); - await runCommand([bun, "scripts/ci-release-build-binaries.ts", "--targets", "linux-x64"], { + await runCommand([bun, "scripts/ci-release-build-binaries.ts", "--targets", target.id], { cwd: repoRoot, // Codegen steps inside the release build shell out to `bun`; keep them on the pin too. env: { ...process.env, PATH: `${path.dirname(bun)}${path.delimiter}${process.env.PATH ?? ""}` }, inherit: true, description: `Aura release binary build with ${packageManager}`, }); - const binary = path.join(repoRoot, "packages", "coding-agent", "binaries", "aura-linux-x64"); + const binary = path.join(repoRoot, "packages", "coding-agent", "binaries", target.auraBinaryName); await requireFile(binary, path.relative(repoRoot, binary), true); return binary; } -async function verifyBundle(bundle: string): Promise { +async function verifyBundle(bundle: string, target: BundleTarget): Promise { const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "aura-bundle-smoke-")); const env = { ...process.env, @@ -255,25 +444,27 @@ async function verifyBundle(bundle: string): Promise { await runCommand([launcher, "--version"], { env, description: "Bundled Aura version probe" }); await runCommand([launcher, "--smoke-test"], { env, description: "Bundled Aura worker smoke test" }); await runCommand([launcher, "runtime", "status", "--json"], { env, description: "Bundled runtime status probe" }); - const linkage = await runCommand(["ldd", path.join(bundle, "lib", "libelide_embed.so")], { - description: "Embedded runtime linkage probe", - }); - if (linkage.includes("not found")) throw new Error(`Embedded runtime has unresolved dependencies.\n${linkage}`); + await verifyEmbeddedLinkage(bundle, target); } finally { await fs.rm(agentDir, { recursive: true, force: true }); } } -async function installBundle(runtimeDist: string, auraBinary: string, staging: string): Promise { +async function installBundle( + runtimeDist: string, + auraBinary: string, + staging: string, + target: BundleTarget, +): Promise { await fs.cp(runtimeDist, staging, { recursive: true, dereference: true, preserveTimestamps: true }); const binDir = path.join(staging, "bin"); await fs.mkdir(binDir, { recursive: true }); await fs.copyFile(auraBinary, path.join(binDir, "aura.bin")); await fs.chmod(path.join(binDir, "aura.bin"), 0o755); await Bun.write(path.join(staging, "etc", "aura-bundle.yml"), BUNDLE_CONFIG); - await Bun.write(path.join(binDir, "aura"), BUNDLE_LAUNCHER); + await Bun.write(path.join(binDir, "aura"), bundleLauncher(target)); await fs.chmod(path.join(binDir, "aura"), 0o755); - await validateRuntimeDistribution(staging); + await validateRuntimeDistribution(staging, target); } async function replaceBundle(staging: string, bundle: string): Promise { @@ -287,41 +478,66 @@ async function sha256(file: string): Promise { return hasher.digest("hex"); } +interface ArchiveTool { + command: string; + /** Flags that normalize order, ownership, and timestamps so the archive hashes reproducibly. */ + createFlags: string[]; + deterministic: boolean; +} + +/** + * macOS ships bsdtar as `tar`, which rejects `--sort`, `--owner`, `--group`, and `--mtime` + * outright. Prefer a GNU tar wherever one is installed (`gtar` on Homebrew and the GitHub macOS + * runners) and degrade to bsdtar's narrower set rather than failing the build. + */ +export function bsdtarCreateFlags(platform: NodeJS.Platform = process.platform): string[] { + // Without --no-mac-metadata bsdtar emits an AppleDouble `._` entry beside every file with xattrs. + return ["--numeric-owner", "--uname=", "--gname=", ...(platform === "darwin" ? ["--no-mac-metadata"] : [])]; +} + +async function resolveArchiveTool(): Promise { + for (const command of ["tar", "gtar"]) { + const version = await runCommand([command, "--version"], { description: `${command} version probe` }).catch( + () => null, + ); + if (version?.startsWith("tar (GNU tar)")) { + return { + command, + createFlags: ["--sort=name", "--owner=0", "--group=0", "--numeric-owner", "--mtime=@0"], + deterministic: true, + }; + } + } + return { command: "tar", createFlags: bsdtarCreateFlags(), deterministic: false }; +} + async function archiveBundle( outputDir: string, bundle: string, skipSmoke: boolean, + target: BundleTarget, ): Promise<{ archive: string; checksum: string }> { - const archive = path.join(outputDir, `${BUNDLE_NAME}.tar.gz`); + const archive = path.join(outputDir, `${target.bundleName}.tar.gz`); const checksum = `${archive}.sha256`; await fs.rm(archive, { force: true }); await fs.rm(checksum, { force: true }); - await runCommand( - [ - "tar", - "--sort=name", - "--owner=0", - "--group=0", - "--numeric-owner", - "--mtime=@0", - "-C", - outputDir, - "-czf", - archive, - path.basename(bundle), - ], - { description: "Bundle archive creation" }, - ); + const tar = await resolveArchiveTool(); + if (!tar.deterministic) { + console.warn("No GNU tar found; the archive will not be byte-reproducible. Install GNU tar (gtar) to fix this."); + } + await runCommand([tar.command, ...tar.createFlags, "-C", outputDir, "-czf", archive, path.basename(bundle)], { + description: "Bundle archive creation", + }); const digest = await sha256(archive); await Bun.write(checksum, `${digest} ${path.basename(archive)}\n`); if (!skipSmoke) { const extractedRoot = await fs.mkdtemp(path.join(outputDir, ".bundle-verify-")); try { - await runCommand(["tar", "-xzf", archive, "-C", extractedRoot], { + await runCommand([tar.command, "-xzf", archive, "-C", extractedRoot], { description: "Bundle archive extraction", }); - await verifyBundle(path.join(extractedRoot, BUNDLE_NAME)); + await verifyBundle(path.join(extractedRoot, target.bundleName), target); } finally { await fs.rm(extractedRoot, { recursive: true, force: true }); } @@ -330,29 +546,25 @@ async function archiveBundle( } async function main(): Promise { - if (process.platform !== "linux" || process.arch !== "x64") { - throw new Error( - `Relocatable embedded bundles currently require linux/x64; received ${process.platform}/${process.arch}.`, - ); - } + const target = hostBundleTarget(); const options = parseOptions(process.argv.slice(2)); - await validateRuntimeDistribution(options.runtimeDist); + await validateRuntimeDistribution(options.runtimeDist, target); const repoRoot = path.join(import.meta.dir, ".."); - const auraBinary = await resolveAuraBinary(options, repoRoot); + const auraBinary = await resolveAuraBinary(options, repoRoot, target); await fs.mkdir(options.outputDir, { recursive: true }); - const bundle = path.join(options.outputDir, BUNDLE_NAME); - const staging = path.join(options.outputDir, `.${BUNDLE_NAME}.staging-${crypto.randomUUID()}`); + const bundle = path.join(options.outputDir, target.bundleName); + const staging = path.join(options.outputDir, `.${target.bundleName}.staging-${crypto.randomUUID()}`); try { - await installBundle(options.runtimeDist, auraBinary, staging); - if (!options.skipSmoke) await verifyBundle(staging); + await installBundle(options.runtimeDist, auraBinary, staging, target); + if (!options.skipSmoke) await verifyBundle(staging, target); await replaceBundle(staging, bundle); } finally { await fs.rm(staging, { recursive: true, force: true }); } - const artifacts = options.archive ? await archiveBundle(options.outputDir, bundle, options.skipSmoke) : null; + const artifacts = options.archive ? await archiveBundle(options.outputDir, bundle, options.skipSmoke, target) : null; const bundleSize = await fs.stat(path.join(bundle, "bin", "aura.bin")); console.log(`Bundle: ${bundle}`); console.log(`Aura binary: ${bundleSize.size} bytes`);