From 3c5987ddeeae38f69516bb47e5f972c2a25f0e98 Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Tue, 24 Mar 2026 21:56:31 +0000 Subject: [PATCH] feat: eliminate deps/hyperlight-js git clone Replace the manual git clone of hyperlight-js with Cargo's built-in git dependency resolution. The runtime's Cargo.toml now uses a git dep on hyperlight-js-runtime, and the NAPI addon (js-host-api) is built from Cargo's checkout at ~/.cargo/git/checkouts/ with a symlink at deps/js-host-api for npm file: dependency resolution. Changes: - Cargo git dep for hyperlight-js-runtime (was path dep to clone) - HYPERLIGHT_CFLAGS resolved via cargo metadata + Node.js one-liner - Justfile: resolve-hyperlight-dir discovers checkout, builds NAPI addon, creates symlink (no more fetch-hyperlight/git clone) - package.json: file:deps/js-host-api (was file:deps/hyperlight-js/...) - Scripts/tests: all paths updated to deps/js-host-api - extract-hyperlight-builtins.ts: uses cargo metadata to find source - Removed: jq dependency, build-with-runtime alias, dead PWD variable - Docs: all clone references removed, setup descriptions updated - Docker/CI: COPY path and comments updated for symlink pattern Signed-off-by: Simon Davies --- .claude/CLAUDE.md | 2 +- .dockerignore | 2 - .github/copilot-instructions.md | 2 +- .github/workflows/pr-validate.yml | 10 +- .github/workflows/publish.yml | 8 + CLAUDE.md | 2 +- Dockerfile | 5 +- Justfile | 114 ++++--- README.md | 4 +- docs/DEVELOPMENT.md | 11 +- package-lock.json | 32 +- package.json | 2 +- scripts/build-binary.js | 4 +- scripts/extract-hyperlight-builtins.ts | 444 ++++++++++++++++++++----- src/sandbox/runtime/Cargo.toml | 2 +- tests/fetch-binary.test.ts | 2 +- tests/hyperlight-host-module.test.ts | 2 +- 17 files changed, 462 insertions(+), 186 deletions(-) diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 8449c8d..cb502e6 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -36,7 +36,7 @@ Use `just` commands for development (preferred) or npm scripts: ```bash # ── Development ── -just setup # First-time setup: clone deps, build native addons, npm install +just setup # First-time setup: build native addons, npm install just build # Rebuild native addons and install deps just start # Run agent (tsx src/agent/index.ts) diff --git a/.dockerignore b/.dockerignore index 14bd938..fbea698 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,8 +13,6 @@ tests/ .claude/ # Rust build artifacts (huge - 9GB+) -deps/hyperlight-js/target/ -src/hyperlight-analysis-guest/target/ **/target/ # KEEP dist/lib/node_modules (bundled @github/copilot SDK) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 6c665f9..b39ad77 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -29,7 +29,7 @@ Key config files at root: `package.json`, `tsconfig.json`, `vitest.config.ts`, ` Always use these commands in this order for a clean build: ```bash -just setup # First-time only: clone deps, build native addons, npm install +just setup # First-time only: build native addons, npm install just build # Rebuild native addons and install deps (run after Rust changes) ``` diff --git a/.github/workflows/pr-validate.yml b/.github/workflows/pr-validate.yml index 3b55b92..7d83ecc 100644 --- a/.github/workflows/pr-validate.yml +++ b/.github/workflows/pr-validate.yml @@ -146,7 +146,7 @@ jobs: path: dist/ retention-days: 7 - # Build Docker image (just setup clones deps so Dockerfile COPY works) + # Build Docker image (just setup builds deps + creates symlinks for Dockerfile COPY) build-docker: name: Build Docker Image needs: [docs-pr] @@ -168,6 +168,14 @@ jobs: - name: Setup run: just setup + - name: Resolve symlinks for Docker context + run: | + if [ -L deps/js-host-api ]; then + target=$(readlink -f deps/js-host-api) + rm deps/js-host-api + cp -r "$target" deps/js-host-api + fi + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a88f35d..8a6d008 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -118,6 +118,14 @@ jobs: - name: Setup run: just setup + - name: Resolve symlinks for Docker context + run: | + if [ -L deps/js-host-api ]; then + target=$(readlink -f deps/js-host-api) + rm deps/js-host-api + cp -r "$target" deps/js-host-api + fi + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/CLAUDE.md b/CLAUDE.md index 8449c8d..cb502e6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -36,7 +36,7 @@ Use `just` commands for development (preferred) or npm scripts: ```bash # ── Development ── -just setup # First-time setup: clone deps, build native addons, npm install +just setup # First-time setup: build native addons, npm install just build # Rebuild native addons and install deps just start # Run agent (tsx src/agent/index.ts) diff --git a/Dockerfile b/Dockerfile index b7170b0..0a3f05b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,8 +27,9 @@ WORKDIR /build # Copy package files first for layer caching COPY package*.json ./ -# Copy the hyperlight deps (for file: dependency) -COPY deps/hyperlight-js/src/js-host-api/ ./deps/hyperlight-js/src/js-host-api/ +# Copy the NAPI addon (must be a real directory, not a symlink — +# use `just docker-build` or resolve symlinks before `docker build`) +COPY deps/js-host-api/ ./deps/js-host-api/ COPY src/code-validator/guest/ ./src/code-validator/guest/ # Install dependencies diff --git a/Justfile b/Justfile index 6827da9..261d4ab 100644 --- a/Justfile +++ b/Justfile @@ -9,26 +9,22 @@ # - KVM support (for running the Hyperlight micro-VM) # # First-time setup: -# just setup # clones + builds hyperlight-js, installs npm deps +# just setup # builds native addons, installs npm deps # # ───────────────────────────────────────────────────────────────────── -# Windows: use PowerShell, replace backslashes for clang compatibility +# Windows: use PowerShell set windows-shell := ["pwsh.exe", "-NoLogo", "-Command"] -# In windows we need to replace the backslashes with forward slashes -# otherwise clang will misinterpret the paths -PWD := replace(justfile_dir(), "\\", "/") - # On Windows, use Ninja generator for CMake to avoid aws-lc-sys build issues export CMAKE_GENERATOR := if os() == "windows" { "Ninja" } else { "" } -# The hyperlight-js repo URL and ref to build against. -# Currently using Simon's fork with in-flight PRs (binary types, call stats). -# TODO: Switch to hyperlight-dev/hyperlight-js main once PRs land upstream. -hyperlight-repo := "https://github.com/simongdavies/hyperlight-js.git" -hyperlight-ref := "hyperagent" -hyperlight-dir := justfile_dir() / "deps" / "hyperlight-js" +# The hyperlight-js workspace root, discovered from Cargo's git checkout. +# The runtime's Cargo.toml uses a git dep on hyperlight-js-runtime, so Cargo +# already clones the full workspace — we reuse that checkout to build the +# NAPI addon (js-host-api) without a separate git clone. +# Resolved lazily by the resolve-hyperlight-dir recipe. +hyperlight-link := justfile_dir() / "deps" / "js-host-api" # Hyperlight analysis guest (secure code validation in micro-VM) analysis-guest-dir := justfile_dir() / "src" / "code-validator" / "guest" @@ -37,10 +33,11 @@ analysis-guest-dir := justfile_dir() / "src" / "code-validator" / "guest" runtime-dir := justfile_dir() / "src" / "sandbox" / "runtime" # HYPERLIGHT_CFLAGS needed for building guests that link rquickjs/QuickJS: -# -I include/ provides stubs for the hyperlight target (no libc) -# -D__wasi__=1 disables pthread support in QuickJS -# Uses forward slashes (PWD) so clang works on Windows -runtime-cflags := "-I" + PWD + "/deps/hyperlight-js/src/hyperlight-js-runtime/include -D__wasi__=1" +# The hyperlight target has no libc, so QuickJS needs stub headers plus +# -D__wasi__=1 to disable pthreads. Uses cargo metadata to find the +# include/ dir from the hyperlight-js-runtime dependency. +# Fails loudly if resolution fails — empty CFLAGS causes cryptic build errors. +runtime-cflags := `node -e "var m=JSON.parse(require('child_process').execSync('cargo +1.89 metadata --format-version 1 --manifest-path src/sandbox/runtime/Cargo.toml',{encoding:'utf8',stdio:['pipe','pipe','inherit'],maxBuffer:20*1024*1024}));var p=m.packages.find(function(p){return p.name==='hyperlight-js-runtime'});if(!p){process.stderr.write('ERROR: hyperlight-js-runtime not found in cargo metadata\n');process.exit(1)}console.log('-I'+require('path').join(require('path').dirname(p.manifest_path),'include')+' -D__wasi__=1')"` # Export HYPERLIGHT_CFLAGS so cargo-hyperlight picks them up when building runtimes export HYPERLIGHT_CFLAGS := runtime-cflags @@ -50,12 +47,29 @@ export HYPERLIGHT_CFLAGS := runtime-cflags # Without this, the default runtime (no ha:ziplib) would be embedded. export HYPERLIGHT_JS_RUNTIME_PATH := runtime-dir / "target" / "x86_64-hyperlight-none" / "release" / "hyperagent-runtime" -# Clone (or update) the hyperlight-js dependency at the pinned ref. -# Cross-platform: - prefix ignores clone failure (dir already exists). +# Resolve the hyperlight-js workspace root from Cargo's git checkout. +# Uses cargo metadata to find where hyperlight-js-runtime lives, then +# derives the workspace src/ dir (js-host-api is a sibling crate). +# Outputs the workspace root path (parent of src/). [private] -fetch-hyperlight: - -git clone --branch "{{hyperlight-ref}}" --single-branch --depth 1 "{{hyperlight-repo}}" "{{hyperlight-dir}}" - cd "{{hyperlight-dir}}" && git fetch origin "{{hyperlight-ref}}" --depth 1 && git checkout FETCH_HEAD +[unix] +resolve-hyperlight-dir: + #!/usr/bin/env bash + set -euo pipefail + dir=$(node -e "\ + var m=JSON.parse(require('child_process').execSync(\ + 'cargo +1.89 metadata --format-version 1 --manifest-path src/sandbox/runtime/Cargo.toml',\ + {encoding:'utf8',stdio:['pipe','pipe','pipe'],maxBuffer:20*1024*1024}));\ + var p=m.packages.find(function(p){return p.name==='hyperlight-js-runtime'});\ + if(p)console.log(require('path').resolve(require('path').dirname(p.manifest_path),'..','..'));\ + else{process.stderr.write('hyperlight-js-runtime not found in cargo metadata');process.exit(1)}") + js_host_api="${dir}/src/js-host-api" + if [ ! -d "$js_host_api" ]; then + echo "❌ js-host-api not found at ${js_host_api}" + echo " Run: cargo +1.89 fetch --manifest-path src/sandbox/runtime/Cargo.toml" + exit 1 + fi + echo "$dir" # Install required Rust toolchains and cargo subcommands. # Cross-platform (Linux/macOS/Windows) — no bash required. @@ -65,19 +79,27 @@ ensure-tools: rustup toolchain install 1.89 --no-self-update rustup toolchain install nightly --no-self-update -# Build the native hyperlight-js NAPI addon (debug — default) -# Depends on build-runtime-release so the custom runtime with native modules -# is always embedded. cargo clean -p prevents stale cached builds. -[private] -build-hyperlight: fetch-hyperlight (build-runtime-release) - -cd "{{hyperlight-dir}}/src/hyperlight-js" && cargo clean -p hyperlight-js - cd "{{hyperlight-dir}}" && just build - -# Build the native hyperlight-js NAPI addon (release — optimised) +# Build the native hyperlight-js NAPI addon. +# 1. Builds the custom runtime (Cargo git dep fetches hyperlight-js automatically) +# 2. Discovers the hyperlight-js workspace from Cargo's checkout +# 3. Builds the NAPI addon with our custom runtime embedded +# 4. Symlinks deps/js-host-api → checkout/src/js-host-api for npm file: dep +# NOTE: [unix] only — Windows support is disabled pending upstream fix (issue #1). +# When Windows lands, add [windows] variants using mklink /J for junctions. [private] -build-hyperlight-release: fetch-hyperlight (build-runtime-release) - -cd "{{hyperlight-dir}}/src/hyperlight-js" && cargo clean -p hyperlight-js - cd "{{hyperlight-dir}}" && just build release +[unix] +build-hyperlight target="debug": (build-runtime-release) + #!/usr/bin/env bash + set -euo pipefail + hl_dir=$(just resolve-hyperlight-dir) + # Clean stale hyperlight-js builds so build.rs re-embeds the runtime + cd "${hl_dir}/src/hyperlight-js" && cargo clean -p hyperlight-js 2>/dev/null || true + # Build the NAPI addon (inherits HYPERLIGHT_JS_RUNTIME_PATH from env) + cd "${hl_dir}" && just build {{ if target == "debug" { "" } else { target } }} + # Symlink for npm file: dependency resolution + mkdir -p "{{justfile_dir()}}/deps" + ln -sfn "${hl_dir}/src/js-host-api" "{{hyperlight-link}}" + echo "🔗 deps/js-host-api → ${hl_dir}/src/js-host-api" # Build the hyperlight-analysis-guest NAPI addon (debug) [private] @@ -89,19 +111,19 @@ build-analysis-guest: build-analysis-guest-release: cd "{{analysis-guest-dir}}" && just build release && just build-napi release -# Install npm deps (links hyperlight-js and analysis-guest from local dirs) +# Install npm deps (builds native addons, symlinks js-host-api) [private] -install: build-hyperlight build-analysis-guest +install: (build-hyperlight) build-analysis-guest npm install # Install npm deps with release-built native addons [private] -install-release: build-hyperlight-release build-analysis-guest-release +install-release: (build-hyperlight "release") build-analysis-guest-release npm install # ── First-time setup ───────────────────────────────────────────────── -# Clone hyperlight-js, build native addon, install npm deps +# First-time setup: build native addons, install npm deps setup: ensure-tools install @echo "✅ Setup complete — run 'just start' to launch the agent" @@ -187,18 +209,13 @@ test-analysis-guest: # ── HyperAgent Runtime (native modules) ────────────────────────────── # Build the custom runtime for the hyperlight target (debug) -build-runtime: fetch-hyperlight +build-runtime: cd "{{runtime-dir}}" && cargo +1.89 hyperlight build --target-dir target # Build the custom runtime for the hyperlight target (release) -build-runtime-release: fetch-hyperlight +build-runtime-release: cd "{{runtime-dir}}" && cargo +1.89 hyperlight build --target-dir target --release -# Legacy alias — the standard build now always uses the custom runtime. -# Kept for backwards compatibility with existing scripts/docs. -build-with-runtime: build - @echo "✅ (build-with-runtime is now a no-op — 'just build' always uses the custom runtime)" - # Lint Rust code in the custom runtime lint-runtime: cd "{{runtime-dir}}" && cargo +1.89 clippy --workspace -- -D warnings @@ -238,7 +255,7 @@ check: lint-all test-all clean: rm -rf dist node_modules -# Clean everything including the hyperlight-js clone +# Clean everything including deps/ symlinks clean-all: clean rm -rf deps @@ -271,6 +288,13 @@ docker-build: version="0.0.0-dev" fi echo "📦 Docker build version: ${version}" + # Dereference symlinks — Docker COPY can't follow symlinks outside the build context + if [ -L deps/js-host-api ]; then + target=$(readlink -f deps/js-host-api) + rm deps/js-host-api + cp -r "$target" deps/js-host-api + trap 'rm -rf deps/js-host-api && ln -sfn "'"$target"'" deps/js-host-api' EXIT + fi docker build -t hyperagent --build-arg VERSION="${version}" . # Run hyperagent in Docker (requires /dev/kvm or /dev/mshv) diff --git a/README.md b/README.md index 0dceb9c..e4a4238 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ Requires Node.js 22+, Rust toolchain, and [just](https://github.com/casey/just) git clone https://github.com/hyperlight-dev/hyperagent.git cd hyperagent -# First-time setup — clones deps, builds native Rust addons, installs npm packages +# First-time setup — builds native Rust addons, installs npm packages just setup # Run the agent (tsx transpiles on the fly — no build step needed) @@ -140,7 +140,7 @@ Key `just` commands: | Command | What it does | |---------|-------------| -| `just setup` | First-time setup (clone deps, build native addons, npm install) | +| `just setup` | First-time setup (build native addons, npm install) | | `just build` | Rebuild native addons after Rust changes | | `just start` | Run agent with tsx (fast iteration) | | `just binary-release` | Build optimised standalone binary to `dist/bin/hyperagent` | diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 6f42b18..edc93b8 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -12,20 +12,17 @@ For contribution guidelines (issues, PRs, DCO), see [CONTRIBUTING.md](../CONTRIB - **just** command runner (`cargo install just`) - **GitHub CLI** (`gh`) authenticated -## Clone and Build +## Build ```bash git clone https://github.com/hyperlight-dev/hyperagent cd hyperagent -# Install Node dependencies -npm install - -# Build native Hyperlight components -just build +# First-time setup — builds native addons, installs npm deps +just setup # Verify setup -npm test +just test ``` ## Running from Source diff --git a/package-lock.json b/package-lock.json index f9410b3..035a0a5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "hasInstallScript": true, "dependencies": { "@github/copilot-sdk": "^0.1.32", - "@hyperlight/js-host-api": "file:deps/hyperlight-js/src/js-host-api", + "@hyperlight/js-host-api": "file:deps/js-host-api", "boxen": "^8.0.1", "hyperlight-analysis": "file:src/code-validator/guest", "zod": "^3.25.0" @@ -27,20 +27,7 @@ "vitest": "^4.0.18" } }, - "deps/hyperlight-analysis-guest": { - "name": "hyperlight-analysis", - "version": "0.1.0", - "extraneous": true, - "license": "Apache-2.0", - "devDependencies": { - "@napi-rs/cli": "^3.0.0-alpha.63", - "vitest": "^3.0.0" - }, - "engines": { - "node": ">= 18" - } - }, - "deps/hyperlight-js/src/js-host-api": { + "deps/js-host-api": { "name": "@hyperlight/js-host-api", "version": "0.17.0", "license": "Apache-2.0", @@ -857,7 +844,7 @@ } }, "node_modules/@hyperlight/js-host-api": { - "resolved": "deps/hyperlight-js/src/js-host-api", + "resolved": "deps/js-host-api", "link": true }, "node_modules/@inquirer/ansi": { @@ -4983,19 +4970,6 @@ "optional": true } } - }, - "src/hyperlight-analysis-guest": { - "name": "hyperlight-analysis", - "version": "0.1.0", - "extraneous": true, - "license": "Apache-2.0", - "devDependencies": { - "@napi-rs/cli": "^3.0.0-alpha.63", - "vitest": "^3.0.0" - }, - "engines": { - "node": ">= 18" - } } } } diff --git a/package.json b/package.json index ee4089d..5e6fb25 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ }, "dependencies": { "@github/copilot-sdk": "^0.1.32", - "@hyperlight/js-host-api": "file:deps/hyperlight-js/src/js-host-api", + "@hyperlight/js-host-api": "file:deps/js-host-api", "boxen": "^8.0.1", "hyperlight-analysis": "file:src/code-validator/guest", "zod": "^3.25.0" diff --git a/scripts/build-binary.js b/scripts/build-binary.js index f59c897..203d973 100644 --- a/scripts/build-binary.js +++ b/scripts/build-binary.js @@ -179,7 +179,7 @@ if (!napiTriple) { } console.log(` Platform: ${platformKey} → ${napiTriple}`); -const hyperlightNode = join(ROOT, `deps/hyperlight-js/src/js-host-api/js-host-api.${napiTriple}.node`); +const hyperlightNode = join(ROOT, `deps/js-host-api/js-host-api.${napiTriple}.node`); const analysisNode = join(ROOT, `src/code-validator/guest/host/hyperlight-analysis.${napiTriple}.node`); if (!existsSync(hyperlightNode)) { @@ -214,7 +214,7 @@ if (existsSync(analysisPkg)) copyFileSync(analysisPkg, join(analysisPkgDir, "pac // failures ("InvalidArg, Call the PromiseRaw::then failed"). // Files are renamed to .cjs because the host package.json has "type": "module" // which makes Node.js treat .js as ESM — but lib.js uses require(). -const hyperlightLibJs = join(ROOT, "deps/hyperlight-js/src/js-host-api/lib.js"); +const hyperlightLibJs = join(ROOT, "deps/js-host-api/lib.js"); const hyperlightHostApiDir = join(LIB_DIR, "js-host-api"); mkdirSync(hyperlightHostApiDir, { recursive: true }); copyFileSync(hyperlightNode, join(hyperlightHostApiDir, `js-host-api.${napiTriple}.node`)); diff --git a/scripts/extract-hyperlight-builtins.ts b/scripts/extract-hyperlight-builtins.ts index 5dd104e..c4d5b84 100644 --- a/scripts/extract-hyperlight-builtins.ts +++ b/scripts/extract-hyperlight-builtins.ts @@ -9,10 +9,35 @@ * Run with: npx tsx scripts/extract-hyperlight-builtins.ts */ -import { readFileSync, readdirSync } from 'fs'; -import { join } from 'path'; +import { readFileSync, readdirSync } from "fs"; +import { join, dirname } from "path"; +import { execSync } from "child_process"; -const RUNTIME_SRC = join(import.meta.dirname, '..', 'deps/hyperlight-js/src/hyperlight-js-runtime/src'); +// Resolve hyperlight-js-runtime source from Cargo's git checkout via cargo metadata. +// The runtime crate is a dependency of our custom runtime, so cargo metadata knows where it is. +function resolveRuntimeSrc(): string { + const metadata = JSON.parse( + execSync( + "cargo +1.89 metadata --format-version 1 --manifest-path src/sandbox/runtime/Cargo.toml", + { + encoding: "utf8", + stdio: ["pipe", "pipe", "pipe"], + maxBuffer: 20 * 1024 * 1024, + cwd: join(import.meta.dirname, ".."), + }, + ), + ); + const pkg = metadata.packages.find( + (p: { name: string }) => p.name === "hyperlight-js-runtime", + ); + if (!pkg) { + console.error("❌ hyperlight-js-runtime not found in cargo metadata"); + process.exit(1); + } + return join(dirname(pkg.manifest_path), "src"); +} + +const RUNTIME_SRC = resolveRuntimeSrc(); interface Builtins { globals: string[]; @@ -27,14 +52,15 @@ function extractFunctions(source: string): string[] { let match; while ((match = funcRegex.exec(source)) !== null) { const name = match[1]; - if (name !== 'default') funcs.push(name); + if (name !== "default") funcs.push(name); } return funcs; } function extractMethods(source: string): string[] { // Match pub fn name(...) inside #[rquickjs::methods] impl blocks - const methodsBlockRegex = /#\[rquickjs::methods\]\s*impl\s+\w+\s*\{([\s\S]*?)\n\}/g; + const methodsBlockRegex = + /#\[rquickjs::methods\]\s*impl\s+\w+\s*\{([\s\S]*?)\n\}/g; const methodRegex = /pub\s+fn\s+(\w+)/g; const methods: string[] = []; let blockMatch; @@ -43,7 +69,7 @@ function extractMethods(source: string): string[] { let methodMatch; while ((methodMatch = methodRegex.exec(block)) !== null) { const name = methodMatch[1]; - if (name !== 'new') methods.push(name); + if (name !== "new") methods.push(name); } } return methods; @@ -51,7 +77,8 @@ function extractMethods(source: string): string[] { function extractClasses(source: string): string[] { // Match #[rquickjs::class()] pub struct Name - const classRegex = /#\[rquickjs::class(?:\([^)]*\))?\]\s*(?:#\[[^\]]*\]\s*)*pub\s+struct\s+(\w+)/g; + const classRegex = + /#\[rquickjs::class(?:\([^)]*\))?\]\s*(?:#\[[^\]]*\]\s*)*pub\s+struct\s+(\w+)/g; const classes: string[] = []; let match; while ((match = classRegex.exec(source)) !== null) { @@ -67,12 +94,12 @@ const builtins: Builtins = { }; // Parse modules directory -const modulesDir = join(RUNTIME_SRC, 'modules'); +const modulesDir = join(RUNTIME_SRC, "modules"); for (const file of readdirSync(modulesDir)) { - if (file === 'mod.rs' || !file.endsWith('.rs')) continue; + if (file === "mod.rs" || !file.endsWith(".rs")) continue; - const moduleName = file.replace('.rs', ''); - const source = readFileSync(join(modulesDir, file), 'utf-8'); + const moduleName = file.replace(".rs", ""); + const source = readFileSync(join(modulesDir, file), "utf-8"); const funcs = extractFunctions(source); const methods = extractMethods(source); @@ -89,27 +116,27 @@ for (const file of readdirSync(modulesDir)) { } // Parse globals directory -const globalsDir = join(RUNTIME_SRC, 'globals'); +const globalsDir = join(RUNTIME_SRC, "globals"); for (const file of readdirSync(globalsDir)) { - if (file === 'mod.rs' || !file.endsWith('.rs')) continue; + if (file === "mod.rs" || !file.endsWith(".rs")) continue; - const source = readFileSync(join(globalsDir, file), 'utf-8'); + const source = readFileSync(join(globalsDir, file), "utf-8"); // Check what's added to globals if (source.includes('globals.prop("print"')) { - builtins.globals.push('print'); + builtins.globals.push("print"); } if (source.includes('globals.prop("console"')) { - builtins.globals.push('console'); + builtins.globals.push("console"); // Only log is available - builtins.globalMethods['console'] = ['log']; + builtins.globalMethods["console"] = ["log"]; } if (source.includes('globals.prop("require"')) { - builtins.globals.push('require'); + builtins.globals.push("require"); } if (source.includes('string.prop("bytesFrom"')) { - builtins.globalMethods['String'] = builtins.globalMethods['String'] || []; - builtins.globalMethods['String'].push('bytesFrom'); + builtins.globalMethods["String"] = builtins.globalMethods["String"] || []; + builtins.globalMethods["String"].push("bytesFrom"); } } @@ -117,67 +144,291 @@ for (const file of readdirSync(globalsDir)) { const quickjsBuiltins = { globals: [ // Constructors - 'Array', 'Object', 'String', 'Number', 'Boolean', 'Symbol', 'BigInt', - 'Function', 'Error', 'TypeError', 'RangeError', 'SyntaxError', 'ReferenceError', - 'Date', 'RegExp', 'Promise', 'Map', 'Set', 'WeakMap', 'WeakSet', - 'Uint8Array', 'Int8Array', 'Uint16Array', 'Int16Array', 'Uint32Array', 'Int32Array', - 'Float32Array', 'Float64Array', 'BigInt64Array', 'BigUint64Array', - 'ArrayBuffer', 'DataView', + "Array", + "Object", + "String", + "Number", + "Boolean", + "Symbol", + "BigInt", + "Function", + "Error", + "TypeError", + "RangeError", + "SyntaxError", + "ReferenceError", + "Date", + "RegExp", + "Promise", + "Map", + "Set", + "WeakMap", + "WeakSet", + "Uint8Array", + "Int8Array", + "Uint16Array", + "Int16Array", + "Uint32Array", + "Int32Array", + "Float32Array", + "Float64Array", + "BigInt64Array", + "BigUint64Array", + "ArrayBuffer", + "DataView", // Static objects - 'Math', 'JSON', 'Reflect', 'Proxy', + "Math", + "JSON", + "Reflect", + "Proxy", // Global functions - 'parseInt', 'parseFloat', 'isNaN', 'isFinite', 'decodeURI', 'encodeURI', - 'decodeURIComponent', 'encodeURIComponent', 'eval', + "parseInt", + "parseFloat", + "isNaN", + "isFinite", + "decodeURI", + "encodeURI", + "decodeURIComponent", + "encodeURIComponent", + "eval", ], methods: { // Array methods - 'Array': ['push', 'pop', 'shift', 'unshift', 'slice', 'splice', 'concat', 'join', - 'map', 'filter', 'reduce', 'reduceRight', 'forEach', 'find', 'findIndex', - 'includes', 'indexOf', 'lastIndexOf', 'every', 'some', 'flat', 'flatMap', - 'fill', 'sort', 'reverse', 'at', 'entries', 'keys', 'values', 'copyWithin', - 'toSorted', 'toReversed', 'toSpliced', 'with'], + Array: [ + "push", + "pop", + "shift", + "unshift", + "slice", + "splice", + "concat", + "join", + "map", + "filter", + "reduce", + "reduceRight", + "forEach", + "find", + "findIndex", + "includes", + "indexOf", + "lastIndexOf", + "every", + "some", + "flat", + "flatMap", + "fill", + "sort", + "reverse", + "at", + "entries", + "keys", + "values", + "copyWithin", + "toSorted", + "toReversed", + "toSpliced", + "with", + ], // String methods - 'String': ['split', 'trim', 'trimStart', 'trimEnd', 'toLowerCase', 'toUpperCase', - 'substring', 'substr', 'slice', 'replace', 'replaceAll', 'match', 'matchAll', - 'search', 'charAt', 'charCodeAt', 'codePointAt', 'startsWith', 'endsWith', - 'padStart', 'padEnd', 'repeat', 'normalize', 'localeCompare', 'includes', - 'indexOf', 'lastIndexOf', 'at', 'concat'], + String: [ + "split", + "trim", + "trimStart", + "trimEnd", + "toLowerCase", + "toUpperCase", + "substring", + "substr", + "slice", + "replace", + "replaceAll", + "match", + "matchAll", + "search", + "charAt", + "charCodeAt", + "codePointAt", + "startsWith", + "endsWith", + "padStart", + "padEnd", + "repeat", + "normalize", + "localeCompare", + "includes", + "indexOf", + "lastIndexOf", + "at", + "concat", + ], // Object methods - 'Object': ['hasOwnProperty', 'toString', 'valueOf', 'toJSON', 'toLocaleString', - 'keys', 'values', 'entries', 'assign', 'freeze', 'seal', 'create', - 'defineProperty', 'defineProperties', 'getOwnPropertyDescriptor', - 'getOwnPropertyNames', 'getOwnPropertySymbols', 'getPrototypeOf', - 'setPrototypeOf', 'is', 'fromEntries'], + Object: [ + "hasOwnProperty", + "toString", + "valueOf", + "toJSON", + "toLocaleString", + "keys", + "values", + "entries", + "assign", + "freeze", + "seal", + "create", + "defineProperty", + "defineProperties", + "getOwnPropertyDescriptor", + "getOwnPropertyNames", + "getOwnPropertySymbols", + "getPrototypeOf", + "setPrototypeOf", + "is", + "fromEntries", + ], // Promise methods - 'Promise': ['then', 'catch', 'finally', 'all', 'allSettled', 'any', 'race', 'resolve', 'reject'], + Promise: [ + "then", + "catch", + "finally", + "all", + "allSettled", + "any", + "race", + "resolve", + "reject", + ], // JSON methods - 'JSON': ['parse', 'stringify'], + JSON: ["parse", "stringify"], // Math methods - 'Math': ['abs', 'ceil', 'floor', 'round', 'max', 'min', 'random', 'sqrt', 'pow', - 'sign', 'trunc', 'log', 'log10', 'log2', 'exp', 'sin', 'cos', 'tan', - 'asin', 'acos', 'atan', 'atan2', 'sinh', 'cosh', 'tanh', 'hypot', 'cbrt'], + Math: [ + "abs", + "ceil", + "floor", + "round", + "max", + "min", + "random", + "sqrt", + "pow", + "sign", + "trunc", + "log", + "log10", + "log2", + "exp", + "sin", + "cos", + "tan", + "asin", + "acos", + "atan", + "atan2", + "sinh", + "cosh", + "tanh", + "hypot", + "cbrt", + ], // Map/Set methods - 'Map': ['get', 'set', 'has', 'delete', 'clear', 'keys', 'values', 'entries', 'forEach'], - 'Set': ['add', 'has', 'delete', 'clear', 'keys', 'values', 'entries', 'forEach'], + Map: [ + "get", + "set", + "has", + "delete", + "clear", + "keys", + "values", + "entries", + "forEach", + ], + Set: [ + "add", + "has", + "delete", + "clear", + "keys", + "values", + "entries", + "forEach", + ], // Date methods - 'Date': ['getTime', 'getFullYear', 'getMonth', 'getDate', 'getDay', 'getHours', - 'getMinutes', 'getSeconds', 'getMilliseconds', 'setTime', 'setFullYear', - 'setMonth', 'setDate', 'setHours', 'setMinutes', 'setSeconds', 'setMilliseconds', - 'toISOString', 'toJSON', 'toDateString', 'toTimeString', 'toLocaleString'], + Date: [ + "getTime", + "getFullYear", + "getMonth", + "getDate", + "getDay", + "getHours", + "getMinutes", + "getSeconds", + "getMilliseconds", + "setTime", + "setFullYear", + "setMonth", + "setDate", + "setHours", + "setMinutes", + "setSeconds", + "setMilliseconds", + "toISOString", + "toJSON", + "toDateString", + "toTimeString", + "toLocaleString", + ], // RegExp methods - 'RegExp': ['test', 'exec', 'toString'], + RegExp: ["test", "exec", "toString"], // TypedArray methods (same for all typed arrays) - 'TypedArray': ['set', 'subarray', 'slice', 'fill', 'copyWithin', 'find', 'findIndex', - 'indexOf', 'lastIndexOf', 'includes', 'every', 'some', 'filter', 'map', - 'reduce', 'reduceRight', 'forEach', 'join', 'reverse', 'sort', 'at'], + TypedArray: [ + "set", + "subarray", + "slice", + "fill", + "copyWithin", + "find", + "findIndex", + "indexOf", + "lastIndexOf", + "includes", + "every", + "some", + "filter", + "map", + "reduce", + "reduceRight", + "forEach", + "join", + "reverse", + "sort", + "at", + ], // ArrayBuffer methods - 'ArrayBuffer': ['slice'], + ArrayBuffer: ["slice"], // DataView methods - 'DataView': ['getInt8', 'getUint8', 'getInt16', 'getUint16', 'getInt32', 'getUint32', - 'getFloat32', 'getFloat64', 'getBigInt64', 'getBigUint64', - 'setInt8', 'setUint8', 'setInt16', 'setUint16', 'setInt32', 'setUint32', - 'setFloat32', 'setFloat64', 'setBigInt64', 'setBigUint64'], - } + DataView: [ + "getInt8", + "getUint8", + "getInt16", + "getUint16", + "getInt32", + "getUint32", + "getFloat32", + "getFloat64", + "getBigInt64", + "getBigUint64", + "setInt8", + "setUint8", + "setInt16", + "setUint16", + "setInt32", + "setUint32", + "setFloat32", + "setFloat64", + "setBigInt64", + "setBigUint64", + ], + }, }; // Merge hyperlight-specific builtins with QuickJS builtins @@ -195,49 +446,64 @@ for (const methods of Object.values(builtins.globalMethods)) { } // Output -console.log('=== HYPERLIGHT-JS SANDBOX BUILTINS ===\n'); +console.log("=== HYPERLIGHT-JS SANDBOX BUILTINS ===\n"); -console.log('// Available globals (BUILTIN_FUNCTIONS):'); -console.log('const BUILTIN_FUNCTIONS: &[&str] = &['); +console.log("// Available globals (BUILTIN_FUNCTIONS):"); +console.log("const BUILTIN_FUNCTIONS: &[&str] = &["); for (const g of [...allGlobals].sort()) { console.log(` "${g}",`); } -console.log('];\n'); +console.log("];\n"); -console.log('// Available methods (BUILTIN_METHODS):'); -console.log('const BUILTIN_METHODS: &[&str] = &['); +console.log("// Available methods (BUILTIN_METHODS):"); +console.log("const BUILTIN_METHODS: &[&str] = &["); for (const m of [...allMethods].sort()) { console.log(` "${m}",`); } -console.log('];\n'); +console.log("];\n"); -console.log('=== NOT AVAILABLE (do not assume these exist) ==='); -console.log('// These are commonly assumed but NOT in hyperlight-js:'); +console.log("=== NOT AVAILABLE (do not assume these exist) ==="); +console.log("// These are commonly assumed but NOT in hyperlight-js:"); const notAvailable = [ - 'setTimeout', 'setInterval', 'clearTimeout', 'clearInterval', - 'fetch', 'Request', 'Response', 'Headers', - 'console.warn', 'console.error', 'console.info', 'console.debug', 'console.trace', 'console.table', - 'atob', 'btoa', - 'TextEncoder', 'TextDecoder', - 'URL', 'URLSearchParams', - 'queueMicrotask', - 'structuredClone', - 'crypto.getRandomValues', 'crypto.randomUUID', + "setTimeout", + "setInterval", + "clearTimeout", + "clearInterval", + "fetch", + "Request", + "Response", + "Headers", + "console.warn", + "console.error", + "console.info", + "console.debug", + "console.trace", + "console.table", + "atob", + "btoa", + "TextEncoder", + "TextDecoder", + "URL", + "URLSearchParams", + "queueMicrotask", + "structuredClone", + "crypto.getRandomValues", + "crypto.randomUUID", ]; for (const na of notAvailable) { console.log(`// - ${na}`); } -console.log('\n=== HYPERLIGHT-SPECIFIC ==='); -console.log('// Custom globals added by hyperlight-js:'); +console.log("\n=== HYPERLIGHT-SPECIFIC ==="); +console.log("// Custom globals added by hyperlight-js:"); for (const g of builtins.globals) { console.log(`// - ${g}`); } -console.log('// Custom modules:'); +console.log("// Custom modules:"); for (const [mod, funcs] of Object.entries(builtins.modules)) { - console.log(`// - ${mod}: ${funcs.join(', ')}`); + console.log(`// - ${mod}: ${funcs.join(", ")}`); } -console.log('// Custom methods:'); +console.log("// Custom methods:"); for (const [obj, methods] of Object.entries(builtins.globalMethods)) { - console.log(`// - ${obj}.${methods.join(', ')}`); + console.log(`// - ${obj}.${methods.join(", ")}`); } diff --git a/src/sandbox/runtime/Cargo.toml b/src/sandbox/runtime/Cargo.toml index 374e439..ab3eace 100644 --- a/src/sandbox/runtime/Cargo.toml +++ b/src/sandbox/runtime/Cargo.toml @@ -13,7 +13,7 @@ name = "hyperagent-runtime" path = "src/main.rs" [dependencies] -hyperlight-js-runtime = { path = "../../../deps/hyperlight-js/src/hyperlight-js-runtime" } +hyperlight-js-runtime = { git = "https://github.com/simongdavies/hyperlight-js.git", branch = "hyperagent" } native-deflate = { path = "modules/native-deflate" } native-image = { path = "modules/native-image" } native-html = { path = "modules/native-html" } diff --git a/tests/fetch-binary.test.ts b/tests/fetch-binary.test.ts index b8b5f17..777c14a 100644 --- a/tests/fetch-binary.test.ts +++ b/tests/fetch-binary.test.ts @@ -15,7 +15,7 @@ import { createServer, type Server } from "node:http"; import { type AddressInfo } from "node:net"; // Import hyperlight-js directly -import { SandboxBuilder } from "../deps/hyperlight-js/src/js-host-api/lib.js"; +import { SandboxBuilder } from "../deps/js-host-api/lib.js"; // Import the fetch plugin createHostFunctions function import { createHostFunctions } from "../plugins/fetch/index.js"; diff --git a/tests/hyperlight-host-module.test.ts b/tests/hyperlight-host-module.test.ts index 184c85c..5f00fc8 100644 --- a/tests/hyperlight-host-module.test.ts +++ b/tests/hyperlight-host-module.test.ts @@ -16,7 +16,7 @@ import { join, dirname } from "node:path"; import { fileURLToPath } from "node:url"; // Import hyperlight-js directly -import { SandboxBuilder } from "../deps/hyperlight-js/src/js-host-api/lib.js"; +import { SandboxBuilder } from "../deps/js-host-api/lib.js"; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename);