diff --git a/.gitignore b/.gitignore index d99042ec..f43331fe 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,3 @@ npm/ # deno task build:web output — built into the package at release time, # never a repository source file (specs/release-process-spec.md) packages/web/generated/ - -# spike #349 working artifacts — vendored build output and databases stay out -spikes/349-dofs/vendor-build/ diff --git a/deno.json b/deno.json index d7f4a9ae..926a8eb4 100644 --- a/deno.json +++ b/deno.json @@ -1,12 +1,6 @@ { - "workspace": [ - "packages/*", - "site" - ], - "exclude": [ - "scripts/tests/fixtures", - "spikes" - ], + "workspace": ["packages/*", "site"], + "exclude": ["scripts/tests/fixtures"], "nodeModulesDir": "auto", "lock": { "frozen": true @@ -63,12 +57,6 @@ "review:local": "deno run --allow-all packages/cli/src/deno.ts run .reviews/ReviewPR.local.md --component-dir .reviews/components --component-dir .reviews/policies --component-dir packages/core/components -j .reviews/journal.local.jsonl", "analyze": "deno run --allow-all packages/cli/src/deno.ts run .reviews/AnalyzeRepo.md --component-dir .reviews/components --component-dir .reviews/policies --component-dir packages/core/components -j .reviews/journal.analyze.jsonl", "analyze:ci": "deno run --allow-all packages/cli/src/deno.ts run .reviews/AnalyzeRepoCI.md --component-dir .reviews/components --component-dir .reviews/policies --component-dir packages/core/components -j .reviews/journal.analyze.ci.jsonl", - "analyze:dispatch": "deno run --allow-all packages/cli/src/deno.ts run .reviews/DispatchRepoAnalysis.md --component-dir .reviews/components --component-dir .reviews/policies --component-dir packages/core/components -j .reviews/journal.dispatch.jsonl", - "spike:349": "deno task --cwd spikes/349-dofs all", - "spike:349:build": "deno task --cwd spikes/349-dofs build", - "spike:349:test": "deno task --cwd spikes/349-dofs test", - "spike:351": "deno task --cwd spikes/351-worker-backends all", - "spike:351:build": "deno task --cwd spikes/351-worker-backends build", - "spike:351:test": "deno task --cwd spikes/351-worker-backends test" + "analyze:dispatch": "deno run --allow-all packages/cli/src/deno.ts run .reviews/DispatchRepoAnalysis.md --component-dir .reviews/components --component-dir .reviews/policies --component-dir packages/core/components -j .reviews/journal.dispatch.jsonl" } -} \ No newline at end of file +} diff --git a/spikes/349-dofs/README.md b/spikes/349-dofs/README.md deleted file mode 100644 index a2c56214..00000000 --- a/spikes/349-dofs/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# Spike 349: Cloudflare DOFS directly in Deno - -Proves that Executable.md can host a persistent SQLite Workspace directly in -the Deno process by reusing Cloudflare Computer's DOFS filesystem layer over -a file-backed `node:sqlite` adapter — no `workerd`, no Node, no Wrangler. -Compares this topology with the bundled-`workerd` evidence in #347 / PR #348 -for the decision in #346. Findings live in -[evidence/EVIDENCE.md](evidence/EVIDENCE.md). - -## Run it - -From the repository root: - -```bash -deno task spike:349 # vendor-build + compile + test -``` - -Stepwise: `deno task --cwd spikes/349-dofs vendor` installs the vendored -package's build toolchain and compiles it with `tsc`; -`deno task spike:349:build` compiles `dist/proof`; -`deno task spike:349:test` runs the scenario suite. One prerequisite the -tasks assume: `npm install --install-links --no-audit --no-fund` in this -directory (dependencies are npm-owned here — see Layout). - -The proof executable performs one filesystem op per invocation against a -workspace database file, so consecutive invocations are full restart cycles: - -```bash -dist/proof /tmp/ws.db write /notes/a.md "alpha" -dist/proof /tmp/ws.db read /notes/a.md -dist/proof /tmp/ws.db ls /notes -``` - -## Layout - -- `vendor/dofs/` — pinned vendored copy of `packages/dofs` from - cloudflare/computer `v0.1.1` (`63d3636`), MIT; provenance, the two manifest - edits, and the upgrade procedure are in - [vendor/dofs/PROVENANCE.md](vendor/dofs/PROVENANCE.md). Source files are - unmodified. -- `host/file-storage.ts` — the entire adapter: a file-backed - `DurableObjectStorageLike` over Deno's `node:sqlite` (~100 lines, - mirroring upstream's own in-memory test fixture). -- `host/main.ts` — the proof CLI; `host/types/` — typed facade for the - consumed surface (Deno pairs `.js` with `.d.ts` for registry packages but - not `file:`-resolved ones). -- `tests/spike.test.ts` — the scenario suite backing the evidence. -- `evidence/probes/` — raw probe ledgers with every command and error. - -This spike uses npm-owned dependencies (`nodeModulesDir: "manual"`, -`package.json` + `package-lock.json`) because the vendored package is -consumed as a `file:` dependency through its exports map — itself part of -the reuse-boundary evidence. The directory stays outside the workspace via -the root `deno.json` `exclude`, so the four verification gates keep their -scope. diff --git a/spikes/349-dofs/build.ts b/spikes/349-dofs/build.ts deleted file mode 100644 index 7f634d11..00000000 --- a/spikes/349-dofs/build.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { main } from "effection"; -import { exec } from "@effectionx/process"; - -const here = new URL("./", import.meta.url).pathname; - -main(function* () { - yield* exec(Deno.execPath(), { - arguments: [ - "compile", - "--allow-all", - "--frozen", - "--node-modules-dir=manual", - "--output", - "dist/proof", - "host/main.ts", - ], - cwd: here, - }).expect(); - yield* exec(Deno.execPath(), { - arguments: [ - "compile", - "--allow-all", - "--frozen", - "--node-modules-dir=manual", - "--output", - "dist/proof-shim", - "host/shim-main.ts", - ], - cwd: here, - }).expect(); - console.log("built dist/proof and dist/proof-shim"); -}); diff --git a/spikes/349-dofs/deno.json b/spikes/349-dofs/deno.json deleted file mode 100644 index 555dabb7..00000000 --- a/spikes/349-dofs/deno.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "exclude": [ - "vendor", - "vendor-build", - "dist" - ], - "nodeModulesDir": "manual", - "tasks": { - "all": "deno task vendor && deno task build && deno task test", - "vendor": "deno run --allow-all vendor.ts", - "build": "deno run --allow-all build.ts", - "test": "deno test --allow-all tests/", - "check": "deno check ." - } -} \ No newline at end of file diff --git a/spikes/349-dofs/evidence/COMPARISON.md b/spikes/349-dofs/evidence/COMPARISON.md deleted file mode 100644 index a1944eab..00000000 --- a/spikes/349-dofs/evidence/COMPARISON.md +++ /dev/null @@ -1,60 +0,0 @@ -# Comparison: Deno-local DOFS (#349) vs bundled workerd (#347 / PR #348) - -Both spikes ran on the same host (macOS 15 arm64, Deno 2.9.1) against the -same pinned `cloudflare/computer` v0.1.1. #347 numbers come from -`spikes/347-workerd/evidence/EVIDENCE.md` on PR #348; #349 numbers from -[EVIDENCE.md](EVIDENCE.md). - -| Axis | #349 Deno-local DOFS | #347 bundled workerd | -| --- | --- | --- | -| Runtime processes | one (Deno; plus a Node sidecar only if real FUSE is required) | two: xmd host + supervised workerd child | -| Artifact size | 110 MB proof (no extra runtime) | 191 MB proof (109 MB embedded workerd) | -| Startup / attach | 0.08 s full op cycle; no materialization step | 0.25 s serve-to-ready warm + 1.46 s first-run materialization of the 109 MB binary | -| Filesystem semantics | the DOFS filesystem API in-process; native subprocess access via shim (dev-only) or FUSE (Node sidecar; release-only durability, kernel-cache staleness ≤1 s) | the same DOFS filesystem behind the Worker boundary; native subprocess access only via the Container backend (Docker) | -| Execution backends | none of Computer's backends — exec is XMD's own process capability against the mount | all three Computer backends work (worker-shell, worker-javascript; container with Docker) | -| Platforms | linux-x64 proven; darwin-arm64: filesystem+shim only (FUSE dead end at this pin); linux-arm64 no addon prebuild; Windows: no FUSE at all | all five release targets have pinned workerd binaries; Linux glibc 2.35+, no musl; Windows lacks SIGTERM drain | -| Host prerequisites | none for filesystem+shim; `/dev/fuse` + libfuse2 (+ Node sidecar) for real FUSE | none for core; Docker for the container backend | -| Sandbox / security | none — DOFS code and subprocesses run with the host process's privileges | workerd isolates Worker JS but explicitly disclaims hardened sandboxing; native exec only inside Docker containers | -| Schema ownership / upgrades | identical schema, consumed at source (vendored 6.5k LOC, MIT provenance; 3-line upstream diff drafted as the exit) | identical schema, consumed through the published package (exact pin; refuse-on-downgrade) | -| Supervision | no child processes for the core path; FUSE adds mount lifecycle (auto_unmount proven; deadlock runbook recorded) | workerd child supervision (clean stop proven; SIGKILL orphans; container leak on SIGTERM) | -| Streaming / cancellation / retained exec handles | XMD's existing process capability (its own semantics) | Computer's `runtime.exec` handles (status/stdout/value round-trip proven) | -| Hosted-Cloudflare compatibility | filesystem layer byte-compatible; **no Durable Object identity, no Workers surface, no backends** — a local SQLite file is not a Durable Object | runs the actual published Computer package inside the actual Workers runtime — behaviorally closest to hosted | -| Migration / replication path | dofs ships its sync protocol helpers (`applyChanges`, manifests, watermarks) — the same protocol computerd speaks over capnweb; unexercised in this spike | same protocol, exercised end-to-end by the container backend's sync in #347 | - -## What each topology is best at - -**#349 wins on weight and directness**: a persistent SQLite Workspace with -Cloudflare's exact schema, in-process, 40% smaller artifact, ~3× faster -per operation, no child processes, no materialization cache, and identity -as simple as a database file path. The costs: no Computer execution -backends, no Workers isolation of any kind, a dev-only shim as the only -portable subprocess bridge today, and real FUSE gated on a Deno uv-polyfill -gap (upstream-ready repro committed) plus a Node sidecar in the interim. - -**#347 wins on fidelity**: it runs the real published package in the real -runtime with all three backends, so anything proven there transfers to -hosted Cloudflare almost by construction. The costs: 81 MB of extra -artifact, a supervised child process with the recorded teardown gaps, and -Docker for anything native. - -## Recommendation for #346 - -**Limit, not select-or-reject.** The evidence supports a split by concern: - -1. For the *filesystem-only Workspace* — the substrate `` needs - first — select the #349 topology: same schema, dramatically cheaper, - no supervision surface, and the vendoring path is proven with a small - upstream diff as its exit. -2. For *Computer execution backends* (worker-shell, worker-javascript, - containers) — if and when #346 wants them locally — the #347 topology - is the only one that provides them; keep it as the documented option - for that subset rather than the default local host. -3. Treat native-subprocess workspace access under #349 as explicitly - limited today: shim = development-only, FUSE = Linux with a Node - sidecar until the Deno N-API uv gap closes (file it upstream) — - and note that XMD's own exec against a mount is *not* behaviorally - equivalent to Computer's backends. - -Both spikes leave the door open to hosted Cloudflare through the same -sync protocol and identical schema; neither forecloses the other. The -final selection is recorded on #346. diff --git a/spikes/349-dofs/evidence/EVIDENCE.md b/spikes/349-dofs/evidence/EVIDENCE.md deleted file mode 100644 index 4b5272ee..00000000 --- a/spikes/349-dofs/evidence/EVIDENCE.md +++ /dev/null @@ -1,193 +0,0 @@ -# Evidence: Deno-local DOFS (#349) - -Answers to the spike's eight questions, measured on macOS 15 (Darwin -25.5.0, arm64), Deno 2.9.1, cloudflare/computer `v0.1.1` -(`63d363632e558f7e077794988d36ed75017c2a62`), 2026-08-06. Claims marked -**[test]** are asserted by `tests/spike.test.ts`; **[measured]** were -observed and are reproducible from the commands shown; **[probe]** carry -their full command-by-command ledger in `probes/`. - -## 1. Does a file-backed `node:sqlite` adapter satisfy DOFS's contract? - -Yes, with zero changes to Cloudflare's schema or filesystem primitives. -The whole contract is `{ sql.exec(query, ...bindings) → { toArray() }, -transactionSync }`; `host/file-storage.ts` implements it over -`new DatabaseSync(path)` in ~100 lines, mirroring upstream's own -`SQLiteTestStorage` fixture (which is `:memory:`-only). **[test]** - -- `mkdir`, `writeFile`, `readFile`, `readdir`, `stat`, `lstat`, `rename`, - `symlink`, `readlink`, `rm` all work through Cloudflare's unmodified - `Database` → `initializeSchema` → `WorkspaceFilesystem` stack; `readFile` - follows symlinks, `lstat` reports the link. **[test]** -- The filesystem frontier survives full process restarts — every test op is - a separate compiled-binary invocation — including deletion persistence and - a create → delete → create sequence. **[test]** -- Separate database paths are separate, initially-empty workspaces. There is - no Durable Object identity layer in this topology: the database file path - *is* the workspace identity. **[test]** -- Schema safety: `initializeSchema` is idempotent (IF-NOT-EXISTS DDL) and is - itself the version gate — a database stamped with a newer - `vfs_meta.schema_version` is refused loudly - (`Unsupported workspace filesystem schema version 99`), not recreated; - the failed open leaves the database untouched. `SCHEMA_VERSION` is 5. - **[test]** -- WAL mode: `-wal`/`-shm` exist only while open; `close()` checkpoints and - removes them, leaving a single-file artifact. **[test]** -- `node:sqlite` under Deno needed no workarounds: `DatabaseSync`, prepared - statements, and transactions all behave; dofs's own row normalization - handles the null-prototype rows. **[probe]** -- Sizes and timings **[measured]**: compiled proof 110 MB (no workerd, no - Worker bundle, no materialization step); full op cycle - (process start → op → close) 0.08 s wall, 14 ms in-process on a cold - database; upstream build of the vendored package is a clean plain `tsc`. - -## 2. Which reuse boundary works? - -Tested independently **[probe: probes/slice2-reuse.md]**: - -| Approach | Works? | Upgrades | MIT notices | Cloudflare code owned | -| --- | --- | --- | --- | --- | -| 1. Direct import of published `@cloudflare/computer@0.1.1` | **No** — `dist/index.js` unconditionally imports `cloudflare:workers` (`ERR_UNSUPPORTED_ESM_URL_SCHEME`); deep imports blocked by the exports map; no `cloudflare:`-free DOFS chunk exists in dist | — | none | zero | -| 2. Tree-shaken bundle of the published package | **No (structurally)** — one stub silences `cloudflare:workers`, but the published surface never exports `Database`/`initializeSchema`/`WorkspaceFilesystem`, so the schema DDL is not even present in the bundle | re-bundle + re-prove stubs per release | bundle must carry LICENSE | stubs + reimplemented schema DDL | -| 3. Pinned vendored `packages/dofs` | **Yes, fully** — clean `tsc` build, zero runtime deps, source unmodified; consumed as a `file:` npm dependency through its exports map | re-vendor from tag, re-apply 3 manifest edits, rebuild, re-run tests | copy root LICENSE + provenance note (no per-package license upstream) | 48 files / ~6.5k LOC (src minus tests) | -| 4. Upstream export change | **Proven mechanically** — a 3-line upstream diff (re-export module + bundler input + exports entry) yields a DOFS chunk whose module graph imports only `node:crypto`/`node:events`; verified under Deno | `npm i` once accepted | none | zero after acceptance | - -The spike ships approach 3 (`vendor/dofs/` + `PROVENANCE.md`), with -approach 4 drafted in the probe ledger as the exit path — the vendored copy -keeps the `@cloudflare/dofs` specifier, so landing the upstream change -reduces to a dependency swap. This slice satisfies "the exact supported -code-reuse mechanism is demonstrated": vendoring is the only mechanism that -works today, and its obligations are notice retention plus a -three-edit-manifest upgrade procedure. - -## 7. Userspace shim — verdict: development-only fallback - -Separate from the real-FUSE verdict (§3-6), per the spike's terms. Upstream's -`shim.ts` runs under Deno **byte-identical, zero runtime failures** — it is -vendored as a subset package (`vendor/computerd-shim/`, provenance + -sha1), and a compiled `proof-shim` executes a native subprocess that reads -an API-written file through the mount and writes back into SQLite, and -rematerializes an emptied mount directory from the persisted frontier. -**[test]** Full ledger: `probes/slice6-shim.md`. - -Measured (medians over 10 rounds, default 100 ms provider watch / 250 ms -shim poll) **[probe]**: - -| Direction | Median | -| --- | --- | -| VFS API write → visible on disk | 80.8 ms | -| external disk write → visible via API | 108.3 ms (worst 205.7 ms) | -| external disk write → committed in SQLite | 109.4 ms | -| rematerialize 100 files / 10 MB | 78.9 ms | -| 50 MB file, 1-byte change, reconcile | ~600 ms each way (full re-read) | - -Demonstrated losses: conflicts within a poll window resolve **VFS-wins** -(3/3 both orders); symlinks degrade to content copies on both sides -(dangling links dropped); chmod is invisible; after SIGKILL the WAL -recovers and convergence completes in ~1 s, but up to one poll window -(250 ms) of external disk writes is silently clobbered by the VFS copy. -Two integration facts a host must honor: the mount path is embedded in the -workspace namespace (the same database mounted at a different absolute path -materializes nothing), and `@platformatic/vfs`'s `create()` silently falls -back to a MemoryProvider unless the prototype splice is verified -(`host/vfs-wiring.ts` guards this explicitly). - -Verdict: viable as a **development-only fallback** — sub-poll-window -durability, symlink/metadata fidelity, and large-file costs disqualify it -as a supported production path. On darwin-arm64 it is currently the *only* -path (see §4/§5 platform record). - -## 3.-6. Real FUSE — verdict: works, but not in-process under Deno today - -Measured in a Linux/amd64 container with `/dev/fuse` (the same environment -upstream's own FUSE tests use); full ledger -`probes/slice3-5-fuse-linux.md`, key artifacts (container recipe, minimal -Deno repro, mount harness) in `fuse-linux/`. **[probe]** - -**The gating fact.** Deno 2.9.1/2.9.4 loads the fuse-native N-API addon -cleanly (both `import` through node-gyp-build and raw `process.dlopen`), -but any actual `fuse.mount()` aborts the whole Deno process uncatchably: -`bad result in uv polyfill: 1` (SIGABRT, exit 134) — Deno's -`uv_default_loop` polyfill returns null and fuse-native's mount path -exercises it. Reproduced under `deno run` and `deno compile`, under both -Rosetta and qemu; Node 22 mounts the identical script in 27 ms. A minimal -upstream-issue-ready repro is committed (`fuse-linux/g2-min.mjs`). Until -that Deno gap closes, real FUSE requires a **Node sidecar process**. - -**The stack itself is sound.** Under a Node sidecar, the full chain — -file-backed SQLite → dofs → spliced provider → `@platformatic/vfs` → -upstream `driver.ts` compiled verbatim — mounts in 118-133 ms cold, and a -second process (Deno) reads **and writes** the same WAL database with -changes visible through the mount immediately: the -Deno-main + Node-FUSE-sidecar topology over one shared database is proven -live, not hypothesized. All exercised ops pass: create, overwrite, -positional write (`dd seek`), truncate, rename, symlink, delete, -traversal, concurrent readers. Two semantic caveats: API-side overwrites -of kernel-cached files can read stale through the mount for ≤1 s -(`auto_cache` + `attr_timeout=1`), and `writeFileRangesSync` is confirmed -unreachable (probed by the driver, never forwarded by upstream's wiring — -dead code by omission). - -**Durability (slice 5 matrix).** The write-commit boundary is *release*, -observed strictly: - -| Scenario | Result | -| --- | --- | -| write + close, then read from a separate process | committed | -| write + `fsync`, no close, SIGKILL | **lost — the inode never reaches the database before release** | -| write + close, SIGKILL immediately after | **lost** (RELEASE is async; +500 ms → committed) | -| SIGTERM / SIGKILL of the mount host | `auto_unmount` clears the mount; no ENOTCONN hang; WAL recovers on next open | -| second process opens the live database | reads and writes concurrently (WAL); visible through the mount | - -`close()` is not a durability barrier and `fsync` is a durability no-op: -a host needs release-observed barriers (or explicit provider-level flush) -before treating a file as persisted. - -**The forbidden-spawn hazard, demonstrated.** Spawning with `cwd` inside -the mount from the process serving FUSE deadlocked the mount on the first -try — parent blocked in `pipe_read`, forked child in -`request_wait_answer`, readers unkillable in D state — and SIGKILL of the -host does **not** recover it (the forked child inherits the `/dev/fuse` -fd); recovery required `fusermount -z` plus a fusectl connection abort. -Upstream's `cd`-prefix technique works and is mandatory. - -**Packaging (slice 4).** `deno compile` can carry and load the addon both -ways (static import of the embedded npm graph, and `--include` → -materialize → `process.dlopen`), 142 MB binary — the packaging story is -proven even though the in-process mount then hits the same uv-polyfill -abort. Platform matrix, tested rather than inferred: linux-x64 works (via -sidecar); darwin-arm64 is a dead end at this pin (no prebuild; a source -build loads but SIGSEGVs on mount — no arm64 slice in the bundled -osxfuse-era dylib; macFUSE absent and would need kernel approval); -darwin-x64 has a prebuild but was not tested on real hardware; linux-arm64 -has no addon prebuild; Windows has no fuse-native support at all. One -tooling landmine recorded: Docker's Rosetta runner cannot execute -deno-compiled amd64 binaries (ld.so assert) — qemu-user was required. - -Blueprint facts established from source and confirmed by the probes -(cloudflare/computer@v0.1.1): - -- The FUSE adapter is `makeFUSEOps(vfs, mountPoint)` — one 1092-line file - whose only dependency is a `@platformatic/vfs` instance; `fuse-native@2.2.6` - (libfuse 2.9 API) with prebuilds for linux-x64 and darwin-x64 only — no - darwin-arm64, no linux-arm64 addon prebuild. -- Wiring `SQLiteWorkspaceProvider` into `@platformatic/vfs` requires two - upstream hacks replicated verbatim: a prototype splice (vfs's `create()` - silently falls back to a MemoryProvider on an instanceof failure) and - explicit forwarding of ten dofs-specific sync methods onto the vfs facade. -- The FUSE write-commit boundary is **release-only**: dofs's write buffer - commits when the open count reaches zero; `flush` and `fsync` are - durability no-ops in the production configuration. -- The darwin-arm64 platform record **[probe: probes/slice6-shim.md]**: - `fuse-native@2.2.6` ships no darwin-arm64 prebuild; a manual source build - produces an arm64 addon that loads under Deno and Node, but its bundled - `libosxfuse.dylib` carries no arm64 slice (kext-era osxfuse 3.x) and an - actual mount SIGSEGVs under both runtimes; macFUSE is not installed on - this host (`/Library/Filesystems/macfuse.fs` absent). Real FUSE on Apple - Silicon is a dead end at this pinned version even before the - macFUSE-install/kernel-approval prerequisite. - -## 8. Comparison with #347 - -Recorded in [COMPARISON.md](COMPARISON.md), including the -select/reject/limit recommendation for #346. diff --git a/spikes/349-dofs/evidence/fuse-linux/Dockerfile b/spikes/349-dofs/evidence/fuse-linux/Dockerfile deleted file mode 100644 index 7733f78a..00000000 --- a/spikes/349-dofs/evidence/fuse-linux/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM --platform=linux/amd64 node:22-slim -RUN apt-get update && apt-get install -y --no-install-recommends \ - fuse libfuse2 procps curl unzip ca-certificates \ - && rm -rf /var/lib/apt/lists/* -RUN curl -fsSL -o /tmp/deno.zip https://dl.deno.land/release/v2.9.1/deno-x86_64-unknown-linux-gnu.zip \ - && unzip /tmp/deno.zip -d /usr/local/bin \ - && rm /tmp/deno.zip \ - && chmod +x /usr/local/bin/deno -RUN mkdir -p /mnt/ws -CMD ["sleep", "infinity"] diff --git a/spikes/349-dofs/evidence/fuse-linux/g2-min.mjs b/spikes/349-dofs/evidence/fuse-linux/g2-min.mjs deleted file mode 100644 index 0adb72aa..00000000 --- a/spikes/349-dofs/evidence/fuse-linux/g2-min.mjs +++ /dev/null @@ -1,44 +0,0 @@ -// Minimal mount repro, runtime-agnostic (node g2-min.mjs / deno run -A g2-min.mjs). -// Mounts a trivial one-file FS, stats it, unmounts. Isolates the FUSE binding -// from the dofs/vfs stack. -import { createRequire } from "node:module"; -const require = createRequire(import.meta.url); -const Fuse = require("fuse-native"); - -const mountPoint = process.argv[2] ?? "/mnt/ws"; -const stat = (mode, size) => ({ - mtime: new Date(), atime: new Date(), ctime: new Date(), - size, mode, uid: 0, gid: 0, -}); -const ops = { - readdir: (path, cb) => cb(0, path === "/" ? ["hello.txt"] : []), - getattr: (path, cb) => { - if (path === "/") return cb(0, stat(0o40755, 4096)); - if (path === "/hello.txt") return cb(0, stat(0o100644, 6)); - return cb(Fuse.ENOENT); - }, - open: (path, flags, cb) => cb(0, 42), - read: (path, fd, buf, len, pos, cb) => { - const data = Buffer.from("hello\n").subarray(pos, pos + len); - data.copy(buf); - cb(data.length); - }, - release: (path, fd, cb) => cb(0), -}; - -console.log(JSON.stringify({ event: "constructing", mountPoint })); -const fuse = new Fuse(mountPoint, ops, { autoUnmount: true, debug: false }); -const t0 = performance.now(); -fuse.mount((err) => { - if (err) { - console.log(JSON.stringify({ event: "mount-error", error: String(err) })); - process.exit(1); - } - console.log(JSON.stringify({ event: "mounted", ms: Math.round(performance.now() - t0) })); - setTimeout(() => { - fuse.unmount((err2) => { - console.log(JSON.stringify({ event: "unmounted", error: err2 ? String(err2) : null })); - process.exit(0); - }); - }, 10_000); -}); diff --git a/spikes/349-dofs/evidence/fuse-linux/mount-host.mjs b/spikes/349-dofs/evidence/fuse-linux/mount-host.mjs deleted file mode 100644 index 9dd8b21a..00000000 --- a/spikes/349-dofs/evidence/fuse-linux/mount-host.mjs +++ /dev/null @@ -1,148 +0,0 @@ -// Goal 2 host: full stack. -// FileSQLiteStorage(file db) -> Database -> initializeSchema -// -> SQLiteWorkspaceProvider -> prototype splice -> vfs create (+forwarding) -// -> makeFUSEOps/mountFuse (compiled CJS driver) -> /mnt/ws -// Then serves an HTTP control API on 127.0.0.1:9976 so separate processes -// (curl in docker exec) can drive the dofs API side while shells poke the mount. -// -// Runtime-agnostic on purpose: `deno run --allow-all mount-host.mjs ...` -// aborts in the uv polyfill at mount() (goal 1/2 finding); `node -// mount-host.mjs ...` is the working sidecar topology. -import { createRequire } from "node:module"; -import { createServer } from "node:http"; -import process from "node:process"; -import { createFileVfs, verifySqliteBacked } from "./vfs-wiring.mjs"; - -const [dbPath = "/probe/data/ws.db", mountPoint = "/mnt/ws"] = process.argv.slice(2); -const require = createRequire(import.meta.url); - -const t0 = performance.now(); -const handle = createFileVfs(dbPath); -const { vfs, wfs, storage } = handle; -// makeFUSEOps uses mountPoint as the VFS namespace prefix: kernel "/" -// maps to vfs "". The backing dir chain must exist in the vfs -// or every op (including getattr of the mount root) returns ENOENT. -{ - const segments = mountPoint.split("/").filter(Boolean); - let acc = ""; - for (const seg of segments) { - acc += `/${seg}`; - if (!vfs.existsSync(acc)) vfs.mkdirSync(acc, { mode: 0o755 }); - } -} -if (!vfs.existsSync("/workspace")) vfs.mkdirSync("/workspace", { mode: 0o755 }); -const sqliteProof = await verifySqliteBacked(handle, dbPath); -const tVfs = performance.now(); - -// Record whether writeFileRangesSync reached the facade (upstream omits it -// from the forward list even though driver.ts probes for it). -const forwarding = Object.fromEntries( - [ - "linkSync", "createFileSync", "writeRangeSync", "truncateFileSync", - "chmodSync", "readRangeSync", "openWriteBufferSync", - "openWriteBufferForCreateSync", "releaseWriteBufferSync", - "writeFileRangesSync", - ].map((name) => [name, typeof vfs[name]]), -); - -const { mountFuse } = require("./dist-cjs/driver.js"); -const tRequire = performance.now(); -const mount = await mountFuse({ mountPoint, vfs }); -const tMounted = performance.now(); - -console.log(JSON.stringify({ - event: "mounted", - runtime: typeof Deno === "undefined" ? `node ${process.version}` : `deno ${Deno.version.deno}`, - pid: process.pid, - dbPath, - mountPoint, - sqliteProof, - forwarding, - timingsMs: { - vfsUp: r(tVfs - t0), - driverRequire: r(tRequire - tVfs), - fuseMount: r(tMounted - tRequire), - total: r(tMounted - t0), - }, -})); - -function r(x) { return Math.round(x * 100) / 100; } - -const server = createServer(async (req, res) => { - const url = new URL(req.url, "http://127.0.0.1:9976"); - const q = (k) => url.searchParams.get(k); - const json = (obj, status = 200) => { - res.writeHead(status, { "content-type": "application/json" }); - res.end(JSON.stringify(obj)); - }; - try { - switch (url.pathname) { - case "/write": { - const t = performance.now(); - await wfs.writeFile(q("path"), q("body") ?? ""); - return json({ ok: true, ms: r(performance.now() - t) }); - } - case "/read": { - const t = performance.now(); - const body = await wfs.readFile(q("path"), "utf8"); - return json({ ok: true, body, ms: r(performance.now() - t) }); - } - case "/ls": { - const entries = (await wfs.readdir(q("path"))).map((e) => e.name).sort(); - return json({ ok: true, entries }); - } - case "/stats": - return json({ ok: true, bufferStats: mount.getBufferStats?.() ?? null }); - case "/bench-direct": { - const n = Number(q("n") ?? 100); - const t = performance.now(); - for (let i = 0; i < n; i++) { - await wfs.writeFile(`${mountPoint}/bench-api-${i}.txt`, `payload-${i}\n`); - } - const total = r(performance.now() - t); - return json({ ok: true, n, totalMs: total, perOpMs: r(total / n) }); - } - case "/exec": { - // Upstream exec/runner.ts technique: NEVER pass cwd: to spawn from the process serving FUSE (uv_spawn forks then - // chdirs pre-exec; the chdir triggers a FUSE request only this - // process can serve -> deadlock). Instead prefix `cd &&` so the - // chdir happens in the child shell after exec. - const dir = q("cwd") ?? "/"; - const cmd = q("cmd") ?? "pwd"; - const { execFile } = await import("node:child_process"); - const t = performance.now(); - const out = await new Promise((resolve) => { - execFile("/bin/sh", ["-c", `cd ${dir} && ${cmd}`], { timeout: 10_000 }, - (error, stdout, stderr) => resolve({ error: error ? String(error) : null, stdout, stderr })); - }); - return json({ ok: out.error === null, technique: "cd-prefix", ...out, ms: r(performance.now() - t) }); - } - case "/exec-cwd": { - // The hazardous variant, for characterization only: spawn with the - // cwd option pointing inside our own mount. - const dir = q("cwd") ?? "/mnt/ws"; - const { execFile } = await import("node:child_process"); - const t = performance.now(); - const out = await new Promise((resolve) => { - execFile("/bin/sh", ["-c", q("cmd") ?? "pwd"], { cwd: dir, timeout: 10_000 }, - (error, stdout, stderr) => resolve({ error: error ? String(error) : null, stdout, stderr })); - }); - return json({ ok: out.error === null, technique: "spawn-cwd-option", ...out, ms: r(performance.now() - t) }); - } - case "/unmount": { - await mount.unmount(); - storage.close(); - setTimeout(() => process.exit(0), 100); - return json({ ok: true, unmounted: true }); - } - default: - return json({ ok: false, error: "unknown endpoint" }, 404); - } - } catch (error) { - return json({ ok: false, error: String(error?.message ?? error), code: error?.code }, 500); - } -}); -server.listen(9976, "127.0.0.1", () => { - console.log(JSON.stringify({ event: "control-api", port: 9976 })); -}); diff --git a/spikes/349-dofs/evidence/fuse-linux/vfs-wiring.mjs b/spikes/349-dofs/evidence/fuse-linux/vfs-wiring.mjs deleted file mode 100644 index a0f57e5b..00000000 --- a/spikes/349-dofs/evidence/fuse-linux/vfs-wiring.mjs +++ /dev/null @@ -1,107 +0,0 @@ -// Re-implementation of computerd's src/fuse/vfs.ts wiring in plain JS, -// minus the @cloudflare/computer-rpc sync loop, plus a file-backed -// SQLite storage instead of SQLiteTestStorage(:memory:). -// -// Wiring steps (mirroring vfs.ts): -// 1. prototype-splice SQLiteWorkspaceProvider -> VirtualProvider -// (else @platformatic/vfs's create() SILENTLY falls back to MemoryProvider) -// 2. create(provider, { moduleHooks: false }) -// 3. Object.defineProperty-forward EXTRA_VFS_METHODS onto the vfs facade. -// NOTE: writeFileRangesSync is probed by driver.ts but deliberately -// NOT in the upstream forward list; we replicate that exactly and -// record whether it reaches the vfs. - -import { - Database, - initializeSchema, - SQLiteWorkspaceProvider, - WorkspaceFilesystem, -} from "./dofs/dist/index.js"; -import { FileSQLiteStorage } from "./file-storage.mjs"; -import { create, VirtualProvider } from "@platformatic/vfs"; - -// Exactly the upstream list from packages/computerd/src/fuse/vfs.ts. -const EXTRA_VFS_METHODS = [ - "linkSync", - "createFileSync", - "writeRangeSync", - "truncateFileSync", - "chmodSync", - "readRangeSync", - "openWriteBufferSync", - "openWriteBufferForCreateSync", - "releaseWriteBufferSync", -]; - -let prototypePatched = false; -function ensureVirtualProviderPrototype() { - if (prototypePatched) return; - const proto = SQLiteWorkspaceProvider.prototype; - const parent = Object.getPrototypeOf(proto); - if (parent === VirtualProvider.prototype) { - prototypePatched = true; - return; - } - Object.setPrototypeOf(proto, VirtualProvider.prototype); - prototypePatched = true; -} - -export function createFileVfs(dbPath) { - ensureVirtualProviderPrototype(); - const storage = new FileSQLiteStorage(dbPath); - const db = new Database(storage); - initializeSchema(db, () => Date.now()); - - const provider = new SQLiteWorkspaceProvider(db); - const vfs = create(provider, { moduleHooks: false }); - for (const name of EXTRA_VFS_METHODS) { - const fn = provider[name]; - if (typeof fn !== "function") continue; - Object.defineProperty(vfs, name, { - value: (...args) => fn.apply(provider, args), - writable: true, - configurable: true, - }); - } - - // WorkspaceFilesystem over the same Database = the "API side". - const wfs = new WorkspaceFilesystem(db); - return { vfs, db, wfs, storage, provider }; -} - -// Prove we're on SQLite, not the silent MemoryProvider fallback: -// write through the vfs facade, read back through a FRESH node:sqlite -// connection over the same db file. -export async function verifySqliteBacked(vfsHandle, dbPath) { - const marker = `sqlite-proof-${Date.now()}`; - if (!vfsHandle.vfs.existsSync("/workspace")) { - vfsHandle.vfs.mkdirSync("/workspace", { mode: 0o755 }); - } - vfsHandle.vfs.writeFileSync("/workspace/.sqlite-proof", marker); - const { DatabaseSync } = await import("node:sqlite"); - const fresh = new DatabaseSync(dbPath); - try { - const rows = fresh - .prepare("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name") - .all(); - const tables = rows.map((r) => r.name); - // Find the inode row for the proof file. - let proofRow = null; - for (const t of tables) { - try { - const hit = fresh - .prepare(`SELECT * FROM ${t} WHERE CAST(name AS TEXT) = ? LIMIT 1`) - .all(".sqlite-proof"); - if (hit.length > 0) { - proofRow = { table: t }; - break; - } - } catch { - // table without a name column; ignore - } - } - return { tables, proofFileFoundInTable: proofRow?.table ?? null, marker }; - } finally { - fresh.close(); - } -} diff --git a/spikes/349-dofs/evidence/probes/slice1-adapter.md b/spikes/349-dofs/evidence/probes/slice1-adapter.md deleted file mode 100644 index 5900219f..00000000 --- a/spikes/349-dofs/evidence/probes/slice1-adapter.md +++ /dev/null @@ -1,259 +0,0 @@ -# Probe: file-backed node:sqlite adapter for Cloudflare DOFS under Deno (issue #349, slice 1) - -Runtime under test: Deno 2.9.1 (aarch64-apple-darwin). Reference: cloudflare/computer v0.1.1 -(63d363632e558f7e077794988d36ed75017c2a62), `packages/dofs`, read-only clone at `../cf-computer`. -Working vendored copy: `./dofs` (modified only in `package.json` devDeps + `tsconfig.build.json` -types array — see Step 1). - -## Verdict (condensed) - -PASS on every axis. A ~70-line file-backed `DurableObjectStorageLike` adapter over -`new DatabaseSync(path)` from `node:sqlite` satisfies the whole contract with ZERO changes to -dofs source: the filesystem frontier (writes, deletes, renames, symlinks, delete-then-recreate) -survives full Deno process restarts; a second db file is a fully isolated empty workspace; the -schema-version gate refuses a database stamped by a newer binary with a loud -`WorkspaceFsError: Unsupported workspace filesystem schema version 99` and exit code 1. -Warm per-op process wall time ~35 ms (in-process op time 1.5–4 ms). - -## Step 1 — vendor + build - -``` -cp -R cf-computer/packages/dofs probe-dofs/dofs -cd probe-dofs/dofs && npm i -D typescript @cloudflare/workers-types @types/node -``` - -First install FAILED (verbatim core): - -``` -npm error code ERESOLVE -npm error While resolving: wrangler@4.119.0 -npm error Found: @cloudflare/workers-types@4.20260702.1 -npm error peerOptional @cloudflare/workers-types@"^5.20260801.1" from wrangler@4.119.0 -npm error Conflicting peer dependency: @cloudflare/workers-types@5.20260804.1 -``` - -Cause: the package's own devDeps pin `wrangler@^4.107.1`, and the wrangler line has since moved -to a `workers-types@^5` peer — the conflict is between two TEST-ONLY devDeps (wrangler, -@cloudflare/vitest-pool-workers), not anything the build needs. - -Fix (in the vendored copy only): -1. `package.json` devDependencies trimmed to `typescript@^6.0.3`, - `@cloudflare/workers-types@^4.20260616.1`, `@types/node@^24`. -2. `tsconfig.build.json` overrides `compilerOptions.types` to - `["@cloudflare/workers-types", "node"]` (the base tsconfig demands `vitest/globals` and - `@cloudflare/vitest-pool-workers/types`, again test-only). - -Then: - -``` -npm i # clean, 0 vulnerabilities -./node_modules/.bin/tsc -p tsconfig.build.json # exit 0, no diagnostics -``` - -Build is CLEAN. Emits `dist/` as ESM `.js` + `.d.ts` mirroring `src/` (21 top-level entries incl. -`index.js`, `storage.js`, `schema/`, `fs/` with every op — `fs/rename.js` included). No -`--unstable-sloppy-imports` fallback needed; the tsc-build vendoring story works as-is. (Caveat: -`npx tsc` without a local install grabs the squatter `tsc@2.0.4` npm package — use -`./node_modules/.bin/tsc` or install typescript first.) - -## Step 2 — adapter: `file-storage.mjs` - -Mirrors `src/testing.ts` `SQLiteTestStorage` exactly (statement cache, `toSQLiteValue` -normalization: undefined/null→null, boolean→1/0, Uint8Array passthrough, string/number/bigint -passthrough, TypeError otherwise; BEGIN/COMMIT/ROLLBACK `transactionSync`) but opens a real path. -Pragmas chosen: `journal_mode = WAL`, `foreign_keys = ON`, `synchronous = NORMAL`. Exposes -`close()` and a `pragma()` helper for the probe. - -```js -// FileSQLiteStorage — file-backed DurableObjectStorageLike over -// node:sqlite under Deno. Mirrors dofs's SQLiteTestStorage -// (src/testing.ts) exactly — prepared-statement cache, binding -// normalization, BEGIN/COMMIT/ROLLBACK transactionSync — but opens -// a real database file instead of ":memory:". - -import { DatabaseSync } from "node:sqlite"; - -class Cursor { - #rows; - constructor(rows) { - this.#rows = rows; - } - toArray() { - return this.#rows; - } -} - -export class FileSQLiteStorage { - #db; - #cache = new Map(); - sql; - - constructor(path, { journalMode = "WAL" } = {}) { - this.#db = new DatabaseSync(path); - this.#db.exec(`PRAGMA journal_mode = ${journalMode}`); - this.#db.exec("PRAGMA foreign_keys = ON"); - this.#db.exec("PRAGMA synchronous = NORMAL"); - this.sql = { - exec: (query, ...bindings) => { - let stmt = this.#cache.get(query); - if (stmt === undefined) { - stmt = this.#db.prepare(query); - this.#cache.set(query, stmt); - } - const normalized = bindings.map(toSQLiteValue); - const rows = stmt.all(...normalized) ?? []; - return new Cursor(rows); - }, - }; - } - - transactionSync(closure) { - this.#db.exec("BEGIN"); - try { - const result = closure(); - this.#db.exec("COMMIT"); - return result; - } catch (error) { - this.#db.exec("ROLLBACK"); - throw error; - } - } - - pragma(name) { - return this.#db.prepare(`PRAGMA ${name}`).all(); - } - - close() { - this.#cache.clear(); - this.#db.close(); - } -} - -// Same normalization as SQLiteTestStorage.toSQLiteValue. -function toSQLiteValue(value) { - if (value === undefined || value === null) return null; - if (typeof value === "boolean") return value ? 1 : 0; - if (value instanceof Uint8Array) return value; - if (typeof value === "string" || typeof value === "number" || typeof value === "bigint") { - return value; - } - throw new TypeError(`FileSQLiteStorage cannot bind value of type ${typeof value}`); -} -``` - -## Step 3 — probe harness: `probe.mjs` - -One op per invocation → one Deno process per op. Construction recipe (canonical order taken -from `src/fs/filesystem.test.ts` `withFs`): - -```js -import { Database, initializeSchema, SCHEMA_VERSION, WorkspaceFilesystem } from "./dofs/dist/index.js"; -import { rename } from "./dofs/dist/fs/rename.js"; // NOT on the class or index.ts — free function only -import { FileSQLiteStorage } from "./file-storage.mjs"; - -const storage = new FileSQLiteStorage(dbPath); // 1. adapter -const db = new Database(storage); // 2. wrap -initializeSchema(db, Date.now); // 3. idempotent DDL + migration gate + root inode -const fs = new WorkspaceFilesystem(db); // 4. ops surface ({ now } optional) -// ... run op ... -storage.close(); // 5. checkpoints WAL, removes -wal/-shm -``` - -`initializeSchema(db, now)` is safe to call on EVERY process start: all DDL is -`CREATE ... IF NOT EXISTS`, version stamping is insert-or-ignore + update, and it is exactly -where the too-new-schema gate lives. - -Surprise: `WorkspaceFilesystem` has no `rename` method and `index.ts` does not export the free -function; `rename(db, oldPath, newPath)` must be imported from `dist/fs/rename.js` directly. - -## Step 4 — frontier across fresh processes (every line = a separate `deno run -A probe.mjs ws.db ...`) - -``` -init → {"op":"init","schemaVersion":5,"ms":2.92} (cold; 0.050s wall) -mkdir /notes → ok ms:2.7 -write /notes/a.md alpha → ok ms:4.18 -read /notes/a.md → "alpha" -write /notes/b.md beta → ok -rm /notes/a.md → ok -ls /notes → ["b.md"] -write /f.txt v1 → ok -rm /f.txt → ok -write /f.txt v2 → ok -read /f.txt → "v2" (delete-then-recreate survives) -stat /notes/b.md → {"name":"b.md","inode":4,"mode":420,"mtime":1785988833560, - "size":4,"isFile":true,"isDirectory":false,"isSymbolicLink":false} -rename /notes/b.md /notes/c.md → ok -ls /notes → ["c.md"]; lsflat / → ["/f.txt","/notes/c.md"] -symlink /notes/c.md /link → ok (signature: fs.symlink(target, path)) -readlink /link → "/notes/c.md" -read /link → "beta" → YES, readFile FOLLOWS symlinks -lstat /link → {"isSymbolicLink":true,"isFile":false,"mode":511,"size":11} -read /notes/c.md → "beta" (final frontier proof after ALL of the above) -``` - -VERDICT: PASS. Every mutation made in one process is observed by the next; nothing leaked from -deleted files. - -## Step 5 — isolation & identity - -`deno run -A probe.mjs ws2.db lsflat /` → `{"paths":[]}`; `ls /` → `[]`. A second db path is a -fresh, empty workspace. Identity model: ONE DB FILE = ONE WORKSPACE. There is no Durable Object -identity layer here — in Cloudflare's runtime the DO id names the storage; file-backed, the -filesystem path of the .db file IS the workspace identity, and nothing inside the schema names -or authenticates the workspace. Callers own the path→workspace mapping (and any locking between -concurrent writers — SQLite WAL allows one writer at a time across processes). - -## Step 6 — schema guard - -`meta` on the live db: binary `SCHEMA_VERSION = 5`; `vfs_meta` = `[{k:"rev",v:10}, -{k:"schema_version",v:5}]`; `journal_mode` = `wal`. (Version lives in table `vfs_meta`, -key `schema_version`.) - -Guard test: `cp ws.db ws-future.db`, then `bump-version.mjs` sets stored `schema_version = 99`. -Opening it: - -``` -error: Uncaught (in promise) WorkspaceFsError: Unsupported workspace filesystem schema version 99 - at createWorkspaceError (dofs/dist/errors.js:2:19) - at dofs/dist/schema/index.js:25:19 - at FileSQLiteStorage.transactionSync (file-storage.mjs:46:22) - at Database.transactionSync (dofs/dist/storage.js:39:36) - at initializeSchema (dofs/dist/schema/index.js:7:8) -``` - -Exit code 1. The throw happens inside `transactionSync`, so the adapter's ROLLBACK ran and the -future-versioned db is untouched. VERDICT: the "binary older than on-disk schema_version" -gate works loudly and exactly as in the DO runtime. - -## Step 7 — timings (M-series mac, warm OS cache) - -- Cold (fresh db file, full init): 0.050 s process wall; 2.92 ms in-process (open → init done). -- Warm (already-initialized db): 0.035 s process wall; 1.5–4.2 ms in-process per op - (reads ~1.5–2 ms, writes ~3.5–4.2 ms — writes pay the transaction fsync at - `synchronous = NORMAL`). -- Per-process overhead is dominated by Deno startup (~30 ms), not dofs or SQLite. - -## Step 8 — node:sqlite under Deno 2.9.1 - -- `DatabaseSync(path)`, `prepare()`, `StatementSync.all()/run()/get()`, `exec()` for - BEGIN/COMMIT/ROLLBACK, and SAVEPOINT/RELEASE (used by `Database`'s reentrant - `transactionSync`) all work. `PRAGMA journal_mode/foreign_keys/synchronous` work. -- No Deno-specific failures were hit anywhere in the probe. -- node:sqlite quirks dofs ALREADY handles: rows come back with a null prototype - (`storage.ts` `normalizeRow` re-keys into plain objects) and BLOBs come back as Uint8Array - (also normalized there). `bump-version.mjs` output showed the raw - `[Object: null prototype] { v: 99 }` shape, confirming the normalization is doing real work. -- WAL sidecars: `wal-check.mjs` proves `ws.db-wal` and `ws.db-shm` EXIST while the database is - open with uncommitted-to-main-file writes, and `close()` checkpoints and REMOVES both - (`{"during":{"wal":true,"shm":true},"after":{"wal":false,"shm":false}}`). After every probe - run only the bare `.db` file remains, so a process that closes cleanly leaves a - single-file artifact; a killed process would leave `-wal`/`-shm` behind and the next open - replays them (standard SQLite WAL recovery). - -## Files - -- `file-storage.mjs` — the adapter (final code inline above) -- `probe.mjs` — CLI probe harness -- `bump-version.mjs`, `wal-check.mjs` — step 6 / step 8 helpers -- `dofs/` — vendored package (2 build-config edits, zero source edits), `dofs/dist/` — tsc output -- `ws.db` — the surviving workspace; `ws2.db` — isolation check; `ws-future.db` — guard fixture diff --git a/spikes/349-dofs/evidence/probes/slice2-reuse.md b/spikes/349-dofs/evidence/probes/slice2-reuse.md deleted file mode 100644 index 0f01c13c..00000000 --- a/spikes/349-dofs/evidence/probes/slice2-reuse.md +++ /dev/null @@ -1,318 +0,0 @@ -# Probe: DOFS reuse boundaries (issue #349 slice 2) - -Reference clone: `scratchpad/cf-computer` @ tag v0.1.1 (63d3636). Deno 2.9.1. - -## Verdict table - -| Approach | Works? | Upgrade path | MIT obligations | Cloudflare code XMD owns | -|---|---|---|---|---| -| 1. Direct import of published `@cloudflare/computer@0.1.1` | **No** — every entry/deep path either hits `ERR_UNSUPPORTED_ESM_URL_SCHEME` (`cloudflare:`) or `ERR_PACKAGE_PATH_NOT_EXPORTED`; and 3 of the 4 needed symbols aren't exported at all | `npm i` (moot) | None beyond upstream's own tarball | Zero | -| 2. Tree-shaken esbuild bundle of the published package | **Half** — bundle loads under Deno with exactly 1 trivial stub, but only `SQLiteWorkspaceProvider` is reachable; `Database`/`initializeSchema`/`WorkspaceFilesystem` are unexported, so the schema can never be created | `npm i` + re-bundle + re-prove the stub-is-dead-code fact each release | LICENSE text must ship next to the bundle (embeds CF source) | Stub + build script + a hand-rolled `Database` duck-type + reimplemented schema DDL (DOFS internals) | -| 3. Pinned vendored copy of `packages/dofs` | **Yes** — tsc build clean; all 4 symbols import under Deno; schema v5 initializes over in-memory `node:sqlite`; `npm pack` tarball + bare-specifier consumer also works | Re-vendor from upstream tag + rebuild + re-run tests (manual, diffable — src is unmodified) | Copy root LICENSE into the vendored dir + provenance note (VENDOR.txt) | 48 files / 6,554 LOC of unmodified src + ~40 lines of package/tsconfig scaffolding | -| 4. Upstream `./dofs` subpath export (or publishing `@cloudflare/dofs`) | **Yes (proven mechanically)** — a 3-line rolldown entry + package.json exports entry yields a `dofs.js` whose chunk graph is 100% cloudflare:-free; runs under Deno, schema initializes | `npm i @cloudflare/computer` once accepted; Approach 3 vendored copy in the interim | None beyond upstream's tarball | Zero (after acceptance) | - -## License facts - -Single `LICENSE` at the monorepo root; **no per-package LICENSE files** anywhere -(`find cf-computer -iname 'LICENSE*'` returns only the root file, and -`packages/dofs/package.json` has no `license` field — it is `"private": true`, -version `0.0.0`). The published `@cloudflare/computer` package.json declares -`"license": "MIT"` but the npm tarball ships **no LICENSE file** (files: dist, -README.md only). - -Root LICENSE text: `MIT License Copyright (c) 2026 Cloudflare, Inc.` followed by -the standard MIT grant. Its condition: - -> The above copyright notice and this permission notice (including the next -> paragraph) shall be included in all copies or substantial portions of the -> Software. - -So any redistribution of DOFS code (vendored source OR a bundle containing it) -must carry, verbatim: the line `MIT License Copyright (c) 2026 Cloudflare, Inc.`, -the permission notice, **and** the warranty-disclaimer paragraph (the "next -paragraph" is explicitly pulled into the retention requirement). Placing a copy -of the root LICENSE file next to the vendored/bundled code plus a provenance -header (upstream repo URL, tag, commit) satisfies this. No copyleft, no -source-disclosure duty; modifications are allowed. - -## Approach 1 — direct import of the published package under Deno - -Setup: - -```sh -# approach1/package.json: { "type": "module" } -npm install --prefix approach1 @cloudflare/computer@0.1.1 @platformatic/vfs zod -# added 86 packages -``` - -### Dist inspection - -`node_modules/@cloudflare/computer/dist` layout: `index.js`, `git.js`, -`artifacts/`, `assets/`, `backends/{container,worker-javascript,worker-shell}/`, -`observe/`, `tools/`, and six `shared-*.js` chunks (rolldown build). - -- Chunks containing DOFS symbols (`initializeSchema`, `SQLiteWorkspaceProvider`, - `vfs_nodes`): **only `dist/index.js` and `dist/git.js`**. -- `dist/git.js` merely *proxies* method calls to a `SQLiteWorkspaceProvider` - instance; the class/schema definitions live **solely in `dist/index.js`** - (regions `../dofs/src/errors.ts`, `../dofs/src/path.ts`, … are inlined there). -- `dist/index.js` line 6, top-level and unconditional: - `import { RpcTarget as RpcTarget$1, WorkerEntrypoint } from "cloudflare:workers";` -- Files with `cloudflare:` scheme imports: `index.js`, - `backends/container/index.js`, `backends/worker-javascript/index.js`, - `backends/worker-shell/index.js`. -- No `shared-*` chunk carries the DOFS layer, so **no cloudflare:-free chunk - containing DOFS exists in the published dist**. -- `dist/index.js` exports include `SQLiteWorkspaceProvider`, `Workspace`, - `getWorkspace`, `withWorkspace` — but NOT `Database`, `initializeSchema`, or - `WorkspaceFilesystem` (those are internal to the chunk). - -### Runs - -`import { Workspace, SQLiteWorkspaceProvider } from "@cloudflare/computer"`: - -``` -$ deno run --allow-all --node-modules-dir=manual approach1/import-main.ts -error: [ERR_UNSUPPORTED_ESM_URL_SCHEME] Only file and data URLs are supported by the default ESM loader. Received protocol 'cloudflare' -``` - -Deep import `@cloudflare/computer/dist/index.js`: - -``` -error: [ERR_PACKAGE_PATH_NOT_EXPORTED] Package subpath './dist/index.js' is not defined by "exports" in .../@cloudflare/computer/package.json -``` - -Direct file-path import `./node_modules/@cloudflare/computer/dist/index.js` -(bypasses the exports map): - -``` -error: [ERR_UNSUPPORTED_ESM_URL_SCHEME] Only file and data URLs are supported by the default ESM loader. Received protocol 'cloudflare' -``` - -**Verdict: does not work.** The only chunk that defines the DOFS layer -statically imports `cloudflare:workers`, and Deno's ESM loader rejects the -`cloudflare:` protocol. No published entrypoint or deep import yields the DOFS -layer under Deno without patching. (Upgrades would have been `npm i`-simple and -MIT notice duty would sit with Cloudflare's tarball, but the approach is moot.) - -Additional blocker independent of the scheme problem: the published export -surface contains `SQLiteWorkspaceProvider` but **not** `Database`, -`initializeSchema`, or `WorkspaceFilesystem` — those never left `packages/dofs` -(`dist/index.js`'s export statement ends with `... SQLiteWorkspaceProvider, -TestBackend, Workspace, ... getWorkspace, noopObserver, sh, shellQuote, -withWorkspace`). - -## Approach 2 — tree-shaken esbuild bundle of the published package - -Setup: `approach2/package.json` with `@cloudflare/computer@0.1.1`, -`@platformatic/vfs`, `zod`, devDep `esbuild@0.28.1`; `npm install --prefix -approach2`. Facade (`facade.ts`): - -```ts -export { SQLiteWorkspaceProvider } from "@cloudflare/computer"; -``` - -(Only symbol of the four we need that the package exports — see Approach 1.) - -### Round 1 — externals only - -```sh -./node_modules/.bin/esbuild facade.ts --bundle --format=esm --platform=neutral \ - --main-fields=module,main --conditions=import \ - '--external:cloudflare:*' '--external:node:*' --outfile=bundle.js -# bundle.js 150.2kb -``` - -Output still contains (line 2501): -`import { RpcTarget as RpcTarget$1, WorkerEntrypoint } from "cloudflare:workers";` -— but grep shows **neither `RpcTarget$1` nor `WorkerEntrypoint` is referenced -anywhere else in the bundle**: esbuild keeps external imports for potential side -effects; the symbols themselves are fully tree-shaken. - -### Round 2 — one stub via alias - -Stub `stub-cloudflare-workers.ts` (empty `RpcTarget`, `WorkerEntrypoint`, -`DurableObject` classes) + `'--alias:cloudflare:workers=./stub-cloudflare-workers.ts'` -instead of the external. Result: `grep -c "cloudflare:" bundle.js` → **0**. -Exactly **one stub**, and it does not touch DOFS internals (its symbols are -dead code in the bundle). - -### Run under Deno - -`run-bundle.ts` constructs `SQLiteWorkspaceProvider` over a hand-rolled -duck-type of the (unexported) `Database` interface backed by -`node:sqlite` `DatabaseSync`: - -``` -$ deno run --allow-all approach2/run-bundle.ts -provider constructed: SQLiteWorkspaceProvider { supportsSymlinks: true, supportsWatch: true } -statSync without schema fails as expected: no such table: vfs_nodes -``` - -**Verdict: half-works, not viable as the sole source.** The bundle is -cloudflare:-free with a single trivial stub and loads under Deno — but -tree-shaking follows the *export surface*, and the published surface exposes -only 1 of the 4 needed symbols. `initializeSchema`'s body is not even present -in the bundle (only an error-message string mentioning it), so the schema can -never be created through this path; `Database` and `WorkspaceFilesystem` are -likewise unreachable. XMD would have to reimplement the Database wrapper and -the entire schema DDL from DOFS internals — at which point it owns the very -code it tried to reuse. Also drags in ~2.5k lines of capnweb RPC runtime the -DOFS layer doesn't need. Upgrades: `npm i` + re-bundle + re-verify stub -assumptions each release (the "unused import" fact must be re-proven per -version — brittle). MIT: the bundle embeds Cloudflare source, so the LICENSE -text must ship next to `bundle.js`. - -## Approach 3 — pinned vendored copy of `packages/dofs` - -### Build steps - -```sh -cp -R cf-computer/packages/dofs approach3/dofs -cp cf-computer/LICENSE approach3/dofs/LICENSE # root LICENSE, verbatim -# + VENDOR.txt (upstream URL, tag v0.1.1, commit 63d3636, list of local changes) -# package.json edits: drop "private", version 0.0.0-vendored.63d3636, -# license: MIT, files: [dist, LICENSE, VENDOR.txt, README.md], -# devDeps: typescript ^5.9.2 + @types/node (upstream wants typescript ^6 + -# workers-types + vitest — none needed for the build) -# tsconfig.vendored.json = upstream tsconfig.build.json merged with base, -# with types: ["node"] instead of [workers-types, vitest, node, pool-workers] -npm install --prefix approach3/dofs # 3 packages -approach3/dofs/node_modules/.bin/tsc -p tsconfig.vendored.json # clean, no edits to src/ -``` - -Source is fully self-contained: zero runtime deps, no `cloudflare:` imports, -storage abstracted behind its own `DurableObjectStorageLike` interface -(src/types.ts); the only Cloudflare types mentioned are in comments. -`src/testing.ts` (part of the `./testing` export) is backed by `node:sqlite` — -usable as-is under Deno. - -### Import-and-construct under Deno (direct dist path) - -``` -$ deno run --allow-all approach3/run-vendored.ts -schema initialized: version 5 (SCHEMA_VERSION = 5) -tables: _vfs_fetch_cursor, _vfs_mounts, _vfs_watermark, sqlite_sequence, vfs_blob_bytes, vfs_blobs, vfs_changes, vfs_chunks, vfs_dirents, vfs_manifests, vfs_meta, vfs_nodes -provider constructed: SQLiteWorkspaceProvider -WorkspaceFilesystem loaded: function -``` - -All four symbols reachable: `Database`, `initializeSchema`, -`SQLiteWorkspaceProvider` from the root export; `WorkspaceFilesystem` too -(root export, src/index.ts line 5). One API gotcha hit while consuming the -untyped dist: `initializeSchema(db, now)` requires the `now: () => number` -argument (first run: `TypeError: now is not a function` from -dist/schema/index.js line 45); the .d.ts files catch this in typed consumers. - -### Packaging variant (npm pack → tarball consumer) - -```sh -(cd approach3/dofs && npm pack --pack-destination ..) -# cloudflare-dofs-0.0.0-vendored.63d3636.tgz — 100 files -# consumer/package.json: "@cloudflare/dofs": "file:../cloudflare-dofs-....tgz" -npm install --prefix approach3/consumer # 1 package -deno run --allow-all --node-modules-dir=manual approach3/consumer/main.ts -# → bare-specifier import OK: SQLiteWorkspaceProvider function 5 -``` - -Friction: only the package.json fixes already listed (name kept as -`@cloudflare/dofs` so import specifiers match upstream; `private` removed; -`files` added). Bare specifiers `@cloudflare/dofs` and -`@cloudflare/dofs/testing` both resolve under Deno with a node_modules dir. - -### Upgrade consumption / owned surface - -Upgrade = re-copy `packages/dofs/src` from the new upstream tag, re-apply the -two scaffold files (package.json, tsconfig.vendored.json — src itself is -unmodified so `diff -r` against upstream is clean), rebuild, re-run XMD's -adapter tests. Owned surface: **48 source files, 6,554 LOC** (src minus -`*.test.ts`, `bench/`, `with-db.workers.ts`) — owned in the "must re-vendor and -re-verify" sense, not forked. MIT: LICENSE copy + provenance note as above. - -## Approach 4 — upstream package/export change - -### What the published dist shows (from Approach 1/2) - -A pure package.json `exports` addition is **not** sufficient today: the v0.1.1 -dist has no cloudflare:-free chunk containing DOFS (the layer is inlined into -`dist/index.js`, which imports `cloudflare:workers` at top level). A build -change is required — one new rolldown entry. - -### Mechanical proof (COPY of packages/computer, clone untouched) - -Copied `packages/{computer,dofs,rpc}` into `approach4/`. Added -`src/dofs-entry.ts` (`export * from "@cloudflare/dofs";`) and a probe rolldown -config = the original with `dofs: "src/dofs-entry.ts"` added to `input` -(entries trimmed to index+git+dofs; dts plugin dropped — it needs the monorepo -type env, irrelevant to the chunk-graph question; also added two alias lines -for `@cloudflare/computer-rpc/{client,debug}` that the trimmed install needed). - -``` -$ ./node_modules/.bin/rolldown -c rolldown.probe.config.ts -dofs.js 2.67 kB · shared-DtkNvqpY.js 107.79 kB · index.js 111.29 kB · git.js 102.42 kB ... -``` - -Rolldown hoists the entire DOFS layer into `shared-DtkNvqpY.js`; -`dofs.js` re-exports the **full** `@cloudflare/dofs` surface (Database, -initializeSchema, SQLiteWorkspaceProvider, WorkspaceFilesystem, -RecordingStorage, sync/*, …). `grep -c "cloudflare:"` per chunk in the dofs -graph: `dofs.js` 0, `shared-DtkNvqpY.js` 0 (its only imports are `node:crypto`, -`node:events`); `cloudflare:workers` remains confined to `index.js`. Run proof: - -``` -$ deno run --allow-all approach4/run-dofs-entry.ts -dofs entry under Deno OK: SQLiteWorkspaceProvider function function schema v5 -``` - -### Draft upstream diff (what we'd ask Cloudflare for) - -```diff ---- a/packages/computer/src/dofs.ts -+++ b/packages/computer/src/dofs.ts (new file) -@@ -0,0 +1,4 @@ -+// Runtime-neutral re-export of the DOFS layer. This entry's module -+// graph must stay free of cloudflare:workers so non-workerd runtimes -+// (Node, Deno, Bun) can consume the SQLite filesystem directly. -+export * from "@cloudflare/dofs"; - ---- a/packages/computer/rolldown.config.ts -+++ b/packages/computer/rolldown.config.ts -@@ export default defineConfig({ - input: { - index: "src/index.ts", -+ dofs: "src/dofs.ts", - git: "src/git/index.ts", - ---- a/packages/computer/package.json -+++ b/packages/computer/package.json -@@ "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - }, -+ "./dofs": { -+ "types": "./dist/dofs.d.ts", -+ "import": "./dist/dofs.js" -+ }, -``` - -(Optionally a CI guard: `grep -L cloudflare: dist/dofs.js` + its chunk -imports.) The simpler alternative ask — publish `@cloudflare/dofs` itself — -is equally proven: Approach 3 showed the package builds standalone with plain -tsc and zero runtime deps; upstream would only delete `"private": true` and add -`license`/`repository`/`files` fields. - -### Interim story - -Until Cloudflare accepts either change, ship Approach 3's vendored copy; the -vendored import surface (`@cloudflare/dofs`) is specifier-identical to the -published-package future, so the eventual migration is a dependency swap, not a -code change. (If they add `./dofs` instead of publishing the package, migration -is a one-line import-path change to `@cloudflare/computer/dofs`.) - -## Files in this probe - -- `approach1/` — npm consumer of the published package + 3 failing Deno import scripts -- `approach2/` — facade, stub, esbuild bundle, Deno runner -- `approach3/dofs/` — vendored build; `approach3/consumer/` — tarball consumer; `run-vendored.ts` -- `approach4/computer-copy/` — probe rolldown build (`dist-probe/`); `run-dofs-entry.ts` diff --git a/spikes/349-dofs/evidence/probes/slice3-5-fuse-linux.md b/spikes/349-dofs/evidence/probes/slice3-5-fuse-linux.md deleted file mode 100644 index 89782033..00000000 --- a/spikes/349-dofs/evidence/probes/slice3-5-fuse-linux.md +++ /dev/null @@ -1,263 +0,0 @@ -# probe-fuse-linux — real FUSE under Deno (issue #349 slices 3-5) - -Date: 2026-08-06. Host: macOS (Apple Silicon), Docker Desktop. -Container: `fuse-probe:latest` = node:22-slim (amd64 under Rosetta) + `fuse` + -`libfuse2` (2.9.9-6+b1) + procps + curl + unzip + Deno 2.9.1 (linux x64). -Run flags: `--platform linux/amd64 --device /dev/fuse --cap-add SYS_ADMIN ---security-opt apparmor:unconfined -v :/probe`. -Second container `qemuprobe2`: debian:stable-slim arm64 (native) + qemu-user + -fuse, same FUSE flags — used to run amd64 binaries under qemu-x86_64 instead of -Rosetta (differential, and because Rosetta cannot run deno-compiled binaries at -all; see Goal 5). - -Stack under test: `FileSQLiteStorage(file db, WAL)` → dofs `Database` → -`initializeSchema` → `SQLiteWorkspaceProvider` → prototype-splice onto -`@platformatic/vfs` `VirtualProvider` → `create(provider, { moduleHooks: -false })` + `EXTRA_VFS_METHODS` forwarding (re-implemented from computerd -`src/fuse/vfs.ts` in `vfs-wiring.mjs`) → upstream `driver.ts`/`options.ts` -compiled verbatim with tsc → `mountFuse` → `/mnt/ws`. - -## Per-goal verdicts - -| # | Goal | Verdict | -|---|------|---------| -| 1 | Deno loads fuse-native N-API addon | **YES — loads perfectly. But `mount()` SIGABRTs the whole Deno process** (`bad result in uv polyfill: 1`), uncatchable, on deno run AND deno compile, v2.9.1 AND v2.9.4, under Rosetta AND qemu. Node 22 control mounts fine. Deno cannot *serve* FUSE via fuse-native today. | -| 2 | Full-stack mount | **PASS under Node 22 sidecar** (the only viable host). fuseMount 57 ms, whole stack up 118-133 ms. SQLite-backed proven (no silent MemoryProvider). `writeFileRangesSync` confirmed NOT forwarded. | -| 3 | Bidirectional + subprocess | **PASS**, with a ≤1 s staleness window for API overwrites of kernel-cached files (auto_cache + attr_timeout=1). cd-prefix exec works from the FUSE host; the spawn-cwd variant deadlocks the entire mount (demonstrated + forensics + recovery runbook). | -| 4 | Durability matrix | **Release-only commit CONFIRMED and sharpened**: fsync is a durability no-op AND the inode doesn't exist in the db until release; write+close is only durable once the *async* RELEASE op lands (kill 0 ms after close() → LOST; +500 ms → committed). | -| 5 | Compiled executable | **Addon loading from `deno compile` binary: YES both ways** (embedded npm graph require, and `--include` → materialize → `process.dlopen`). Mount: same uv-polyfill abort. Binary 142 MB. Rosetta cannot execute deno-compiled binaries at all (even hello-world) — measured under qemu-user. | -| 6 | Sizes/latencies | Table below. | - ---- - -## Goal 1 — Deno + fuse-native N-API addon - -``` -docker exec fuseprobe sh -c 'cd /probe && deno run --allow-all g1-load.mjs' -``` -→ `import("fuse-native")` through node-gyp-build resolves the prebuild and -returns the Fuse class: `ok: true, ms: 297.14, typeofDefault: "function"`, -full errno table + `beforeMount/beforeUnmount/configure/...` statics present. - -``` -deno run --allow-all g1-dlopen.mjs # process.dlopen on prebuilds/linux-x64/node.napi.node -``` -→ `ok: true`, 70 exports (`fuse_native_mount`, 35 signal fns, 35 op ids). -No LD_LIBRARY_PATH needed in-container: the prebuild's only non-libc DT_NEEDED -is `libfuse.so.2`, satisfied by apt `libfuse2`. (`fuse-shared-library-linux` -also ships its own `libfuse/lib/libfuse.so`, 939,064 B, for hosts without it.) - -**But**: any actual `fuse.mount()` under Deno: - -``` -{"event":"constructing","mountPoint":"/mnt/ws"} -bad result in uv polyfill: 1 -Aborted # exit code 134 (SIGABRT) -``` - -- Reproduced: `deno run` 2.9.1, `deno run` 2.9.4, `deno compile` binary; under - Docker Rosetta and under qemu-x86_64 → not an emulation artifact. -- Node 22.23.2 control with the identical script (`g2-min.mjs`): mounts in - 27 ms, `cat` works, clean unmount, exit 0. -- Attribution: the message comes from `assert_ok` in Deno `ext/napi/uv.rs` — - it `std::process::abort()`s whenever a polyfilled uv init fn returns - non-zero (`uv_mutex_init`, `uv_async_init`, ...). Deno's `uv_default_loop` - polyfill **returns a null pointer**; fuse-native's `fuse_native_mount` uses - `uv_default_loop` + `uv_async_init` + `uv_mutex_init` + `uv_sem_init/wait/post` - (symbol scrape of the prebuild). A null loop into `uv_async_init` is the - prime suspect for the non-zero result. Upstream issue material: minimal - repro is `g2-min.mjs`; abort is pre-kernel (happens with or without a - usable /dev/fuse once fusermount succeeds; with fusermount missing the - mount fails earlier with a *catchable* "fuse: failed to exec fusermount"). - -## Goal 2 — full-stack mount (Node 22 sidecar) - -``` -node mount-host.mjs /probe/data/ws.db /mnt/ws -{"event":"mounted","runtime":"node v22.23.2", ... - "sqliteProof":{"tables":["_vfs_fetch_cursor","_vfs_mounts","_vfs_watermark","sqlite_sequence", - "vfs_blob_bytes","vfs_blobs","vfs_changes","vfs_chunks","vfs_dirents","vfs_manifests", - "vfs_meta","vfs_nodes"],"proofFileFoundInTable":"vfs_dirents"}, - "forwarding":{"linkSync":"function","createFileSync":"function","writeRangeSync":"function", - "truncateFileSync":"function","chmodSync":"function","readRangeSync":"function", - "openWriteBufferSync":"function","openWriteBufferForCreateSync":"function", - "releaseWriteBufferSync":"function","writeFileRangesSync":"undefined"}, - "timingsMs":{"vfsUp":57.73,"driverRequire":3.52,"fuseMount":56.59,"total":117.85}} -mount | grep fuse -/dev/fuse on /mnt/ws type fuse (rw,nosuid,nodev,user_id=0,group_id=0,max_read=524288) -``` - -- SQLite proof: vfs-facade write found in `vfs_dirents` via a **fresh** - `node:sqlite` connection on the db file → prototype splice worked, no silent - MemoryProvider fallback. -- `max_read=524288` in the mount table → the `_fuseOptions()` monkeypatch - (options.ts profile) reached the kernel. -- **`writeFileRangesSync`: `undefined` at the driver boundary — confirmed.** - driver.ts probes it but vfs.ts's EXTRA_VFS_METHODS omits it, so the ranged - spill path is dead code. Harmless in practice: all direct-write probes - (`createFileSync`/`writeRangeSync`/`truncateFileSync` + buffered-write trio) - are forwarded, so `hasDirectWrites`/`hasBufferedWrites`/`hasDeferredCreate` - are all true and `flushEntry` (the only `writeFileRangesSync` consumer) is - bypassed for driver-created files. -- Gotcha found live: `makeFUSEOps(vfs, mountPoint)` uses the **mountPoint as - the VFS namespace prefix** (kernel `/` → vfs `/mnt/ws`). The dir chain for - the mountPoint must pre-exist in the vfs or every op — including getattr of - the mount root — is ENOENT (`ls: cannot access '/mnt/ws'` while the mount - table shows the mount). Fix: mkdir the chain through the vfs before mounting. -- node:sqlite on Node 22.23: works, prints `ExperimentalWarning: SQLite is an - experimental feature`. - -## Goal 3 — bidirectional + subprocess - -| Test | Result | -|---|---| -| API write → `cat` via mount, new file | immediate (`from-api` readable right after `/write` returns; negative_timeout=0) | -| shell `echo > /mnt/ws/sub.txt` → API read | immediate (`from-shell` visible via WorkspaceFilesystem right after the redirect closes: close→flush… commit landed before the next process turn) | -| API **overwrite** of a file the kernel already cached | **stale ≤ ~1 s**: immediate `cat` returned old content, `cat` after 1.2 s returned new (auto_cache + attr_timeout=1) | -| exec with cwd inside mount, cd-prefix technique (from FUSE host itself) | works: `/exec?cwd=/mnt/ws&cmd=pwd && echo…` → stdout `/mnt/ws`, file visible via API in 0.21 ms | -| create / overwrite / `dd seek=100` / `truncate -s 10` / `mv` / `ln -s` (+readlink+read-through) / `mkdir -p` / `find` / `rm` | all pass (sparse.bin 104 B, ops.txt truncated to `v2-overwri`) | -| 2 × 200 concurrent `cat` loops | no failures | - -### spawn-cwd-inside-mount deadlock (upstream runner.ts rationale) — demonstrated - -`/exec-cwd?cwd=/mnt/ws` (execFile with `cwd:` option from the FUSE-serving -process) wedged the entire mount. Forensic snapshot (`ps -eo pid,stat,wchan`): - -``` -1178 Sl pipe_read node mount-host.mjs … <- parent: event loop blocked in uv_spawn's exec-report pipe read -1313 S request_wait_answer node mount-host.mjs … <- forked child, pre-exec chdir into own mount, waiting on a FUSE answer only the blocked parent can serve -1315 D request_wait_answer ls /mnt/ws <- every other client: uninterruptible sleep, TERM-immune -``` - -- `timeout(1)`'s SIGTERM cannot kill the D-state readers; the docker execs hung. -- **SIGKILL of the host is NOT full recovery**: the forked child inherited the - `/dev/fuse` fd, so the connection (and the wedge) outlived the host; - `fusermount -u` → "Device or resource busy"; `fusermount -u -z` detached the - mount table entry but left the D-state processes. -- Actual recovery: `mount -t fusectl fusectl /sys/fs/fuse/connections` then - `echo 1 > /sys/fs/fuse/connections//abort` → all wedged processes - released instantly. This is the runbook the spike should ship. - -## Goal 4 — durability matrix - -All checks by a separate process (`deno run -A probe.mjs read `, -fresh `FileSQLiteStorage` + `Database` on the db file). FUSE host: Node. - -| # | Scenario | Result | Evidence | -|---|----------|--------|----------| -| a | write+close through mount → fresh-process read | **COMMITTED** | `a-file.txt` → `"a-committed\n"` | -| b | open, write, `fsync(fd)`, **no close**, kill -9 host | **LOST** — and the inode never existed in the db even *before* the kill (separate-process read: `no such file` while held open; same-process API *did* see the bytes via the in-process write buffer — deceptive!). fsync/flush are durability no-ops; `openWriteBufferForCreateSync` defers the INSERT to release. | `b-file.txt` | -| c | write+close, then kill -9 | **COMMITTED — IF the async RELEASE op landed.** close(2) returns after FLUSH; RELEASE is async. kill -9 **0 ms** after close → **LOST** (`race0.txt`, also `term.txt` with SIGTERM); kill **500 ms** after close → committed (`race500.txt`). | race harness | -| d | SIGTERM / SIGKILL mid-mount | Both: `auto_unmount` removes the mount within ~1 s; `ls /mnt/ws` on the (empty) underlying dir returns exit 0 — **no hang, no "transport endpoint is not connected"**. Caveat: not true when a forked child holds the fuse fd (see deadlock above). No unmount-time flush → WAL left. | mount table empty after each | -| e | second Deno process opens same db while mounted | **Works, read AND write** (WAL): `probe.mjs write` from a second process committed in 45 ms, immediately visible through the mount and the host API. No SQLITE_BUSY observed (short transactions). | `second-writer.txt` | -| f | kill -9 leaves `-wal`(1.1 MB)/`-shm`; next open? | **Recovers cleanly**: fresh open reads meta (rev 25, schema 5, journal_mode wal); after clean close the WAL is checkpointed (db 4 KB→106 KB, sidecars gone). | `probe.mjs meta` | - -Net durability contract for the spike: **a write is durable only after -open-count→0 release AND its dofs transaction commit; close() returning is -not a durability barrier, and fsync lies.** A sync point must observe the db -(or drain releases), not the POSIX fd lifecycle. - -## Goal 5 — `deno compile` binary - -``` -deno compile --allow-all --no-check \ - --include node_modules/fuse-native/prebuilds/linux-x64/node.napi.node \ - --include node_modules/fuse-shared-library-linux/libfuse/lib/libfuse.so \ - --output g5-bin g5-compiled.mjs # Files: 32.28MB → binary 142,191,712 B -``` - -- **Environment landmine**: under Docker Desktop **Rosetta**, deno-compiled - binaries do not run at all — `Inconsistency detected by ld.so: rtld.c: 1293: - rtld_setup_main_map: Assertion 'GL(dl_rtld_map).l_libname' failed!` (and a - hello-world compile segfaults, exit 139). Plain `deno` runs fine. Workaround - used: run the amd64 binary in a native arm64 container via `qemu-x86_64 -L - ` (sysroot = container's /lib64 + /lib/x86_64-linux-gnu with - ld.so symlink dereferenced, else qemu resolves the symlink against the wrong - root). -- **Step A — embedded npm graph**: static `import Fuse from "fuse-native"` in - the compiled binary **works**: the addon loads from the embedded graph, - `typeof Fuse === "function"` at 0.16 ms into main. -- **Step B — `--include` + materialize + dlopen**: reading the two included - files out of the binary VFS (`Deno.readFile(new URL(...))` → 68,072 B addon, - 939,064 B libfuse), writing to `Deno.makeTempDir()`, `process.dlopen` on the - real-path copy: **works**, 70 exports, 34.7 ms into main. -- **Mount from the compiled binary**: with fusermount absent → catchable - `Error: fuse failed` (never reached the uv path). With fusermount + /dev/fuse - present → same `bad result in uv polyfill: 1` abort as `deno run`. -- Cold start: 1205 ms wall under qemu-user emulation for start → both load - steps done (includes qemu startup; not representative of native — expect - well under 200 ms native given `deno run` import was 297 ms un-warmed). - -## Goal 6 — sizes & latencies - -All timings under emulation (Rosetta for fuseprobe, qemu for qemuprobe2); -treat as relative, not absolute. - -| Metric | Value | -|---|---| -| Image fuse-probe (node:22-slim + fuse + deno) | 503 MB | -| fuse-native package (with prebuilds) | 1.6 MB; addon `node.napi.node` 68,072 B | -| fuse-shared-library-linux | 3.1 MB; `libfuse.so` 939,064 B | -| node_modules total (incl. typescript+@types) | 34 MB | -| deno-compiled binary (2 includes) | 142 MB (embedded files 32.28 MB) | -| Deno `import("fuse-native")` | 297 ms | -| Full stack cold → mounted (Node) | 118-133 ms (vfsUp 58-72, require 3-4, fuseMount 57-58) | -| Minimal fuse-native mount (Node) | 27 ms | -| 100 small writes THROUGH mount | 225 ms → **2.25 ms/op** | -| 100 small writes direct dofs API (same process) | 110 ms → **1.10 ms/op** (mount overhead ≈ 2.0×) | -| 100 small reads through mount | 101 ms → 1.01 ms/op | -| Separate-process Deno probe (cold start + db open + 1 op) | 24-45 ms script time | - -## Compile & environment fights (verbatim) - -1. tsc on the upstream driver: **zero fights.** `driver.ts`, `options.ts`, - `tracer.ts`, `backend.ts` compiled unmodified, first try, with computerd's - own settings (`module nodenext`, CJS via package.json, `skipLibCheck`, - `@types/node`). Sole accommodation: a hand-written `src/vfs.ts` type shim - (driver.ts's import of it is type-only; the real vfs.ts drags in - `@cloudflare/computer-rpc`). `fuse-native.d.ts` used as-is beside driver.ts. -2. `WorkspaceFsError: path exists: /workspace` — my wiring double-mkdir'd; - guard with existsSync. (Proved en route that the facade dispatches to - SQLiteWorkspaceProvider.) -3. `ls: cannot access '/mnt/ws': No such file or directory` *while mounted* — - the mountPoint-as-vfs-prefix gotcha (Goal 2 above). -4. `sh: 1: time: not found` (slim image) — use `date +%s%N`. -5. `qemu-x86_64: Could not open '/lib64/ld-linux-x86-64.so.2'` — twice: once - for missing sysroot (`-L`), once because `cp -a` preserved the ld.so - symlink whose absolute target qemu resolved against the arm64 root; fix - `cp -L`. -6. `pgrep -f "node mount-host.mjs"` matches the `sh -c` wrapper and the exec's - own shell → killed the wrong processes twice (one `docker exec` died exit - 137). Match on the concrete node child instead. -7. After the fusectl mass-abort + ~15 zombies (pid 1 is `sleep infinity`, - never reaps), `docker exec` on fuseprobe began failing (rc 255, empty) with - the container still "running" — final size measurements taken from the host - side of the bind mount instead. - -## What the committed spike must replicate - -1. **Topology**: FUSE service must live in a **Node sidecar process**; - Deno cannot host fuse-native until the ext/napi uv polyfill gap is fixed - (file upstream with the `g2-min.mjs` repro + `assert_ok`/null - `uv_default_loop` attribution). Everything else — dofs API side, probes, - supervisors — runs fine under Deno, and matrix (e) proves the shared-WAL - two-process design: Deno main + Node FUSE sidecar over one db file works - with immediate cross-visibility. -2. **The vfs wiring invariants**: prototype splice before `create()` (verify - with a fresh-connection read of `vfs_dirents`, or you may be silently on - MemoryProvider); forward the nine methods; pre-create the mountPoint dir - chain in the vfs namespace. Decide whether to also forward - `writeFileRangesSync` (today it's dead code by omission). -3. **Durability rules**: commit is at release only; `fsync`/`close` are not - barriers; a durability-sensitive caller must wait for release + commit - (observe the db) — and any "write then kill" path (worker recycling!) needs - ≥ the close→release async window or an explicit drain. -4. **Exec rules**: `cd &&` prefix, never `cwd:` into the own mount; ship - the recovery runbook (`fusermount -z`, then fusectl `abort` — SIGKILL alone - leaves an fd-inherited wedge if any child was mid-spawn). -5. **Ops expectations**: ≤1 s staleness for API overwrites under the - production option profile; auto_unmount reliably clears the mount on - host death; WAL recovery on next open is clean. -6. **Packaging**: `deno compile` CAN carry and load the addon (both embedded - npm graph and `--include`+dlopen) — the blocker is only the mount call, not - distribution. Don't test deno-compiled binaries under Docker Rosetta. diff --git a/spikes/349-dofs/evidence/probes/slice6-shim.md b/spikes/349-dofs/evidence/probes/slice6-shim.md deleted file mode 100644 index 1dbfccb8..00000000 --- a/spikes/349-dofs/evidence/probes/slice6-shim.md +++ /dev/null @@ -1,177 +0,0 @@ -# Probe: issue #349 slice 6 — userspace shim (cf-computer `mountShim`) under Deno - -Host: darwin-arm64 (Darwin 25.5.0), Deno 2.9.1, Node v-homebrew, 2026-08-06. -Working dir: `scratchpad/probe-shim`. Reuses `scratchpad/probe-dofs` (built dofs dist + -`file-storage.mjs` FileSQLiteStorage adapter). cf-computer v0.1.1 clone read-only. - -## Setup / provenance - -- `src/shim/shim.ts` — byte-identical copy of `cf-computer/packages/computerd/src/shim/shim.ts` - (shasum `05e6a2ac…` matches). Compiled with `npx tsc` (typescript from npm, target ES2022, - module NodeNext, ESM output) → `dist/shim/shim.js`. **Compilation did not fight: zero errors, - zero edits to upstream source.** -- `src/fuse/vfs.ts` — type-only stub (`export type { VirtualFileSystem as NodeVirtualFileSystem } - from "@platformatic/vfs"`), matching upstream vfs.ts's type alias. shim.ts imports only the type. -- `wiring.mjs` — hand-port of upstream `fuse/vfs.ts` runtime wiring. Deviations (full list): - 1. RPC surface dropped (SyncRPC / pullOnce / tick / startSyncLoop — not needed, imports - unavailable). - 2. `SQLiteTestStorage` (in-memory) → `FileSQLiteStorage(dbPath)` (probe-dofs adapter, WAL). - 3. Provider options pass-through (upstream hardcodes defaults). - 4. ESM instead of CommonJS. - `ensureVirtualProviderPrototype` (the prototype splice) and the `EXTRA_VFS_METHODS` - defineProperty loop are ported verbatim. -- Blueprint correction: `EXTRA_VFS_METHODS` in v0.1.1 has **nine** entries, not ten - (linkSync, createFileSync, writeRangeSync, truncateFileSync, chmodSync, readRangeSync, - openWriteBufferSync, openWriteBufferForCreateSync, releaseWriteBufferSync). -- Confirmed the silent-fallback trap in `@platformatic/vfs@0.4.0` `index.js` `create()`: - a provider that fails `instanceof VirtualProvider` and is a plain object is **reassigned as - the options argument** and provider becomes undefined → `new VirtualFileSystem(undefined, …)` - → `provider ?? new MemoryProvider()`. No error, no warning. - -Deps installed in probe-shim: `npm i @platformatic/vfs@0.4.0 typescript @types/node` (clean). - -## Q1 — does the shim run under Deno at all? - -Command: `deno run -A exp1.mjs` - -Result: **yes, first try, zero runtime failures.** No error output at any point. - -- `vfs.provider instanceof MemoryProvider` → false; `vfs.provider === provider` → true; - constructor name `SQLiteWorkspaceProvider`. -- Cross-process verification: wrote `/verify/marker.txt` via the vfs facade, read it back with - a **separate Deno process** opening a fresh Database over the same db file - (`probe-dofs/probe.mjs read`) → `{"op":"read","body":"sqlite-not-memory"}`. Not a - MemoryProvider. -- `mountShim({ vfs, mountPoint })` (default pollIntervalMs 250) returned; watchAsync loop, - setInterval poll, and node:crypto sha1 all work under Deno. -- First-shot latencies: VFS→disk 106 ms (provider's default 100 ms rev-poll watcher), - disk→VFS 149 ms (within the 250 ms reconcile poll). - -## Q2 — bidirectional visibility + latency - -Command: `deno run -A exp2.mjs` (defaults: provider watchIntervalMs 100, shim pollIntervalMs 250; -2 ms detection poll; ~120–130 ms decorrelation sleep between rounds). - -| direction | rounds (ms) | median (ms) | bound | -|---|---|---|---| -| VFS write → visible on disk | 104.7, 82.6, 81.0, 81.1, 78.6, 80.6, 81.6, 78.3, 78.9, 80.2 | **80.8** | provider rev-poll (100 ms) | -| disk write (separate `/bin/sh` process) → readable via vfs API | 205.7, 105.4, 106.4, 109.2, 107.5, 110.4, 104.3, 109.8, 110.3, 105.6 | **108.3** | shim poll (250 ms), phase-dependent | -| disk write → visible via fresh Database over the db file | 207.8, 106.7, 107.0, 110.4, 108.4, 111.7, 105.2, 111.3, 111.3, 106.8 | **109.4** | ≈ vfs visibility + ~1 ms (writes commit synchronously) | - -Both directions are phase-dependent polling: worst case ≈ interval + walk cost. db-visibility -tracks vfs-visibility within ~1 ms because the reconcile's `vfs.writeFileSync` commits to SQLite -synchronously (fresh `FileSQLiteStorage` connection per check, WAL). - -Native subprocess round-trip (`/bin/sh -c 'echo hi > sub.txt && cat other.txt'`, cwd = mount dir, -after `vfs.writeFileSync(other.txt)` + `shim.flush()`): success — `cat` printed the API-written -content, and `sub.txt` appeared in the VFS 88.5 ms later. The upstream flush()-before-exec -contract works as documented. - -## Q3 — restart persistence - -Upstream context: computerd's mountPoint is a **stable configured path** -(`MOUNT_POINT` env / `DEFAULT_MOUNT_POINT`, cli/computerd.ts:457,499) — the VFS namespace embeds -it, so restart re-mounts at the same path. - -- Phase A (`exp3a.mjs`): fresh db, 100 files × 100 KB = 10,000,000 bytes across 10 dirs written - via vfs (56 ms), `mountShim` boot materialisation 77 ms, disk tree verified (100 files/10 MB), - unmount, close, **delete the whole on-disk mount dir**. -- Phase B (`exp3b.mjs`, new process): reopen db — frontier intact (100 files / 10,000,000 bytes - readable via vfs before any mount). Re-mount at the same now-empty path: - **materialisation of ~100 files / 10 MB took 78.9 ms**; disk tree reproduced exactly - (100 files, 10 MB, 5/5 sampled sha1s match db content). `initializeSchema` on reopen does not - wipe (idempotent). -- Trap demonstrated: mounting the same vfs at a **different** fresh mkdtemp path materialises - **nothing** (`[]`) — VFS paths embed the mount prefix. A committed spike must keep the mount - path stable across restarts (or migrate paths in the db). - -## Q4 — documented limitations, demonstrated (`deno run -A exp4.mjs`) - -| probe | result | -|---|---| -| symlink via vfs API (live target) | stays a symlink **in** the VFS (`lstatSync` → symlink); on disk it materialises as a **regular file containing the target's content** (`safeVfsStat` uses `statSync`, which follows the link) — not "absent" as naively expected, but the link-ness is lost | -| symlink via vfs API (dangling) | **absent on disk** (`statSync` throws → entry skipped) | -| symlink created on disk (live, `ln -s`) | reconciled into the VFS as a **regular file** with duplicated content (`walkDisk` uses `fsStat`, follows) | -| symlink created on disk (dangling) | **ignored** — never enters the VFS (ENOENT) | -| `chmod 755` on a synced disk file | **not reflected**: vfs mode stays 644 while disk shows 755; the shadow diff keys on (size, mtime) only | -| conflict, both sides write same path in one poll window (3 rounds, both orders) | **VFS wins 3/3** — both sides converge to the vfs-side content, including when the vfs write happened *first* and the disk write *last* | -| 50 MB file, new, disk→VFS | 283 ms | -| 50 MB file, 1-byte change, disk→VFS | **594 ms** (full re-read + sha1 + full SQLite rewrite) | -| 50 MB file, 1-byte change, VFS→disk | **654 ms** (full read from SQLite + full disk write) | -| 1 KB file, new / change, disk→VFS | 56 ms / 48 ms | -| steady-state `reconcileNow()` with unchanged 50 MB present | 0.7–2.2 ms (shadow (size, mtime) short-circuit works) | - -## Q5 — kill -9 mid-activity (`deno run -A exp5.mjs`) - -Worker (`exp5-worker.mjs`) mounted the shim and wrote on BOTH sides every 5 ms -(20 vfs-side + 20 disk-side files, round-robin). Parent SIGKILLed it at iteration 200. - -- Post-mortem: WAL file 4,169,472 bytes pending; fresh connection `PRAGMA integrity_check` → ok - (standard SQLite WAL crash recovery — no corruption). -- Divergence at death: 40 files present on both sides, **29 of 40 content-diverged** - (writes in flight across the two poll seams). -- Restart + re-mount into the SAME dir: boot `materialiseVfsToDisk` overwrites disk with VFS - content, first poll ticks pull disk-side extras; after 1.2 s **disk and VFS fully converged, - 0 mismatches**. -- Loss window (by design, demonstrated by the convergence direction): disk-side writes not yet - polled into the VFS at crash time are **overwritten by the VFS copy on next boot** — up to - `pollIntervalMs` (250 ms) of external writes can be silently lost across a crash. VFS-side - writes are never lost (committed to SQLite synchronously before the watcher even fires). - -## Q6 — darwin-arm64 real-FUSE record (`fuse-probe/`) - -Host: darwin-arm64 (Apple Silicon), Darwin 25.5.0. - -- `/Library/Filesystems/macfuse.fs` — **does not exist**. -- `/Library/Filesystems/osxfuse.fs` — exists (stale osxfuse 3.11.2 remnant) but has **no - `configured` marker**, so fuse-shared-library treats it as unconfigured. -- `npm i fuse-native@2.2.6` → installs in 3 s, **no darwin-arm64 prebuild** (`prebuilds/` ships - only `darwin-x64` and `linux-x64`). npm's allow-scripts policy on this host additionally - blocked the `node-gyp-build` install script (`npm warn allow-scripts fuse-native@2.2.6`). -- Running `npx node-gyp-build` manually: **compiles from source successfully** (exit 0) — - produces an arm64 `build/Release/fuse.node` linking only `/usr/lib/libc++.1.dylib` and - `libSystem.B.dylib`. -- Loading: `require("fuse-native")` succeeds under **both Node and Deno** (createRequire). - `Fuse.isConfigured` → **false**. -- The bundled userspace library `fuse-shared-library-darwin/osxfuse/libosxfuse.dylib` is a - ppc_7400/ppc64/i386/x86_64 universal binary — **no arm64 slice**. Its `configure()` path - untars an osxfuse 3.x bundle (kext-based; Apple Silicon requires reduced-security mode and - macFUSE ≥ 4, so this can never work on this platform). -- Actual `fuse.mount()` attempt: **SIGSEGV (exit 139), no error surfaced**, identically under - Node and Deno. -- Verdict for macOS-arm64: real FUSE via cf-computer's stack is a dead end — even with macFUSE - installed, fuse-native 2.2.6's darwin support predates Apple Silicon. The shim is the only - viable path on this platform. (macFUSE was NOT installed, per instructions.) - -## Q7 — verdict - -**Development-only fallback — suitable as exactly that, and it is the ONLY option on -darwin-arm64.** - -For it (measured): -- Runs under Deno unmodified: upstream shim.ts compiled with zero edits, zero runtime failures. -- Latency is fine for interactive/dev use: ~81 ms VFS→disk, ~108 ms disk→VFS medians; - subprocesses inside the mount see API-written files (after `flush()`) and their writes land - back in ~90 ms. -- Restart persistence is real: SQLite frontier survives close + dir deletion; 100 files / 10 MB - re-materialise in ~79 ms at the same mount path. -- Crash-safe at the storage layer: SIGKILL mid-activity → WAL recovery ok, remount converges - fully in ~1 s. - -Against production use (demonstrated, not speculative): -- Up to one poll window (250 ms) of **external disk writes silently lost** on crash, and VFS - unconditionally clobbers concurrent disk writes (3/3 conflicts, both orders). -- Symlinks are silently degraded to content copies (or dropped when dangling) in BOTH - directions; chmod invisible; no watch fan-out. -- Large files pay full re-read + full rewrite per change: ~0.6 s per touch of a 50 MB file on - either side; scales linearly with size, not delta. -- The VFS namespace embeds the absolute mount path: a moved/renamed mount dir orphans the - entire tree (materialises nothing). -- Everything is polling (provider rev-poll 100 ms + shim walk 250 ms): steady-state cost is - fine (~1–2 ms/tick at this tree size) but grows with tree breadth, and correctness windows - are timing-defined, not event-defined. - -What a committed spike must replicate: the prototype splice + MemoryProvider-fallback -verification (the failure is SILENT), a stable absolute mount path, `flush()` before any exec -inside the mount, and the FileSQLiteStorage(WAL) adapter; and it must NOT rely on symlinks, -permissions, or sub-poll-window external-write durability. diff --git a/spikes/349-dofs/host/file-storage.ts b/spikes/349-dofs/host/file-storage.ts deleted file mode 100644 index d073d352..00000000 --- a/spikes/349-dofs/host/file-storage.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { DatabaseSync, type StatementSync } from "node:sqlite"; -// @ts-types="./types/dofs.d.ts" -import type { - DurableObjectStorageLike, - SQLCursorLike, -} from "@cloudflare/dofs"; - -class Cursor implements SQLCursorLike { - #rows: Row[]; - constructor(rows: Row[]) { - this.#rows = rows; - } - toArray(): Row[] { - return this.#rows; - } -} - -// File-backed DurableObjectStorageLike over node:sqlite. Mirrors dofs's own -// SQLiteTestStorage (vendor/dofs/src/testing.ts) — prepared-statement cache, -// binding normalization, BEGIN/COMMIT/ROLLBACK — but opens a real database -// file, which is the entire difference between a unit-test fixture and a -// persistent local workspace. -export class FileSQLiteStorage implements DurableObjectStorageLike { - #db: DatabaseSync; - #cache = new Map(); - readonly sql: { - exec: ( - query: string, - ...bindings: unknown[] - ) => SQLCursorLike; - }; - - constructor(path: string) { - this.#db = new DatabaseSync(path); - this.#db.exec("PRAGMA journal_mode = WAL"); - this.#db.exec("PRAGMA foreign_keys = ON"); - this.#db.exec("PRAGMA synchronous = NORMAL"); - this.sql = { - exec: ( - query: string, - ...bindings: unknown[] - ): SQLCursorLike => { - let statement = this.#cache.get(query); - if (statement === undefined) { - statement = this.#db.prepare(query); - this.#cache.set(query, statement); - } - const normalized = bindings.map(toSQLiteValue); - const rows: Row[] = []; - for (const row of statement.all(...normalized)) { - if (isRow(row)) { - rows.push(row); - } - } - return new Cursor(rows); - }, - }; - } - - transactionSync(closure: () => T): T { - this.#db.exec("BEGIN"); - try { - const result = closure(); - this.#db.exec("COMMIT"); - return result; - } catch (error) { - this.#db.exec("ROLLBACK"); - throw error; - } - } - - close(): void { - this.#cache.clear(); - this.#db.close(); - } -} - -type SQLiteValue = null | number | bigint | string | Uint8Array; - -function toSQLiteValue(value: unknown): SQLiteValue { - if (value === undefined || value === null) { - return null; - } - if (typeof value === "boolean") { - return value ? 1 : 0; - } - if (value instanceof Uint8Array) { - return value; - } - if ( - typeof value === "string" || typeof value === "number" || - typeof value === "bigint" - ) { - return value; - } - throw new TypeError( - `FileSQLiteStorage cannot bind value of type ${typeof value}`, - ); -} - -function isRow(value: unknown): value is Row { - return typeof value === "object" && value !== null; -} diff --git a/spikes/349-dofs/host/main.ts b/spikes/349-dofs/host/main.ts deleted file mode 100644 index 359531e6..00000000 --- a/spikes/349-dofs/host/main.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { main, until } from "effection"; -// @ts-types="./types/dofs.d.ts" -import { - Database, - initializeSchema, - SCHEMA_VERSION, - WorkspaceFilesystem, -} from "@cloudflare/dofs"; -// @ts-types="./types/dofs-rename.d.ts" -import { rename } from "@cloudflare/dofs/fs/rename"; -import { FileSQLiteStorage } from "./file-storage.ts"; - -function usage(): never { - console.error( - [ - "usage: proof [args...]", - "ops: init | mkdir

| write

| read

| ls

| rm

[-r]", - " stat

| lstat

| rename | symlink ", - " readlink

| meta", - ].join("\n"), - ); - Deno.exit(2); -} - -main(function* () { - const started = performance.now(); - const [dbPath, op, ...args] = Deno.args; - if (dbPath === undefined || op === undefined) { - usage(); - } - - const storage = new FileSQLiteStorage(dbPath); - const db = new Database(storage); - initializeSchema(db, Date.now); - const fs = new WorkspaceFilesystem(db); - - let result: Record; - switch (op) { - case "init": - result = { schemaVersion: SCHEMA_VERSION }; - break; - case "mkdir": - yield* until(fs.mkdir(args[0], { recursive: true })); - result = { ok: true }; - break; - case "write": - yield* until(fs.writeFile(args[0], args[1])); - result = { ok: true }; - break; - case "read": - result = { body: yield* until(fs.readFile(args[0], "utf8")) }; - break; - case "ls": { - const entries = yield* until(fs.readdir(args[0])); - result = { entries: entries.map((entry) => entry.name).sort() }; - break; - } - case "rm": - yield* until(fs.rm(args[0], { recursive: args.includes("-r") })); - result = { ok: true }; - break; - case "stat": - result = { stat: yield* until(fs.stat(args[0])) }; - break; - case "lstat": - result = { lstat: yield* until(fs.lstat(args[0])) }; - break; - case "rename": - rename(db, args[0], args[1]); - result = { ok: true }; - break; - case "symlink": - yield* until(fs.symlink(args[0], args[1])); - result = { ok: true }; - break; - case "readlink": - result = { target: yield* until(fs.readlink(args[0])) }; - break; - case "meta": - result = { - binarySchemaVersion: SCHEMA_VERSION, - vfsMeta: db.all("SELECT k, v FROM vfs_meta ORDER BY k"), - }; - break; - default: - usage(); - } - - storage.close(); - const ms = Math.round((performance.now() - started) * 100) / 100; - console.log(JSON.stringify({ op, ...result, ms })); -}); diff --git a/spikes/349-dofs/host/shim-main.ts b/spikes/349-dofs/host/shim-main.ts deleted file mode 100644 index 3e5c21cc..00000000 --- a/spikes/349-dofs/host/shim-main.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { main, until } from "effection"; -import { exec } from "@effectionx/process"; -// @ts-types="./types/computerd-shim.d.ts" -import { mountShim } from "@xmd-spike/computerd-shim"; -import { createFileBackedVfs } from "./vfs-wiring.ts"; - -function usage(): never { - console.error( - [ - "usage: proof-shim exec ", - " proof-shim materialize", - "", - "The mount directory is part of the workspace namespace: the same", - "database must always be mounted at the same absolute path.", - ].join("\n"), - ); - Deno.exit(2); -} - -main(function* () { - const [dbPath, mountDir, command, ...rest] = Deno.args; - if (dbPath === undefined || mountDir === undefined) { - usage(); - } - if (command !== "exec" && command !== "materialize") { - usage(); - } - - const started = performance.now(); - const wired = createFileBackedVfs(dbPath); - wired.vfs.mkdirSync(mountDir, { recursive: true }); - Deno.mkdirSync(mountDir, { recursive: true }); - - const shim = yield* until( - mountShim({ vfs: wired.vfs, mountPoint: mountDir }), - ); - const mountedMs = Math.round(performance.now() - started); - - let payload: Record = {}; - if (command === "exec") { - const shellCommand = rest.join(" "); - if (shellCommand === "") { - usage(); - } - yield* until(shim.flush()); - // cwd stays outside the mount: upstream computerd prefixes `cd` instead - // of passing cwd because fork+chdir into a mount served by the same - // process deadlocks under FUSE; the shim keeps the same shape. - const result = yield* exec("/bin/sh", { - arguments: ["-c", `cd ${mountDir} && (${shellCommand})`], - }).join(); - yield* until(shim.reconcileNow()); - payload = { - code: result.code, - stdout: result.stdout, - stderr: result.stderr, - }; - } - - yield* until(shim.unmount()); - wired.storage.close(); - console.log(JSON.stringify({ command, mountedMs, ...payload })); -}); diff --git a/spikes/349-dofs/host/types/computerd-shim.d.ts b/spikes/349-dofs/host/types/computerd-shim.d.ts deleted file mode 100644 index 10e82764..00000000 --- a/spikes/349-dofs/host/types/computerd-shim.d.ts +++ /dev/null @@ -1,16 +0,0 @@ -// Facade for the vendored shim's consumed surface; mirrors -// vendor/computerd-shim/dist/shim/shim.d.ts. - -import type { VirtualFileSystem } from "./platformatic-vfs.d.ts"; - -export interface ShimMount { - unmount(): Promise; - flush(): Promise; - reconcileNow(): Promise; -} - -export declare function mountShim(options: { - vfs: VirtualFileSystem; - mountPoint: string; - pollIntervalMs?: number; -}): Promise; diff --git a/spikes/349-dofs/host/types/dofs-rename.d.ts b/spikes/349-dofs/host/types/dofs-rename.d.ts deleted file mode 100644 index 248d6ec1..00000000 --- a/spikes/349-dofs/host/types/dofs-rename.d.ts +++ /dev/null @@ -1,9 +0,0 @@ -// Facade counterpart of vendor/dofs/dist/fs/rename.d.ts; see dofs.d.ts. - -import type { Database } from "./dofs.d.ts"; - -export declare function rename( - db: Database, - oldPath: string, - newPath: string, -): void; diff --git a/spikes/349-dofs/host/types/dofs.d.ts b/spikes/349-dofs/host/types/dofs.d.ts deleted file mode 100644 index eee36a05..00000000 --- a/spikes/349-dofs/host/types/dofs.d.ts +++ /dev/null @@ -1,72 +0,0 @@ -// Typed facade for the surface this spike consumes from the vendored -// @cloudflare/dofs. Deno pairs .js imports with sibling .d.ts files for -// registry npm packages but not for file:-resolved ones, so the vendored -// package's own declarations cannot be used directly; this file mirrors -// vendor/dofs/dist/{types,storage,schema/index,fs/filesystem}.d.ts for -// exactly the members the host uses. - -export interface SQLCursorLike> { - toArray(): Row[]; -} - -export interface SQLStorageLike { - exec>( - query: string, - ...bindings: unknown[] - ): SQLCursorLike; -} - -export interface DurableObjectStorageLike { - sql: SQLStorageLike; - transaction?(closure: () => T | Promise): T | Promise; - transactionSync?(closure: () => T): T; -} - -export declare class Database { - constructor(storage: DurableObjectStorageLike); - all(query: string, ...bindings: unknown[]): Row[]; -} - -export declare function initializeSchema( - db: Database, - now: () => number, -): void; - -export declare const SCHEMA_VERSION: number; - -export interface SQLiteWorkspaceProviderOptions { - now?: () => number; - watchIntervalMs?: number; -} - -export declare class SQLiteWorkspaceProvider { - readonly db: Database; - constructor(db: Database, options?: SQLiteWorkspaceProviderOptions); -} - -export interface WorkspaceDirentResult { - name: string; -} - -export interface WorkspaceStatResult { - size: number; - mode: number; -} - -export interface WorkspaceFilesystemOptions { - now?: () => number; -} - -export declare class WorkspaceFilesystem { - constructor(db: Database, options?: WorkspaceFilesystemOptions); - readFile(path: string): Promise>; - readFile(path: string, encoding: "utf8"): Promise; - stat(path: string): Promise; - lstat(path: string): Promise; - readlink(path: string): Promise; - readdir(path: string): Promise; - writeFile(path: string, content: string | Uint8Array): Promise; - mkdir(path: string, options?: { recursive?: boolean }): Promise; - rm(path: string, options?: { recursive?: boolean }): Promise; - symlink(target: string, path: string): Promise; -} diff --git a/spikes/349-dofs/host/types/platformatic-vfs.d.ts b/spikes/349-dofs/host/types/platformatic-vfs.d.ts deleted file mode 100644 index 52a9fc45..00000000 --- a/spikes/349-dofs/host/types/platformatic-vfs.d.ts +++ /dev/null @@ -1,22 +0,0 @@ -// Typed facade for the @platformatic/vfs surface this spike consumes; the -// package ships CommonJS whose default-import shape differs by runtime, so -// the host normalizes at this boundary. - -export interface VirtualFileSystem { - provider: unknown; - mkdirSync(path: string, options?: { recursive?: boolean }): void; -} - -export declare class VirtualProvider { - private brand; -} - -declare const vfsModule: { - create( - provider: unknown, - options?: { moduleHooks?: boolean }, - ): VirtualFileSystem; - VirtualProvider: typeof VirtualProvider; -}; - -export default vfsModule; diff --git a/spikes/349-dofs/host/vfs-wiring.ts b/spikes/349-dofs/host/vfs-wiring.ts deleted file mode 100644 index 29885f96..00000000 --- a/spikes/349-dofs/host/vfs-wiring.ts +++ /dev/null @@ -1,85 +0,0 @@ -// Port of upstream packages/computerd/src/fuse/vfs.ts (v0.1.1) minus the -// RPC/sync-loop branches. Deviations from upstream, in full: -// 1. No upstream/SyncRPC support. -// 2. Storage is the file-backed adapter instead of the in-memory -// SQLiteTestStorage. -// 3. After create(), the wiring verifies the vfs kept our provider — -// @platformatic/vfs's create() silently falls back to a -// MemoryProvider when the instanceof check fails, which would make -// every test pass against the wrong store. - -// @ts-types="./types/dofs.d.ts" -import { - Database, - initializeSchema, - SQLiteWorkspaceProvider, -} from "@cloudflare/dofs"; -// @ts-types="./types/platformatic-vfs.d.ts" -import vfsModule from "@platformatic/vfs"; -import type { VirtualFileSystem } from "./types/platformatic-vfs.d.ts"; -import { FileSQLiteStorage } from "./file-storage.ts"; - -const { create, VirtualProvider } = vfsModule; - -let prototypePatched = false; -function ensureVirtualProviderPrototype(): void { - if (prototypePatched) { - return; - } - const proto = SQLiteWorkspaceProvider.prototype; - const parent = Object.getPrototypeOf(proto); - if (parent === VirtualProvider.prototype) { - prototypePatched = true; - return; - } - Object.setPrototypeOf(proto, VirtualProvider.prototype); - prototypePatched = true; -} - -const EXTRA_VFS_METHODS = [ - "linkSync", - "createFileSync", - "writeRangeSync", - "truncateFileSync", - "chmodSync", - "readRangeSync", - "openWriteBufferSync", - "openWriteBufferForCreateSync", - "releaseWriteBufferSync", -]; - -export interface WiredFileSystem { - vfs: VirtualFileSystem; - db: Database; - provider: SQLiteWorkspaceProvider; - storage: FileSQLiteStorage; -} - -export function createFileBackedVfs(dbPath: string): WiredFileSystem { - ensureVirtualProviderPrototype(); - const storage = new FileSQLiteStorage(dbPath); - const db = new Database(storage); - initializeSchema(db, Date.now); - - const provider = new SQLiteWorkspaceProvider(db); - const vfs = create(provider, { moduleHooks: false }); - if (vfs.provider !== provider) { - storage.close(); - throw new Error( - "@platformatic/vfs fell back to a MemoryProvider: the prototype splice did not take", - ); - } - const source: Record = Object(provider); - for (const name of EXTRA_VFS_METHODS) { - const fn = source[name]; - if (typeof fn !== "function") { - continue; - } - Object.defineProperty(vfs, name, { - value: (...args: unknown[]) => fn.apply(provider, args), - writable: true, - configurable: true, - }); - } - return { vfs, db, provider, storage }; -} diff --git a/spikes/349-dofs/package-lock.json b/spikes/349-dofs/package-lock.json deleted file mode 100644 index cefabd07..00000000 --- a/spikes/349-dofs/package-lock.json +++ /dev/null @@ -1,253 +0,0 @@ -{ - "name": "spike-349-dofs", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "spike-349-dofs", - "dependencies": { - "@cloudflare/dofs": "file:./vendor/dofs", - "@effectionx/converge": "0.1.4", - "@effectionx/process": "0.8.1", - "@platformatic/vfs": "0.4.0", - "@xmd-spike/computerd-shim": "file:./vendor/computerd-shim", - "effection": "4.1.0" - } - }, - "node_modules/@cloudflare/dofs": { - "resolved": "vendor/dofs", - "link": true - }, - "node_modules/@cloudflare/workers-types": { - "version": "4.20260702.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20260702.1.tgz", - "integrity": "sha512-mOhf5TUEB1m2vPrxtqoIGfz0fUC9xyxRDx5gWHy5s+OCo6dcV+g7wI1R7gYCMFohhqF/2y2xeKVwMwCJjfn/WA==", - "dev": true, - "license": "MIT OR Apache-2.0" - }, - "node_modules/@effectionx/context-api": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/@effectionx/context-api/-/context-api-0.6.0.tgz", - "integrity": "sha512-t004qvlkJDMB6EhHP1lOQ97PeIn90m7cv4+wsRPnx4YBem+pJzTL+Sm1KWbKMjMeFJz4oqllUWuBJZsCi+nuTw==", - "license": "MIT", - "dependencies": { - "@effectionx/middleware": "0.1.1" - }, - "peerDependencies": { - "effection": "^3 || ^4" - } - }, - "node_modules/@effectionx/converge": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/@effectionx/converge/-/converge-0.1.4.tgz", - "integrity": "sha512-M11KU7jK3gOFGI4RT+Bf6ITMJntzzoGEO2KPx9r8uBoR+Rgwiyeuq+OeIHkuiHLUC+bB1g93KOj7aKR/G0F1/Q==", - "license": "MIT", - "dependencies": { - "@effectionx/timebox": "0.4.3" - }, - "peerDependencies": { - "effection": "^3 || ^4" - } - }, - "node_modules/@effectionx/middleware": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/@effectionx/middleware/-/middleware-0.1.1.tgz", - "integrity": "sha512-ss/bZRkt/xzJNE59r8NR1+0K/xQcIyCm0y9n8FYC8jKdFn51SPe3m3t7EfPcK8zkdjCoTOU7k1UpIXRl26asYA==", - "license": "MIT" - }, - "node_modules/@effectionx/node": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/@effectionx/node/-/node-0.2.4.tgz", - "integrity": "sha512-cPnp3fvfBKjGWekmBHdhZr5ScAr3Mg+x5IXpO8uKFe7AZ8EPAT9Di6skuB4kuGFJtRtS0Z1e5G4+2eJyapKhYA==", - "license": "MIT", - "peerDependencies": { - "effection": "^3 || ^4" - } - }, - "node_modules/@effectionx/process": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/@effectionx/process/-/process-0.8.1.tgz", - "integrity": "sha512-xyXlFja0Ill80lQ3IYfksXtJkqVmWuUOogRn/qlHWCAGlZj+MGGF8gOFbyzk/3Kx4pj14riVGgF/cyT5XCzqDw==", - "license": "MIT", - "dependencies": { - "@effectionx/context-api": "0.6.0", - "@effectionx/node": "0.2.4", - "@effectionx/scope-eval": "0.1.3", - "cross-spawn": "^7", - "ctrlc-windows": "^2", - "shellwords-ts": "^3.0.1" - }, - "peerDependencies": { - "effection": "^3 || ^4" - } - }, - "node_modules/@effectionx/scope-eval": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/@effectionx/scope-eval/-/scope-eval-0.1.3.tgz", - "integrity": "sha512-Acn45lb3H94WYhNVHXYtXOZYzjpBGDPPlsyW1Talb/vYjQzCzus5lkxxOlPyphzvi7d+7mGNXiIVt4JLSZmLnQ==", - "license": "MIT", - "peerDependencies": { - "effection": "^3 || ^4" - } - }, - "node_modules/@effectionx/timebox": { - "version": "0.4.3", - "resolved": "https://registry.npmjs.org/@effectionx/timebox/-/timebox-0.4.3.tgz", - "integrity": "sha512-cc7SLpL3svAYK8M5NS8kLQuL0lrZNoQb+Hi9NSaWOudzAW1HoewuDfUtfXLemPJnnLqLYhbghRhmpVqCm4Xg3Q==", - "license": "MIT", - "peerDependencies": { - "effection": "^3 || ^4" - } - }, - "node_modules/@platformatic/vfs": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/@platformatic/vfs/-/vfs-0.4.0.tgz", - "integrity": "sha512-JwRxSIG63e/VaDSkYXkSBajySt5MJzxYIKuknYda28fhhmJMqyUGf5rbKrGW+vO9L83nIjQH3+YI12fp7EKICQ==", - "license": "MIT", - "engines": { - "node": ">= 22" - } - }, - "node_modules/@types/node": { - "version": "24.13.3", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz", - "integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~7.18.0" - } - }, - "node_modules/@xmd-spike/computerd-shim": { - "resolved": "vendor/computerd-shim", - "link": true - }, - "node_modules/cross-spawn": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", - "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", - "license": "MIT", - "dependencies": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/ctrlc-windows": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/ctrlc-windows/-/ctrlc-windows-2.2.0.tgz", - "integrity": "sha512-t9y568r+T8FUuBaqKK60YGFJdj3b3ktdJW9WXIT3CuBdQhAOYdSZu75jFUN0Ay4Yz5HHicVQqAYCwcnqhOn23g==", - "license": "MIT" - }, - "node_modules/effection": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/effection/-/effection-4.1.0.tgz", - "integrity": "sha512-/BJuaYhzDrvC/nrIW054z1itmCNjVNFtyv/J26xrS4pP2MLfm5TG8VFe5f4zl1oGcAoF1fq1GfDO+v0qx6URPw==", - "license": "MIT", - "engines": { - "node": ">= 16" - } - }, - "node_modules/isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "license": "ISC" - }, - "node_modules/path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "license": "MIT", - "dependencies": { - "shebang-regex": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/shellwords-ts": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/shellwords-ts/-/shellwords-ts-3.0.1.tgz", - "integrity": "sha512-GabK4ApLMqHFRGlpgNqg8dmtHTnYHt0WUUJkIeMd3QaDrUUBEDXHSSNi3I0PzMimg8W+I0EN4TshQxsnHv1cwg==", - "license": "MIT" - }, - "node_modules/typescript": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", - "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=14.17" - } - }, - "node_modules/undici-types": { - "version": "7.18.2", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", - "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", - "dev": true, - "license": "MIT" - }, - "node_modules/which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "license": "ISC", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, - "vendor/computerd-shim": { - "name": "@xmd-spike/computerd-shim", - "version": "0.0.0", - "devDependencies": { - "@types/node": "^24", - "typescript": "^6.0.3" - }, - "peerDependencies": { - "@platformatic/vfs": "^0.4.0" - } - }, - "vendor/dofs": { - "name": "@cloudflare/dofs", - "version": "0.0.0", - "devDependencies": { - "@cloudflare/workers-types": "^4.20260616.1", - "@types/node": "^24", - "typescript": "^6.0.3" - } - } - } -} diff --git a/spikes/349-dofs/package.json b/spikes/349-dofs/package.json deleted file mode 100644 index 2d9fc936..00000000 --- a/spikes/349-dofs/package.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "name": "spike-349-dofs", - "private": true, - "type": "module", - "dependencies": { - "@cloudflare/dofs": "file:./vendor/dofs", - "effection": "4.1.0", - "@effectionx/process": "0.8.1", - "@effectionx/converge": "0.1.4", - "@xmd-spike/computerd-shim": "file:./vendor/computerd-shim", - "@platformatic/vfs": "0.4.0" - } -} \ No newline at end of file diff --git a/spikes/349-dofs/tests/shim.test.ts b/spikes/349-dofs/tests/shim.test.ts deleted file mode 100644 index 6510e1e3..00000000 --- a/spikes/349-dofs/tests/shim.test.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { type Operation, run } from "effection"; -import { exec } from "@effectionx/process"; - -const proofBinary = new URL("../dist/proof", import.meta.url).pathname; -const shimBinary = new URL("../dist/proof-shim", import.meta.url).pathname; - -interface Outcome { - code: number | undefined; - payload: Record; - stderr: string; -} - -function* invoke(binary: string, args: string[]): Operation { - const result = yield* exec(binary, { arguments: args }).join(); - let payload: Record = {}; - if (result.code === 0) { - const lastLine = result.stdout.trim().split("\n").at(-1) ?? ""; - const parsed: unknown = JSON.parse(lastLine); - if (typeof parsed === "object" && parsed !== null) { - payload = Object.fromEntries(Object.entries(parsed)); - } - } - return { code: result.code, payload, stderr: result.stderr }; -} - -function assertEquals(actual: unknown, expected: unknown, detail: string) { - const left = JSON.stringify(actual); - const right = JSON.stringify(expected); - if (left !== right) { - throw new Error(`${detail}: expected ${right}, got ${left}`); - } -} - -// The mount path is part of the workspace namespace, so each scenario pins -// one absolute mount directory and reuses it for the database's lifetime. -Deno.test("a native subprocess reads and mutates the workspace through the userspace shim", () => - run(function* () { - const root = Deno.makeTempDirSync({ prefix: "spike349-shim-" }); - const db = `${root}/ws.db`; - const mount = `${root}/mount`; - - yield* invoke(proofBinary, [db, "mkdir", mount]); - yield* invoke(proofBinary, [db, "write", `${mount}/api.txt`, "from-api"]); - - const execRun = yield* invoke(shimBinary, [ - db, - mount, - "exec", - "cat api.txt > copy.txt && printf sub > sub.txt", - ]); - assertEquals(execRun.payload.code, 0, "subprocess exit code"); - - const sub = yield* invoke(proofBinary, [db, "read", `${mount}/sub.txt`]); - assertEquals( - sub.payload.body, - "sub", - "a subprocess write lands in the SQLite workspace", - ); - const copy = yield* invoke(proofBinary, [db, "read", `${mount}/copy.txt`]); - assertEquals( - copy.payload.body, - "from-api", - "the subprocess read an API-written file through the mount", - ); - })); - -Deno.test("the shim rematerializes the persisted frontier into an emptied mount directory", () => - run(function* () { - const root = Deno.makeTempDirSync({ prefix: "spike349-shim-mat-" }); - const db = `${root}/ws.db`; - const mount = `${root}/mount`; - - yield* invoke(proofBinary, [db, "mkdir", mount]); - yield* invoke(proofBinary, [db, "write", `${mount}/keep.txt`, "durable"]); - - yield* invoke(shimBinary, [db, mount, "materialize"]); - const first = Deno.readTextFileSync(`${mount}/keep.txt`); - assertEquals(first, "durable", "boot materialization writes the file"); - - Deno.removeSync(mount, { recursive: true }); - yield* invoke(shimBinary, [db, mount, "materialize"]); - const again = Deno.readTextFileSync(`${mount}/keep.txt`); - assertEquals( - again, - "durable", - "an emptied mount directory is rebuilt from SQLite state", - ); - })); diff --git a/spikes/349-dofs/tests/spike.test.ts b/spikes/349-dofs/tests/spike.test.ts deleted file mode 100644 index 4097f0b0..00000000 --- a/spikes/349-dofs/tests/spike.test.ts +++ /dev/null @@ -1,126 +0,0 @@ -import { type Operation, run } from "effection"; -import { exec } from "@effectionx/process"; -import { DatabaseSync } from "node:sqlite"; - -const proofBinary = new URL("../dist/proof", import.meta.url).pathname; - -interface ProofOutcome { - code: number | undefined; - payload: Record; - stderr: string; -} - -function* proof( - dbPath: string, - op: string, - ...args: string[] -): Operation { - const result = yield* exec(proofBinary, { - arguments: [dbPath, op, ...args], - }).join(); - let payload: Record = {}; - if (result.code === 0) { - const lastLine = result.stdout.trim().split("\n").at(-1) ?? ""; - const parsed: unknown = JSON.parse(lastLine); - if (typeof parsed === "object" && parsed !== null) { - payload = Object.fromEntries(Object.entries(parsed)); - } - } - return { code: result.code, payload, stderr: result.stderr }; -} - -function assertEquals(actual: unknown, expected: unknown, detail: string) { - const left = JSON.stringify(actual); - const right = JSON.stringify(expected); - if (left !== right) { - throw new Error(`${detail}: expected ${right}, got ${left}`); - } -} - -Deno.test("filesystem frontier survives full process restarts, including create/delete/create", () => - run(function* () { - const dir = Deno.makeTempDirSync({ prefix: "spike349-" }); - const db = `${dir}/ws.db`; - - yield* proof(db, "mkdir", "/notes"); - yield* proof(db, "write", "/notes/a.md", "alpha"); - const alpha = yield* proof(db, "read", "/notes/a.md"); - assertEquals(alpha.payload.body, "alpha", "read after restart"); - - yield* proof(db, "write", "/notes/b.md", "beta"); - yield* proof(db, "rm", "/notes/a.md"); - const listing = yield* proof(db, "ls", "/notes"); - assertEquals( - listing.payload.entries, - ["b.md"], - "deletion survives into the next process", - ); - - yield* proof(db, "write", "/f.txt", "v1"); - yield* proof(db, "rm", "/f.txt"); - yield* proof(db, "write", "/f.txt", "v2"); - const recreated = yield* proof(db, "read", "/f.txt"); - assertEquals( - recreated.payload.body, - "v2", - "create/delete/create keeps the last content", - ); - - yield* proof(db, "rename", "/notes/b.md", "/notes/c.md"); - const renamed = yield* proof(db, "ls", "/notes"); - assertEquals(renamed.payload.entries, ["c.md"], "rename persists"); - - yield* proof(db, "symlink", "/notes/c.md", "/link"); - const linkTarget = yield* proof(db, "readlink", "/link"); - assertEquals(linkTarget.payload.target, "/notes/c.md", "readlink"); - const throughLink = yield* proof(db, "read", "/link"); - assertEquals( - throughLink.payload.body, - "beta", - "readFile follows symlinks across a restart", - ); - - const gone = yield* proof(db, "read", "/notes/a.md"); - if (gone.code === 0) { - throw new Error("reading a deleted file succeeded"); - } - })); - -Deno.test("separate database paths are separate workspaces", () => - run(function* () { - const dir = Deno.makeTempDirSync({ prefix: "spike349-iso-" }); - yield* proof(`${dir}/a.db`, "write", "/only-in-a.txt", "a"); - const other = yield* proof(`${dir}/b.db`, "ls", "/"); - assertEquals(other.payload.entries, [], "second database starts empty"); - })); - -Deno.test("clean close leaves a single-file artifact (WAL checkpointed and removed)", () => - run(function* () { - const dir = Deno.makeTempDirSync({ prefix: "spike349-wal-" }); - const db = `${dir}/ws.db`; - yield* proof(db, "write", "/x.txt", "wal-check"); - const files = Array.from(Deno.readDirSync(dir)).map((entry) => entry.name) - .sort(); - assertEquals(files, ["ws.db"], "no -wal/-shm files after close"); - })); - -Deno.test("a newer on-disk schema version is refused loudly, not recreated", () => - run(function* () { - const dir = Deno.makeTempDirSync({ prefix: "spike349-schema-" }); - const db = `${dir}/ws.db`; - yield* proof(db, "write", "/keep.txt", "content"); - - const raw = new DatabaseSync(db); - raw.exec("UPDATE vfs_meta SET v = '99' WHERE k = 'schema_version'"); - raw.close(); - - const refused = yield* proof(db, "read", "/keep.txt"); - if (refused.code === 0) { - throw new Error("opening a future-schema database succeeded"); - } - if (!refused.stderr.includes("schema version 99")) { - throw new Error( - `refusal does not name the schema version: ${refused.stderr.slice(-300)}`, - ); - } - })); diff --git a/spikes/349-dofs/vendor.ts b/spikes/349-dofs/vendor.ts deleted file mode 100644 index 42951069..00000000 --- a/spikes/349-dofs/vendor.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { main } from "effection"; -import { exec } from "@effectionx/process"; - -const dofsDir = new URL("./vendor/dofs/", import.meta.url).pathname; -const shimDir = new URL("./vendor/computerd-shim/", import.meta.url).pathname; - -main(function* () { - yield* exec("npm", { - arguments: ["install", "--no-audit", "--no-fund"], - cwd: dofsDir, - }).expect(); - yield* exec("npx", { - arguments: ["tsc", "-p", "tsconfig.build.json"], - cwd: dofsDir, - }).expect(); - console.log("vendored dofs built to vendor/dofs/dist"); - yield* exec("npm", { - arguments: ["install", "--no-audit", "--no-fund"], - cwd: shimDir, - }).expect(); - yield* exec("npx", { - arguments: ["tsc", "-p", "tsconfig.json"], - cwd: shimDir, - }).expect(); - console.log("vendored shim built to vendor/computerd-shim/dist"); -}); diff --git a/spikes/349-dofs/vendor/computerd-shim/LICENSE b/spikes/349-dofs/vendor/computerd-shim/LICENSE deleted file mode 100644 index 631c4d3e..00000000 --- a/spikes/349-dofs/vendor/computerd-shim/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License Copyright (c) 2026 Cloudflare, Inc. - -Permission is hereby granted, free of -charge, to any person obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the Software without -restriction, including without limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to the -following conditions: - -The above copyright notice and this permission notice -(including the next paragraph) shall be included in all copies or substantial -portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO -EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/spikes/349-dofs/vendor/computerd-shim/PROVENANCE.md b/spikes/349-dofs/vendor/computerd-shim/PROVENANCE.md deleted file mode 100644 index 4c37ab2d..00000000 --- a/spikes/349-dofs/vendor/computerd-shim/PROVENANCE.md +++ /dev/null @@ -1,26 +0,0 @@ -# Vendored: computerd userspace shim (subset) - -- Source: https://github.com/cloudflare/computer, `packages/computerd` -- Tag: `v0.1.1`, commit `63d363632e558f7e077794988d36ed75017c2a62` -- License: MIT (Cloudflare, Inc.) — see [LICENSE](LICENSE), copied from the - repository root. -- This is a deliberate subset, named `@xmd-spike/computerd-shim` rather than - `@cloudflare/computerd` because it is not the upstream package: it carries - only the userspace mount shim. -- Contents: - - `src/shim/shim.ts` — byte-identical to upstream - `packages/computerd/src/shim/shim.ts` - (sha1 `05e6a2acc0bdbd9970d7e1fea2c1e13d482b528d`). - - `src/fuse/vfs.ts` — a five-line type-only stub replacing upstream's - runtime module: shim.ts imports only the `NodeVirtualFileSystem` type - from it, which upstream aliases to `@platformatic/vfs`'s - `VirtualFileSystem`. The runtime wiring upstream keeps in that module - (prototype splice + method forwarding) lives in this spike's - `host/vfs-wiring.ts` instead, ported with its deviations documented - in-file. - - `package.json` / `tsconfig.json` — authored here (upstream builds the - whole computerd package CommonJS; this subset compiles standalone as - ESM with `tsc`). -- Upgrade procedure: re-copy `shim.ts` from the new upstream tag, verify the - type-only import of `../fuse/vfs.js` is still the only coupling, rebuild, - re-run `deno task spike:349:test`. diff --git a/spikes/349-dofs/vendor/computerd-shim/package-lock.json b/spikes/349-dofs/vendor/computerd-shim/package-lock.json deleted file mode 100644 index 1e26eb31..00000000 --- a/spikes/349-dofs/vendor/computerd-shim/package-lock.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "name": "@xmd-spike/computerd-shim", - "version": "0.0.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "@xmd-spike/computerd-shim", - "version": "0.0.0", - "devDependencies": { - "@types/node": "^24", - "typescript": "^6.0.3" - }, - "peerDependencies": { - "@platformatic/vfs": "^0.4.0" - } - }, - "node_modules/@platformatic/vfs": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/@platformatic/vfs/-/vfs-0.4.0.tgz", - "integrity": "sha512-JwRxSIG63e/VaDSkYXkSBajySt5MJzxYIKuknYda28fhhmJMqyUGf5rbKrGW+vO9L83nIjQH3+YI12fp7EKICQ==", - "license": "MIT", - "peer": true, - "engines": { - "node": ">= 22" - } - }, - "node_modules/@types/node": { - "version": "24.13.3", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz", - "integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~7.18.0" - } - }, - "node_modules/typescript": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", - "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=14.17" - } - }, - "node_modules/undici-types": { - "version": "7.18.2", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", - "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", - "dev": true, - "license": "MIT" - } - } -} diff --git a/spikes/349-dofs/vendor/computerd-shim/package.json b/spikes/349-dofs/vendor/computerd-shim/package.json deleted file mode 100644 index 3d401e6c..00000000 --- a/spikes/349-dofs/vendor/computerd-shim/package.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "name": "@xmd-spike/computerd-shim", - "version": "0.0.0", - "private": true, - "type": "module", - "exports": { - ".": { - "default": "./dist/shim/shim.js" - } - }, - "scripts": { - "build": "tsc -p tsconfig.json" - }, - "devDependencies": { - "typescript": "^6.0.3", - "@types/node": "^24" - }, - "peerDependencies": { - "@platformatic/vfs": "^0.4.0" - } -} diff --git a/spikes/349-dofs/vendor/computerd-shim/src/fuse/vfs.ts b/spikes/349-dofs/vendor/computerd-shim/src/fuse/vfs.ts deleted file mode 100644 index 3a859796..00000000 --- a/spikes/349-dofs/vendor/computerd-shim/src/fuse/vfs.ts +++ /dev/null @@ -1,5 +0,0 @@ -// Type-only stub for the probe. Upstream vfs.ts line 7 is: -// export type NodeVirtualFileSystem = VirtualFileSystem; -// The runtime wiring (prototype splice + EXTRA_VFS_METHODS forwarding) -// is hand-ported to wiring.mjs; shim.ts only imports this type. -export type { VirtualFileSystem as NodeVirtualFileSystem } from "@platformatic/vfs"; diff --git a/spikes/349-dofs/vendor/computerd-shim/src/shim/shim.ts b/spikes/349-dofs/vendor/computerd-shim/src/shim/shim.ts deleted file mode 100644 index 65de9d1f..00000000 --- a/spikes/349-dofs/vendor/computerd-shim/src/shim/shim.ts +++ /dev/null @@ -1,578 +0,0 @@ -// Userspace bidirectional sync between the @platformatic/vfs store -// and a real directory on the host filesystem. Used when FUSE is -// unavailable (no /dev/fuse, no macFUSE) and the user has opted in -// via FUSE_MOUNT=shim. Explicitly not production-grade: races between -// writers across the seam are resolved on the next reconcile tick, -// with VFS winning ties. -// -// Model: -// -// - A "shadow" snapshot records what we last successfully synced. -// Map. -// -// - VFS -> disk runs off vfs.watchAsync("/", { recursive: true }). -// On each event we read the VFS, write the host fs, and update -// the shadow to match what we just wrote. -// -// - Disk -> VFS runs off a periodic reconcile tick. We walk the -// host directory, diff it against the shadow, and push any new -// or changed entries into the VFS. Deletes are inferred from -// shadow keys that no longer exist on disk. -// -// Loop suppression falls out of the shadow: after writing in either -// direction the shadow matches both sides, so the next tick on the -// opposite side sees no diff and emits nothing. - -import { createHash } from "node:crypto"; -import { - mkdir as fsMkdir, - readdir as fsReaddir, - readFile as fsReadFile, - rm as fsRm, - stat as fsStat, - writeFile as fsWriteFile, -} from "node:fs/promises"; -import { dirname, join, posix } from "node:path"; -import type { NodeVirtualFileSystem } from "../fuse/vfs.js"; - -export interface ShimMount { - unmount(): Promise; - /** - * Block until the on-disk tree at `mountPoint` reflects the - * VFS's current state. Called by the SyncRPC `push` handler - * after applying a peer batch so a subsequent `shell.exec` is - * guaranteed to see the just-pushed files — the watcher-driven - * VFS→disk path is async, and reads from spawned processes go - * against the real fs. - * - * Walks the VFS tree and calls `syncVfsPathToDisk` for every - * entry. The shadow short-circuits files that already match, - * so flushing twice in a row is cheap. - */ - flush(): Promise; - /** - * Block until the VFS reflects the on-disk tree's current state. - * Called by the SyncRPC `fetchChanges` handler right before it - * computes the change set the puller will see, so a - * `Workspace.pull()` issued after `shell.exec` returns observes - * files the exec'd process wrote without waiting on the next - * periodic poll tick. - * - * Runs the same disk→VFS reconcile the polling loop runs, - * serialised through the same internal mutex so a request-time - * call can't race with the tick. Idempotent on a clean tree. - */ - reconcileNow(): Promise; -} - -export interface MountShimOptions { - vfs: NodeVirtualFileSystem; - mountPoint: string; - // Disk poll cadence. Default 250ms — fast enough for an editor - // save to surface in the VFS before the next exec command, slow - // enough that the recursive readdir stays in the noise. - pollIntervalMs?: number; -} - -interface ShadowEntry { - kind: "file" | "dir"; - size: number; - mtimeMs: number; - // sha1 of the file content the last time we synced. Used to - // detect "echo" events: when a disk→VFS write triggers the VFS - // watcher, the watcher reads the VFS and sees content that - // already matches the shadow. Skipping in that case prevents the - // shim from resurrecting a file the user just deleted from disk. - contentHash?: string; -} - -type Shadow = Map; - -const DEFAULT_POLL_MS = 250; - -export async function mountShim(options: MountShimOptions): Promise { - const { vfs, mountPoint } = options; - const pollIntervalMs = options.pollIntervalMs ?? DEFAULT_POLL_MS; - - // VFS paths and host paths share the same absolute prefix: - // `${mountPoint}/foo.txt` lives at `${mountPoint}/foo.txt` in the - // VFS and at `${mountPoint}/foo.txt` on disk. The shim drives - // everything off the host path; toVfs / toHost are identities, - // kept as named helpers so the asymmetry shows up if it ever - // creeps back in. - await fsMkdir(mountPoint, { recursive: true }); - vfs.mkdirSync(mountPoint, { recursive: true }); - - // Reconcile mutex. Both the watch loop and the poll loop mutate - // disk + shadow; serialising them lets us keep the shadow as the - // single source of truth for "what we last synced" without - // racing on map updates. - let chain: Promise = Promise.resolve(); - const run = (fn: () => Promise): Promise => { - const next = chain.then(fn, fn); - chain = next.then( - () => {}, - () => {}, - ); - return next; - }; - - const shadow: Shadow = new Map(); - - // Initial materialisation: VFS is the source of truth at boot. - // Walk it depth-first, write everything to disk, populate shadow. - // If the mount point already has content from a previous run we - // leave it alone — the disk-poll on the first reconcile tick - // will treat any extra files as disk-side additions and push - // them into the VFS. That matches the "computerd is not the only - // writer" relaxation the shim explicitly takes on. - await run(async () => { - await materialiseVfsToDisk(vfs, mountPoint, shadow); - }); - - // The watcher is scoped to the mount point so any events outside - // it (e.g. unrelated VFS writes under sibling mounts) don't drive - // the shim. coalesceChanges emits filenames relative to the - // watched root, so we reconstruct the absolute VFS path before - // syncing. - - // VFS -> disk via the platformatic/dofs watcher. watchAsync - // returns an AsyncIterable; the iterator's return() method (called - // implicitly when we break out below, or explicitly in unmount) - // tears down the underlying interval. - let stopped = false; - // watchAsync lives on the provider, not the VFS facade — the dofs - // SQLiteWorkspaceProvider implements it via revision polling. - const watcher = vfs.provider.watchAsync(mountPoint, { recursive: true }) as AsyncIterable<{ - eventType: "rename" | "change"; - filename: string; - }> & { return?(): Promise }; - - const watchLoop = (async () => { - try { - for await (const event of watcher) { - if (stopped) break; - const vfsPath = joinMount(mountPoint, event.filename); - // watchAsync emits filename="" for the watched root on some - // operations; the mount point itself is not materialised - // beneath itself. - if (vfsPath === mountPoint) continue; - await run(() => syncVfsPathToDisk(vfs, mountPoint, vfsPath, shadow)); - } - } catch (error) { - if (!stopped) { - console.error("[shim] VFS watcher loop failed:", error); - } - } - })(); - - // Disk -> VFS via periodic reconcile. Walks the mount point, - // diffs against the shadow, applies changes to the VFS. - // Shared by the periodic poll below and the on-demand - // reconcileNow() hook; both go through `run` so they serialise - // against the VFS watcher loop. - const reconcile = (): Promise => run(() => reconcileDiskToVfs(vfs, mountPoint, shadow)); - const pollTimer = setInterval(() => { - if (stopped) return; - void reconcile().catch((error) => { - console.error("[shim] disk reconcile failed:", error); - }); - }, pollIntervalMs); - pollTimer.unref?.(); - - return { - async unmount(): Promise { - if (stopped) return; - stopped = true; - clearInterval(pollTimer); - // Best-effort: tell the AsyncIterable we're done so its - // backing interval clears. The watcher iterator may be parked - // inside `for await`; calling return() resolves that pending - // next() with done:true. - try { - await watcher.return?.(); - } catch { - // ignore - } - await watchLoop; - }, - async flush(): Promise { - if (stopped) return; - // Wait for anything the watcher loop has already queued, then - // walk the VFS once and reconcile every entry against disk. - // Both steps go through `run` so they serialise with the rest - // of the chain — a concurrent watcher event can't sneak in a - // write between our walk and the resolve of `flush()`. - await run(async () => { - await flushVfsToDisk(vfs, mountPoint, shadow); - }); - }, - async reconcileNow(): Promise { - if (stopped) return; - await reconcile(); - }, - }; -} - -// --- VFS -> disk ---------------------------------------------------------- - -async function materialiseVfsToDisk( - vfs: NodeVirtualFileSystem, - mountPoint: string, - shadow: Shadow, -): Promise { - const queue: string[] = [mountPoint]; - while (queue.length > 0) { - const current = queue.shift() as string; - let entries: string[]; - try { - entries = vfs.readdirSync(current) as string[]; - } catch { - continue; - } - for (const name of entries) { - const vfsPath = `${current}/${name}`; - const stat = safeVfsStat(vfs, vfsPath); - if (stat === undefined) continue; - const hostPath = toHostPath(mountPoint, vfsPath); - if (stat.isDirectory()) { - await fsMkdir(hostPath, { recursive: true }); - shadow.set(vfsPath, dirShadow(stat.mtimeMs)); - queue.push(vfsPath); - } else if (stat.isFile()) { - const bytes = Buffer.from(vfs.readFileSync(vfsPath) as Buffer); - await fsWriteFile(hostPath, bytes); - const after = await fsStat(hostPath); - shadow.set(vfsPath, { - kind: "file", - size: bytes.byteLength, - mtimeMs: after.mtimeMs, - contentHash: hash(bytes), - }); - } - } - } -} - -/** - * Walk the VFS tree and call `syncVfsPathToDisk` for every entry. - * Unlike `materialiseVfsToDisk` (which always writes), this honors - * the shadow so files that already match disk are no-ops. Used by - * `ShimMount.flush()` to settle a freshly-applied push batch. - */ -async function flushVfsToDisk( - vfs: NodeVirtualFileSystem, - mountPoint: string, - shadow: Shadow, -): Promise { - const queue: string[] = [mountPoint]; - while (queue.length > 0) { - const current = queue.shift() as string; - let entries: string[]; - try { - entries = vfs.readdirSync(current) as string[]; - } catch { - continue; - } - for (const name of entries) { - const vfsPath = `${current}/${name}`; - const stat = safeVfsStat(vfs, vfsPath); - if (stat === undefined) continue; - await syncVfsPathToDisk(vfs, mountPoint, vfsPath, shadow); - if (stat.isDirectory()) queue.push(vfsPath); - } - } -} - -async function syncVfsPathToDisk( - vfs: NodeVirtualFileSystem, - mountPoint: string, - vfsPath: string, - shadow: Shadow, -): Promise { - const stat = safeVfsStat(vfs, vfsPath); - const hostPath = toHostPath(mountPoint, vfsPath); - - if (stat === undefined) { - // VFS says it's gone. Remove from disk + shadow. Recursive rm - // covers the case where the VFS dropped an entire subtree in a - // single rev (rare, but cheap to handle). - await fsRm(hostPath, { recursive: true, force: true }); - deleteSubtreeFromShadow(shadow, vfsPath); - return; - } - - if (stat.isDirectory()) { - await fsMkdir(hostPath, { recursive: true }); - shadow.set(vfsPath, dirShadow(stat.mtimeMs)); - return; - } - - if (stat.isFile()) { - const bytes = Buffer.from(vfs.readFileSync(vfsPath) as Buffer); - const digest = hash(bytes); - // Echo guard: if the shadow already records this exact content, - // the watcher event we're servicing was triggered by our own - // disk→VFS write a moment ago. Don't touch disk — the user - // may have deleted or modified the file between then and now, - // and resurrecting it would lose their change. The disk-poll - // is responsible for propagating any such disk-side state. - const prev = shadow.get(vfsPath); - if (prev?.kind === "file" && prev.contentHash === digest) { - return; - } - // Content-equal short-circuit: if disk already matches, skip - // the write to avoid bumping mtime and confusing the next - // disk-poll tick. - const current = await readIfFile(hostPath); - if (current === undefined || !buffersEqual(current, bytes)) { - await fsMkdir(dirname(hostPath), { recursive: true }); - await fsWriteFile(hostPath, bytes); - } - const after = await fsStat(hostPath); - shadow.set(vfsPath, { - kind: "file", - size: bytes.byteLength, - mtimeMs: after.mtimeMs, - contentHash: digest, - }); - } -} - -// --- disk -> VFS ---------------------------------------------------------- - -async function reconcileDiskToVfs( - vfs: NodeVirtualFileSystem, - mountPoint: string, - shadow: Shadow, -): Promise { - const seen = new Set(); - await walkDisk(mountPoint, mountPoint, async (hostPath, vfsPath, stat) => { - seen.add(vfsPath); - const prev = shadow.get(vfsPath); - if (stat.isDirectory()) { - if (prev?.kind === "dir") return; - // Either new directory or replaces a file. Make sure the VFS - // has it. mkdir with recursive matches mkdir -p semantics — - // safe when the path already exists. - if (prev?.kind === "file") { - try { - vfs.unlinkSync(vfsPath); - } catch { - // ignore - } - } - try { - vfs.mkdirSync(vfsPath, { recursive: true }); - } catch { - // ignore — typically EEXIST after a concurrent create - } - shadow.set(vfsPath, dirShadow(stat.mtimeMs)); - return; - } - if (!stat.isFile()) return; - // File. Skip if the shadow says we already have this exact - // (size, mtime) — that's our "did anything change?" check. - if (prev?.kind === "file" && prev.size === stat.size && prev.mtimeMs === stat.mtimeMs) { - return; - } - let bytes: Buffer; - try { - bytes = await fsReadFile(hostPath); - } catch { - return; - } - // Content-equal short-circuit: if the VFS already holds the same - // bytes (e.g. an editor save that wrote identical content) - // refresh the shadow without re-writing the VFS, which would - // otherwise bump vfs_meta.rev and echo back over the watch loop. - const vfsBytes = safeVfsRead(vfs, vfsPath); - if (vfsBytes !== undefined && buffersEqual(vfsBytes, bytes)) { - shadow.set(vfsPath, { - kind: "file", - size: stat.size, - mtimeMs: stat.mtimeMs, - contentHash: hash(bytes), - }); - return; - } - try { - // Ensure the parent dir exists in the VFS — `walkDisk` walks - // breadth-first by default but we don't rely on parent - // ordering being friendly. - const parent = posix.dirname(vfsPath); - if (parent !== "/" && parent !== ".") { - try { - vfs.mkdirSync(parent, { recursive: true }); - } catch { - // ignore - } - } - vfs.writeFileSync(vfsPath, bytes); - } catch (error) { - console.error(`[shim] failed to write ${vfsPath} into VFS:`, error); - return; - } - shadow.set(vfsPath, { - kind: "file", - size: stat.size, - mtimeMs: stat.mtimeMs, - contentHash: hash(bytes), - }); - }); - - // Deletions: anything in the shadow that the walk didn't see is - // gone from disk. Drop it from the VFS too. Sort by depth so - // children come before parents — vfs.rmdirSync rejects non-empty - // directories. - const removed = [...shadow.keys()].filter((p) => !seen.has(p)); - removed.sort((a, b) => depth(b) - depth(a)); - for (const vfsPath of removed) { - const entry = shadow.get(vfsPath); - if (entry === undefined) continue; - try { - if (entry.kind === "dir") { - vfs.rmdirSync(vfsPath); - } else { - vfs.unlinkSync(vfsPath); - } - } catch { - // ignore — most often a race with a concurrent VFS write - // that already removed the entry, or a non-empty dir that - // will get cleaned up on the next pass. - } - shadow.delete(vfsPath); - } -} - -async function walkDisk( - root: string, - current: string, - visit: ( - hostPath: string, - vfsPath: string, - stat: { isDirectory(): boolean; isFile(): boolean; size: number; mtimeMs: number }, - ) => Promise, -): Promise { - let entries: string[]; - try { - entries = await fsReaddir(current); - } catch { - return; - } - for (const name of entries) { - const hostPath = join(current, name); - let stat: Awaited>; - try { - stat = await fsStat(hostPath); - } catch { - continue; - } - const vfsPath = toVfsPath(root, hostPath); - await visit(hostPath, vfsPath, stat); - if (stat.isDirectory()) { - await walkDisk(root, hostPath, visit); - } - } -} - -// --- helpers -------------------------------------------------------------- - -// VFS and host namespaces share the same absolute paths under -// `mountPoint`. The translators are identities on the prefix; they -// strip and re-attach `mountPoint` so the rest of the code can -// stay symmetrical without sprinkling string slicing throughout. -function toHostPath(mountPoint: string, vfsPath: string): string { - if (vfsPath === mountPoint) return mountPoint; - const prefix = mountPoint === "/" ? "/" : `${mountPoint}/`; - const rel = vfsPath.startsWith(prefix) - ? vfsPath.slice(prefix.length) - : vfsPath.replace(/^\/+/, ""); - return rel === "" ? mountPoint : join(mountPoint, rel); -} - -function toVfsPath(mountPoint: string, hostPath: string): string { - const rel = hostPath.slice(mountPoint.length).replace(/\\/g, "/"); - if (rel === "" || rel === "/") return mountPoint; - return rel.startsWith("/") ? `${mountPoint}${rel}` : `${mountPoint}/${rel}`; -} - -function joinMount(mountPoint: string, filename: string): string { - if (filename === "" || filename === "/") return mountPoint; - return filename.startsWith("/") ? `${mountPoint}${filename}` : `${mountPoint}/${filename}`; -} - -function dirShadow(mtimeMs: number): ShadowEntry { - return { kind: "dir", size: 0, mtimeMs }; -} - -function depth(path: string): number { - let count = 0; - for (let i = 0; i < path.length; i++) if (path.charCodeAt(i) === 47) count++; - return count; -} - -function deleteSubtreeFromShadow(shadow: Shadow, vfsPath: string): void { - const prefix = vfsPath === "/" ? "/" : `${vfsPath}/`; - for (const key of [...shadow.keys()]) { - if (key === vfsPath || key.startsWith(prefix)) shadow.delete(key); - } -} - -function safeVfsStat( - vfs: NodeVirtualFileSystem, - path: string, -): - | { - isFile(): boolean; - isDirectory(): boolean; - size: number; - mtimeMs: number; - } - | undefined { - try { - const s = vfs.statSync(path) as { - isFile(): boolean; - isDirectory(): boolean; - size: number; - mtime: Date; - }; - return { - isFile: () => s.isFile(), - isDirectory: () => s.isDirectory(), - size: s.size, - mtimeMs: s.mtime.getTime(), - }; - } catch { - return undefined; - } -} - -function safeVfsRead(vfs: NodeVirtualFileSystem, path: string): Buffer | undefined { - try { - return Buffer.from(vfs.readFileSync(path) as Buffer); - } catch { - return undefined; - } -} - -async function readIfFile(hostPath: string): Promise { - try { - return await fsReadFile(hostPath); - } catch { - return undefined; - } -} - -function buffersEqual(a: Buffer, b: Buffer): boolean { - if (a.byteLength !== b.byteLength) return false; - // For small files just compare directly; for large ones hashing - // avoids touching every byte twice (once for compare, once for - // write). Threshold is conservative. - if (a.byteLength < 64 * 1024) return a.equals(b); - return hash(a) === hash(b); -} - -function hash(buf: Buffer): string { - return createHash("sha1").update(buf).digest("hex"); -} diff --git a/spikes/349-dofs/vendor/computerd-shim/tsconfig.json b/spikes/349-dofs/vendor/computerd-shim/tsconfig.json deleted file mode 100644 index d550a1a8..00000000 --- a/spikes/349-dofs/vendor/computerd-shim/tsconfig.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2022", - "module": "NodeNext", - "moduleResolution": "NodeNext", - "outDir": "dist", - "rootDir": "src", - "declaration": true, - "strict": true, - "skipLibCheck": true, - "isolatedModules": true, - "verbatimModuleSyntax": true, - "types": ["node"] - }, - "include": ["src/**/*.ts"] -} diff --git a/spikes/349-dofs/vendor/dofs/.gitignore b/spikes/349-dofs/vendor/dofs/.gitignore deleted file mode 100644 index bcef19c8..00000000 --- a/spikes/349-dofs/vendor/dofs/.gitignore +++ /dev/null @@ -1 +0,0 @@ -tests/worker-configuration.d.ts diff --git a/spikes/349-dofs/vendor/dofs/LICENSE b/spikes/349-dofs/vendor/dofs/LICENSE deleted file mode 100644 index 631c4d3e..00000000 --- a/spikes/349-dofs/vendor/dofs/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License Copyright (c) 2026 Cloudflare, Inc. - -Permission is hereby granted, free of -charge, to any person obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the Software without -restriction, including without limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to the -following conditions: - -The above copyright notice and this permission notice -(including the next paragraph) shall be included in all copies or substantial -portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO -EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/spikes/349-dofs/vendor/dofs/PROVENANCE.md b/spikes/349-dofs/vendor/dofs/PROVENANCE.md deleted file mode 100644 index 76316937..00000000 --- a/spikes/349-dofs/vendor/dofs/PROVENANCE.md +++ /dev/null @@ -1,22 +0,0 @@ -# Vendored: @cloudflare/dofs - -- Source: https://github.com/cloudflare/computer, `packages/dofs` -- Tag: `v0.1.1`, commit `63d363632e558f7e077794988d36ed75017c2a62` -- License: MIT (Cloudflare, Inc.) — see [LICENSE](LICENSE), copied from the - repository root; the package directory carries no separate license file. -- Local modifications, in full: - - `package.json`: devDependencies trimmed to `typescript`, - `@cloudflare/workers-types`, `@types/node` (the upstream set fails - `npm install` on an ERESOLVE conflict between `wrangler@4.119` and - `@cloudflare/workers-types@^4`; the removed packages are test-only), and - the vitest/wrangler scripts dropped with them. - - `tsconfig.build.json`: `types` pinned to - `["@cloudflare/workers-types", "node"]` so the build resolves without - the removed dev dependencies. - - `package.json`: a `./fs/rename` exports entry added — upstream's - `index.ts` does not re-export `rename`, and the spike consumes the - package through its exports map. - - No source file under `src/` is modified. -- Upgrade procedure: re-copy `packages/dofs` from the new upstream tag, - re-apply the two manifest edits above, rebuild, re-run - `deno task spike:349:test`. diff --git a/spikes/349-dofs/vendor/dofs/README.md b/spikes/349-dofs/vendor/dofs/README.md deleted file mode 100644 index 8eacbf4a..00000000 --- a/spikes/349-dofs/vendor/dofs/README.md +++ /dev/null @@ -1,61 +0,0 @@ -# `@cloudflare/dofs` - -> [!IMPORTANT] -> **PREVIEW ONLY** This package is provided as a preview for feedback only. -> APIs are unstable and the design is subject to change. -> -> Suitable for experiments, exploration and prototypes. It is NOT suitable -> for production use at this time. -> -> The specification under [`docs/`](../../docs/README.md) is forward-looking — read it for -> intent, not as description of the code today. - -Durable Object SQLite-backed virtual filesystem for Cloudflare Computer. - -This package exposes a JavaScript module, not a CLI. It bundles three layers that can be used independently: - -- A `Database` wrapper around Durable Object SQL storage plus `initializeSchema` for the `vfs_*` tables. -- Filesystem primitives under `src/fs/*` (`mkdir`, `writeFile`, `readFile`, `rm`, `readdir`, `stat`, `lstat`, `chmod`, `find`, `ls`, `grep`, `symlink`, `readlink`, `gc`, `watch`) operating on a `Database`. -- `SQLiteWorkspaceProvider`, a `@platformatic/vfs` adapter that composes those primitives into a node-shaped filesystem (fd table, positional `readSync`/`writeSync`, `watchSync`, symlinks). This is what `computerd` mounts via FUSE. -- Sync protocol building blocks operating on the same `Database`: `applyChanges`, `stageBlob`, `materialiseChange`, `coalesceChanges`, `fetchChanges`, `fetchObjects`, `hasObjects`, `pushObjects`, `buildManifest`, `currentRev`, `compareChangeCursors`, `readWatermark`/`writeWatermark`, `assertAppliedPushCursor`, and `DEFAULT_IGNORE`/`isIgnored`. The wire wiring lives in `@cloudflare/computer-rpc`. - -Minimal DO-side usage — initialize the schema; the `Database` becomes the handle every other helper takes: - -```ts -import { Database, initializeSchema } from "@cloudflare/dofs"; - -export class WorkspaceDO extends DurableObject { - private readonly db: Database; - - constructor(ctx: DurableObjectState, env: Env) { - super(ctx, env); - this.db = new Database(ctx.storage); - initializeSchema(this.db, Date.now); - } -} -``` - -> The `src/fs/*` primitives (`mkdir`, `writeFile`, `readFile`, `rm`, `readdir`, `stat`, `find`, `ls`, `grep`, `symlink`, `readlink`, `gc`, `watch`) are not re-exported from the package root yet — they are consumed in-tree by `SQLiteWorkspaceProvider` and by the sync `applyChanges` path. On the node side, instantiate `SQLiteWorkspaceProvider` (the `@platformatic/vfs` adapter) for a familiar node:fs-shaped surface; this is what `@cloudflare/computerd` mounts via FUSE. A higher-level DO-side `Workspace` class with the `fs`/`shell`/`push`/`pull` surface described in [`../../docs/README.md`](../../docs/README.md) is still future work. - -## Implementation status - - -- `Database` wrapper around Durable Object SQL storage in place. -- Schema initialization for the documented `vfs_*` tables (FS and sync) implemented and split into `schema/core.ts` + `schema/sync.ts`. -- `incrementRev()` shared sequencer in place. FS writes stamp the returned value into `vfs_nodes.rev` and pass it to `sync/changes.ts` for tombstones. -- `SQLiteTestStorage` (backed by `node:sqlite`) available from `./testing` for unit tests against a real in-memory database; `RecordingStorage` available from the package root for workerd-safe schema assertions. -- All filesystem primitives listed above are implemented and unit-tested. -- `SQLiteWorkspaceProvider` (the `@platformatic/vfs` adapter) implemented and exported from the package entrypoint; consumed by `@cloudflare/computerd`. -- Buffered-write surface for the FUSE driver: `createFileSync`, - `writeRangeSync`, `truncateFileSync`, `readRangeSync`, `chmodSync`, - `openWriteBufferSync`, `openWriteBufferForCreateSync`, and - `releaseWriteBufferSync` on the provider. The driver opens a buffer - on FUSE create/open, mutates it through subsequent writes and - truncates, and commits chunks in one transaction at release time. - Reads against the same database see the buffered bytes immediately. -- Content-addressed blob cache: `readFile`, `readRangeSync`, - `provider.readFileSync`, and the partial-chunk read-modify-write - helper share a per-`Database` LRU keyed by `vfs_blob_bytes.hash`. - Repeated reads of dedup'd chunks (a file of zeroes, a re-used - package payload) skip SQLite after the first fetch. -- Sync protocol building blocks implemented and exported; the typed RPC surface on top of them lives in `@cloudflare/computer-rpc`. diff --git a/spikes/349-dofs/vendor/dofs/package-lock.json b/spikes/349-dofs/vendor/dofs/package-lock.json deleted file mode 100644 index 2b3bf5ee..00000000 --- a/spikes/349-dofs/vendor/dofs/package-lock.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "name": "@cloudflare/dofs", - "version": "0.0.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "@cloudflare/dofs", - "version": "0.0.0", - "devDependencies": { - "@cloudflare/workers-types": "^4.20260616.1", - "@types/node": "^24", - "typescript": "^6.0.3" - } - }, - "node_modules/@cloudflare/workers-types": { - "version": "4.20260702.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20260702.1.tgz", - "integrity": "sha512-mOhf5TUEB1m2vPrxtqoIGfz0fUC9xyxRDx5gWHy5s+OCo6dcV+g7wI1R7gYCMFohhqF/2y2xeKVwMwCJjfn/WA==", - "dev": true, - "license": "MIT OR Apache-2.0" - }, - "node_modules/@types/node": { - "version": "24.13.3", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz", - "integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~7.18.0" - } - }, - "node_modules/typescript": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", - "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=14.17" - } - }, - "node_modules/undici-types": { - "version": "7.18.2", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", - "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", - "dev": true, - "license": "MIT" - } - } -} diff --git a/spikes/349-dofs/vendor/dofs/package.json b/spikes/349-dofs/vendor/dofs/package.json deleted file mode 100644 index 146ae123..00000000 --- a/spikes/349-dofs/vendor/dofs/package.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "name": "@cloudflare/dofs", - "version": "0.0.0", - "private": true, - "type": "module", - "exports": { - ".": { - "types": "./dist/index.d.ts", - "default": "./dist/index.js" - }, - "./testing": { - "types": "./dist/testing.d.ts", - "default": "./dist/testing.js" - }, - "./fs/rename": { - "types": "./dist/fs/rename.d.ts", - "default": "./dist/fs/rename.js" - } - }, - "types": "./dist/index.d.ts", - "scripts": { - "build": "tsc -p tsconfig.build.json", - "typecheck": "tsc -p tsconfig.build.json --noEmit", - "test": "vitest run", - "test:workers": "vitest run --config vitest.config.workers.ts", - "bench": "vitest run --config vitest.config.bench.ts" - }, - "devDependencies": { - "typescript": "^6.0.3", - "@cloudflare/workers-types": "^4.20260616.1", - "@types/node": "^24" - } -} diff --git a/spikes/349-dofs/vendor/dofs/src/bench/counting-storage.ts b/spikes/349-dofs/vendor/dofs/src/bench/counting-storage.ts deleted file mode 100644 index 70797ae9..00000000 --- a/spikes/349-dofs/vendor/dofs/src/bench/counting-storage.ts +++ /dev/null @@ -1,123 +0,0 @@ -// Statement/row counter that wraps a real storage backend. -// -// The benchmark harness runs against a genuine Durable Object -// SqlStorage (see vitest.config.bench.ts). Wall-clock alone can't -// prove *why* an operation is slow, so this decorator sits between the -// `Database` wrapper and the real backend and records every -// `sql.exec` call: how many statements ran, split into reads -// (SELECT/WITH) and writes (INSERT/UPDATE/DELETE/REPLACE), plus a -// best-effort tally of rows touched. -// -// Statement counts are deterministic and backend-independent — they -// are the primary signal for the O(depth) resolution fingerprint -// (`resolveInode` = 1 + 2D statements) and for write-amplification -// analysis (added rows per mutation). Row counts are read opportunist- -// ically off the cursor (the DO backend exposes rowsRead/rowsWritten) -// and are reported as a secondary, best-effort figure. -// -// The decorator forwards transactionSync/transaction straight through -// to the real backend, so real SQLite transaction semantics are -// preserved; only `sql.exec` is instrumented. - -import type { DurableObjectStorageLike, SQLCursorLike, SQLStorageLike } from "../types.js"; - -export interface StatementCounts { - statements: number; - reads: number; - writes: number; - other: number; - rowsRead: number; - rowsWritten: number; -} - -function readNumber(source: unknown, key: string): number | undefined { - const value = (source as Record | null)?.[key]; - return typeof value === "number" ? value : undefined; -} - -export class CountingStorage implements DurableObjectStorageLike { - statements = 0; - reads = 0; - writes = 0; - other = 0; - rowsRead = 0; - rowsWritten = 0; - - readonly sql: SQLStorageLike; - readonly transactionSync?: (closure: () => T) => T; - readonly transaction?: (closure: () => T | Promise) => T | Promise; - - constructor(inner: DurableObjectStorageLike) { - this.sql = { - exec: >( - query: string, - ...bindings: unknown[] - ): SQLCursorLike => { - this.statements += 1; - this.classify(query); - const cursor = inner.sql.exec(query, ...bindings); - // Writes report rowsWritten eagerly after exec on the DO - // backend; run() never iterates the cursor, so capture it here. - const written = readNumber(cursor, "rowsWritten"); - if (written !== undefined) { - this.rowsWritten += written; - } - return { - toArray: (): Row[] => { - const rows = cursor.toArray(); - // rowsRead is only meaningful once the cursor is drained, - // which all() does exactly once. - this.rowsRead += readNumber(cursor, "rowsRead") ?? rows.length; - return rows; - }, - }; - }, - }; - - if (inner.transactionSync !== undefined) { - const delegate = inner.transactionSync.bind(inner); - this.transactionSync = (closure: () => T): T => delegate(closure); - } - if (inner.transaction !== undefined) { - const delegate = inner.transaction.bind(inner); - this.transaction = (closure: () => T | Promise): T | Promise => delegate(closure); - } - } - - private classify(query: string): void { - const head = query.trimStart().slice(0, 6).toLowerCase(); - if (head.startsWith("select") || head.startsWith("with")) { - this.reads += 1; - } else if ( - head.startsWith("insert") || - head.startsWith("update") || - head.startsWith("delete") || - head.startsWith("replac") - ) { - this.writes += 1; - } else { - // SAVEPOINT/RELEASE/PRAGMA/DDL etc. Not part of per-op data cost. - this.other += 1; - } - } - - reset(): void { - this.statements = 0; - this.reads = 0; - this.writes = 0; - this.other = 0; - this.rowsRead = 0; - this.rowsWritten = 0; - } - - snapshot(): StatementCounts { - return { - statements: this.statements, - reads: this.reads, - writes: this.writes, - other: this.other, - rowsRead: this.rowsRead, - rowsWritten: this.rowsWritten, - }; - } -} diff --git a/spikes/349-dofs/vendor/dofs/src/bench/fs-ops.bench.ts b/spikes/349-dofs/vendor/dofs/src/bench/fs-ops.bench.ts deleted file mode 100644 index 38360b96..00000000 --- a/spikes/349-dofs/vendor/dofs/src/bench/fs-ops.bench.ts +++ /dev/null @@ -1,640 +0,0 @@ -// dofs micro-benchmark harness. -// -// Runs under @cloudflare/vitest-pool-workers (see -// vitest.config.bench.ts) so every operation drives a REAL Durable -// Object SqlStorage. It deliberately does NOT use the node -// SQLiteTestStorage fixture: that backend caches prepared statements -// and would understate the per-statement cost this harness exists to -// measure. -// -// Each scenario is reported two ways: -// * ns/op wall-clock, measured against the raw DO SqlStorage. -// * statement + row counts, measured against the same backend -// wrapped in CountingStorage. Statement counts are deterministic -// and are the primary signal — a resolve is one statement -// regardless of depth (fs.stat = 1, provider.statSync = 2), and the -// cold-vs-warm group isolates the CTE cold walk from the cache hit. -// -// Output is a set of tables plus a single-line JSON blob so before/ -// after deltas are easy to capture and diff. Run with: -// npm run bench --workspace @cloudflare/dofs -// (or: npx vitest run --config vitest.config.bench.ts, from the -// package dir). - -import { env, runInDurableObject } from "cloudflare:test"; -import { expect, it } from "vitest"; -import type { TestBindings } from "../../tests/worker.js"; -import { ls } from "../fs/ls.js"; -import { resolveInode } from "../fs/resolve.js"; -import { clearResolveCache } from "../fs/resolveCache.js"; -import { rm } from "../fs/rm.js"; -import { stat } from "../fs/stat.js"; -import { SQLiteWorkspaceProvider } from "../provider.js"; -import { initializeSchema } from "../schema/index.js"; -import { Database } from "../storage.js"; -import type { DurableObjectStorageLike } from "../types.js"; -import { CountingStorage } from "./counting-storage.js"; - -const NOW = (): number => 1000; - -interface RealStorage extends DurableObjectStorageLike { - readonly databaseSize?: number; -} - -function freshStub(): DurableObjectStub { - const ns = (env as unknown as TestBindings).TestStorage; - return ns.get(ns.newUniqueId()); -} - -// Raw real SqlStorage — used for clean wall-clock numbers with no -// counting overhead. -async function withRealDb( - fn: (db: Database, provider: SQLiteWorkspaceProvider, storage: RealStorage) => T, -): Promise { - const stub = freshStub(); - return runInDurableObject(stub, async (_instance: unknown, state: DurableObjectState) => { - const storage = state.storage as unknown as DurableObjectStorageLike; - const db = new Database(storage); - initializeSchema(db, NOW); - const provider = new SQLiteWorkspaceProvider(db, { now: NOW }); - return fn(db, provider, (storage as { sql: RealStorage }).sql as unknown as RealStorage); - }); -} - -// Counting wrapper over the same real backend — used for deterministic -// statement/row counts. Schema init is excluded via reset(). -async function withCountingDb( - fn: (db: Database, provider: SQLiteWorkspaceProvider, counting: CountingStorage) => T, -): Promise { - const stub = freshStub(); - return runInDurableObject(stub, async (_instance: unknown, state: DurableObjectState) => { - const counting = new CountingStorage(state.storage as unknown as DurableObjectStorageLike); - const db = new Database(counting); - initializeSchema(db, NOW); - counting.reset(); - const provider = new SQLiteWorkspaceProvider(db, { now: NOW }); - return fn(db, provider, counting); - }); -} - -type Build = (db: Database, provider: SQLiteWorkspaceProvider) => void; -type Op = (db: Database, provider: SQLiteWorkspaceProvider) => void; - -interface ReadResult { - name: string; - depth: number; - nsPerOp: number; - statements: number; - reads: number; - writes: number; -} - -interface MutationResult { - name: string; - items: number; - totalMs: number; - nsPerItem: number; - statements: number; - reads: number; - writes: number; - rowsWritten: number; -} - -// Repeatable read op: build the tree once, then time `op` over `iters`. -async function benchRead( - name: string, - depth: number, - build: Build, - op: Op, - iters: number, -): Promise { - const warmup = Math.min(200, iters); - const nsPerOp = await withRealDb((db, provider) => { - build(db, provider); - for (let i = 0; i < warmup; i++) { - op(db, provider); - } - const t0 = performance.now(); - for (let i = 0; i < iters; i++) { - op(db, provider); - } - const t1 = performance.now(); - return ((t1 - t0) * 1e6) / iters; - }); - const counts = await withCountingDb((db, provider, counting) => { - build(db, provider); - counting.reset(); - op(db, provider); - return counting.snapshot(); - }); - return { - name, - depth, - nsPerOp, - statements: counts.statements, - reads: counts.reads, - writes: counts.writes, - }; -} - -// One-shot mutation batch: build fresh state, run the whole batch once, -// report totals and per-item amortized cost. -async function benchMutation( - name: string, - build: Build, - batch: (db: Database, provider: SQLiteWorkspaceProvider) => number, -): Promise { - const timing = await withRealDb((db, provider) => { - build(db, provider); - const t0 = performance.now(); - const items = batch(db, provider); - const t1 = performance.now(); - return { totalMs: t1 - t0, items }; - }); - const counts = await withCountingDb((db, provider, counting) => { - build(db, provider); - counting.reset(); - batch(db, provider); - return counting.snapshot(); - }); - const items = Math.max(1, timing.items); - return { - name, - items: timing.items, - totalMs: timing.totalMs, - nsPerItem: (timing.totalMs * 1e6) / items, - statements: counts.statements, - reads: counts.reads, - writes: counts.writes, - rowsWritten: counts.rowsWritten, - }; -} - -// --- path helpers ------------------------------------------------- - -function chainOf(depth: number): { dir: string | null; file: string } { - const segs = Array.from({ length: depth }, (_, i) => `s${i + 1}`); - const file = `/${segs.join("/")}`; - const dir = depth > 1 ? `/${segs.slice(0, -1).join("/")}` : null; - return { dir, file }; -} - -function buildChainFile(provider: SQLiteWorkspaceProvider, depth: number, content = "x"): string { - const { dir, file } = chainOf(depth); - if (dir !== null) { - provider.mkdirSync(dir, { recursive: true }); - } - provider.writeFileSync(file, content); - return file; -} - -// --- formatting --------------------------------------------------- - -function ns(value: number): string { - if (value >= 1e6) { - return `${(value / 1e6).toFixed(3)}ms`; - } - if (value >= 1e3) { - return `${(value / 1e3).toFixed(2)}\u00b5s`; - } - return `${value.toFixed(0)}ns`; -} - -function pad(value: string | number, width: number): string { - return String(value).padStart(width); -} - -function padEnd(value: string | number, width: number): string { - return String(value).padEnd(width); -} - -// ------------------------------------------------------------------ - -it("dofs micro-benchmark (real DO SqlStorage)", async () => { - const lines: string[] = []; - const readResults: ReadResult[] = []; - const mutationResults: MutationResult[] = []; - - const depths = [1, 2, 4, 8, 16, 20]; - - // Group A — path-resolution depth sweep, both stat surfaces + a - // flat single-lookup baseline that should stay constant. - for (const depth of depths) { - readResults.push( - await benchRead( - "fs.stat", - depth, - (_db, provider) => { - buildChainFile(provider, depth); - }, - (db) => { - stat(db, chainOf(depth).file); - }, - 4000, - ), - ); - readResults.push( - await benchRead( - "provider.statSync", - depth, - (_db, provider) => { - buildChainFile(provider, depth); - }, - (_db, provider) => { - provider.statSync(chainOf(depth).file); - }, - 4000, - ), - ); - const holder = { inode: 0 }; - readResults.push( - await benchRead( - "flat-baseline(inode)", - depth, - (db, provider) => { - const file = buildChainFile(provider, depth); - holder.inode = resolveInode(db, file)?.inode ?? 0; - }, - (db) => { - db.one( - "SELECT inode, type, mode, mtime, size FROM vfs_nodes WHERE inode = ?", - holder.inode, - ); - }, - 4000, - ), - ); - } - - // Group B — exists, present vs missing leaf, at two depths. - for (const depth of [8, 16]) { - readResults.push( - await benchRead( - "exists(present)", - depth, - (_db, provider) => { - buildChainFile(provider, depth); - }, - (_db, provider) => { - provider.existsSync(chainOf(depth).file); - }, - 4000, - ), - ); - readResults.push( - await benchRead( - "exists(missing)", - depth, - (_db, provider) => { - const { dir } = chainOf(depth); - if (dir !== null) { - provider.mkdirSync(dir, { recursive: true }); - } - }, - (_db, provider) => { - provider.existsSync(chainOf(depth).file); - }, - 4000, - ), - ); - } - - // Group C — read paths at a representative depth. - { - const depth = 8; - const content = "x".repeat(4096); - readResults.push( - await benchRead( - "readFile(4KiB)", - depth, - (_db, provider) => { - buildChainFile(provider, depth, content); - }, - (_db, provider) => { - provider.readFileSync(chainOf(depth).file, "utf8"); - }, - 2000, - ), - ); - readResults.push( - await benchRead( - "readRange(1KiB)", - depth, - (_db, provider) => { - buildChainFile(provider, depth, content); - }, - (_db, provider) => { - provider.readRangeSync(chainOf(depth).file, 0, 1024); - }, - 2000, - ), - ); - } - - // Group D — directory listing: readdir (single dir) vs ls (resolves - // the prefix to its inode, then walks that subtree). - { - const width = 200; - const buildWide: Build = (_db, provider) => { - provider.mkdirSync("/wide", { recursive: true }); - for (let i = 0; i < width; i++) { - provider.writeFileSync(`/wide/f${i}.txt`, "x"); - } - }; - readResults.push( - await benchRead( - `readdir(${width})`, - 1, - buildWide, - (_db, provider) => { - provider.readdirSync("/wide"); - }, - 1000, - ), - ); - readResults.push( - await benchRead( - `ls(${width})`, - 1, - buildWide, - (db) => { - ls(db, "/wide"); - }, - 500, - ), - ); - } - - // Group E — recursive delete of a populated tree. - { - const files = 2000; - mutationResults.push( - await benchMutation( - `recursive-delete(${files})`, - (_db, provider) => { - provider.mkdirSync("/tree", { recursive: true }); - for (let i = 0; i < files; i++) { - provider.writeFileSync(`/tree/f${i}.txt`, "x"); - } - }, - (db) => { - rm(db, "/tree", { recursive: true, force: true }); - return files; - }, - ), - ); - } - - // Group F — write-heavy burst (agent edit session): create, then - // edit-in-place (overwrite), then delete N files at depth. - { - const depth = 8; - const count = 1000; - const { dir } = chainOf(depth); - const base = dir ?? ""; - const ensureDir: Build = (_db, provider) => { - if (dir !== null) { - provider.mkdirSync(dir, { recursive: true }); - } - }; - const createAll = (provider: SQLiteWorkspaceProvider, value: string): void => { - for (let i = 0; i < count; i++) { - provider.writeFileSync(`${base}/burst${i}.txt`, value); - } - }; - mutationResults.push( - await benchMutation(`write-burst:create(${count})`, ensureDir, (_db, provider) => { - createAll(provider, "x"); - return count; - }), - ); - mutationResults.push( - await benchMutation( - `write-burst:edit-in-place(${count})`, - (_db, provider) => { - ensureDir(_db, provider); - createAll(provider, "x"); - }, - (_db, provider) => { - createAll(provider, "yy"); - return count; - }, - ), - ); - mutationResults.push( - await benchMutation( - `write-burst:delete(${count})`, - (_db, provider) => { - ensureDir(_db, provider); - createAll(provider, "x"); - }, - (_db, provider) => { - for (let i = 0; i < count; i++) { - provider.unlinkSync(`${base}/burst${i}.txt`); - } - return count; - }, - ), - ); - } - - // Group G — rename: many single-file renames vs one subtree rename. - { - const count = 1000; - mutationResults.push( - await benchMutation( - `single-rename(${count})`, - (_db, provider) => { - provider.mkdirSync("/mv", { recursive: true }); - for (let i = 0; i < count; i++) { - provider.writeFileSync(`/mv/a${i}.txt`, "x"); - } - }, - (_db, provider) => { - for (let i = 0; i < count; i++) { - provider.renameSync(`/mv/a${i}.txt`, `/mv/b${i}.txt`); - } - return count; - }, - ), - ); - const descendants = 500; - mutationResults.push( - await benchMutation( - `subtree-rename(${descendants})`, - (_db, provider) => { - provider.mkdirSync("/sub/inner", { recursive: true }); - for (let i = 0; i < descendants; i++) { - provider.writeFileSync(`/sub/inner/f${i}.txt`, "x"); - } - }, - (_db, provider) => { - provider.renameSync("/sub", "/moved"); - return descendants; - }, - ), - ); - } - - // Group H — DB size / dedup guard: 100 identical 1 MiB files should - // dedup to ~1 MiB of blob bytes plus small metadata. - const dedup = await withRealDb((_db, provider, storage) => { - const oneMiB = "a".repeat(1024 * 1024); - provider.mkdirSync("/dup", { recursive: true }); - for (let i = 0; i < 100; i++) { - provider.writeFileSync(`/dup/f${i}.bin`, oneMiB); - } - return { bytes: storage.databaseSize ?? 0, files: 100, logicalMiB: 100 }; - }); - - // Group I — cold vs warm resolve: isolate the CTE cold walk from the - // cache hit. Cold clears the cache before every timed op (a fresh - // single-statement CTE that reads D rows internally); warm leaves it - // primed (a single readNode, O(1)). Same fs.stat, shallow and deep. - interface ColdWarmResult { - name: string; - depth: number; - coldNsPerOp: number; - warmNsPerOp: number; - } - const coldWarmResults: ColdWarmResult[] = []; - for (const depth of [4, 20]) { - const measured = await withRealDb((db, provider) => { - buildChainFile(provider, depth); - const file = chainOf(depth).file; - const iters = 4000; - const warmup = 200; - for (let i = 0; i < warmup; i++) { - clearResolveCache(db); - stat(db, file); - } - const cold0 = performance.now(); - for (let i = 0; i < iters; i++) { - clearResolveCache(db); - stat(db, file); - } - const cold1 = performance.now(); - for (let i = 0; i < warmup; i++) { - stat(db, file); - } - const warm0 = performance.now(); - for (let i = 0; i < iters; i++) { - stat(db, file); - } - const warm1 = performance.now(); - return { - cold: ((cold1 - cold0) * 1e6) / iters, - warm: ((warm1 - warm0) * 1e6) / iters, - }; - }); - coldWarmResults.push({ - name: "fs.stat", - depth, - coldNsPerOp: measured.cold, - warmNsPerOp: measured.warm, - }); - } - - // --- render report ------------------------------------------------ - - lines.push("=".repeat(96)); - lines.push( - "dofs micro-benchmark — backend: REAL Durable Object SqlStorage (vitest-pool-workers)", - ); - lines.push(`generated: ${new Date().toISOString()}`); - lines.push( - "note: statement counts are deterministic; ns/op is wall-clock under workerd. " + - "resolve = 1 statement (cold CTE or warm cache), depth-independent. " + - "Depth-sweep ns/op below is cache-warm; see COLD-VS-WARM for the CTE cold walk.", - ); - lines.push("=".repeat(96)); - - lines.push(""); - lines.push("READ / RESOLVE OPS"); - lines.push( - `${padEnd("operation", 22)}${pad("depth", 6)}${pad("ns/op", 12)}${pad("stmts", 8)}${pad("reads", 7)}${pad("writes", 8)}`, - ); - lines.push("-".repeat(63)); - for (const r of readResults) { - lines.push( - `${padEnd(r.name, 22)}${pad(r.depth, 6)}${pad(ns(r.nsPerOp), 12)}${pad(r.statements, 8)}${pad(r.reads, 7)}${pad(r.writes, 8)}`, - ); - } - - lines.push(""); - lines.push("MUTATION BATCHES (per-item amortized)"); - lines.push( - `${padEnd("operation", 30)}${pad("items", 7)}${pad("total", 11)}${pad("ns/item", 12)}${pad("stmts", 8)}${pad("writes", 8)}${pad("rowsWr", 8)}`, - ); - lines.push("-".repeat(84)); - for (const m of mutationResults) { - lines.push( - `${padEnd(m.name, 30)}${pad(m.items, 7)}${pad(`${m.totalMs.toFixed(1)}ms`, 11)}${pad(ns(m.nsPerItem), 12)}${pad(m.statements, 8)}${pad(m.writes, 8)}${pad(m.rowsWritten, 8)}`, - ); - } - - lines.push(""); - lines.push("COLD (CTE walk) VS WARM (cached) RESOLVE — fs.stat"); - lines.push( - `${padEnd("operation", 22)}${pad("depth", 6)}${pad("cold ns/op", 12)}${pad("warm ns/op", 12)}`, - ); - lines.push("-".repeat(52)); - for (const c of coldWarmResults) { - lines.push( - `${padEnd(c.name, 22)}${pad(c.depth, 6)}${pad(ns(c.coldNsPerOp), 12)}${pad(ns(c.warmNsPerOp), 12)}`, - ); - } - - lines.push(""); - lines.push("DB SIZE / DEDUP GUARD"); - lines.push( - `100 x 1 MiB identical files -> logical ${dedup.logicalMiB} MiB, on-disk ${(dedup.bytes / (1024 * 1024)).toFixed(2)} MiB (${dedup.bytes} bytes)`, - ); - - lines.push(""); - lines.push("JSON"); - lines.push( - JSON.stringify({ - backend: "durable-object-sqlstorage", - reads: readResults, - mutations: mutationResults, - coldWarm: coldWarmResults, - dedup, - }), - ); - lines.push("=".repeat(96)); - - console.log(`\n${lines.join("\n")}\n`); - - // Signature gate. Statement counts are deterministic, so the O(depth) - // fingerprint doubles as the harness's contract. Asserted AFTER the - // report is printed so the numbers stay visible even when the gate - // trips. A deliberate perf change is expected to update these, which - // is the point — silent drift becomes a failure. - // - // fs.stat = 1 — one recursive-CTE resolve. - // provider.statSync = 2 — one resolve plus linkCount. - // flat-baseline = 1 — a single indexed inode lookup. - // - // The CTE still reads O(depth) rows internally, but the statement - // count — what the DO bills and round-trips — is depth-independent, - // and a warm cache hit re-reads just the one node row. - const find = (name: string, depth: number): ReadResult => { - const row = readResults.find((r) => r.name === name && r.depth === depth); - if (row === undefined) { - throw new Error(`benchmark result missing: ${name} depth=${depth}`); - } - return row; - }; - for (const depth of depths) { - expect(find("fs.stat", depth).statements, `fs.stat depth=${depth}`).toBe(1); - expect(find("provider.statSync", depth).statements, `provider.statSync depth=${depth}`).toBe(2); - expect(find("flat-baseline(inode)", depth).statements, `flat-baseline depth=${depth}`).toBe(1); - } - // exists resolves in a single CTE statement too, whether the leaf is - // present or missing. - expect(find("exists(present)", 8).statements).toBe(1); - expect(find("exists(missing)", 8).statements).toBe(1); - // Dedup guarantee: 100 identical 1 MiB files must not balloon the DB. - expect(dedup.bytes).toBeLessThan(2 * 1024 * 1024); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/errors.ts b/spikes/349-dofs/vendor/dofs/src/errors.ts deleted file mode 100644 index 1c21257d..00000000 --- a/spikes/349-dofs/vendor/dofs/src/errors.ts +++ /dev/null @@ -1,36 +0,0 @@ -export type WorkspaceErrorCode = - | "ENOENT" - | "ENOTEMPTY" - | "ENOTDIR" - | "EISDIR" - | "EEXIST" - | "EINVAL" - | "EACCES" - | "EPERM" - | "EROFS" - | "ENOSYS" - | "EBADF" - | "ELOOP" - | "EUNKNOWN_HASH" - | "EIO"; - -export interface WorkspaceFsError extends Error { - code: WorkspaceErrorCode; - path?: string; -} - -export function createWorkspaceError( - code: WorkspaceErrorCode, - message: string, - path?: string, -): WorkspaceFsError { - const error = new Error(path === undefined ? message : `${message}: ${path}`) as WorkspaceFsError; - error.name = "WorkspaceFsError"; - error.code = code; - error.path = path; - return error; -} - -export function invalidPath(path: string, reason: string): WorkspaceFsError { - return createWorkspaceError("EINVAL", `Invalid path (${reason})`, path); -} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/blobCache.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/blobCache.test.ts deleted file mode 100644 index 9b2f5729..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/blobCache.test.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; -import { clearBlobCache, getBlobBytes } from "./blobCache.js"; -import { readRangeSync } from "./readFile.js"; -import { withDB } from "./with-db.js"; -import { CHUNK_SIZE, writeFileSync } from "./writeFile.js"; - -describe("blobCache", () => { - it("reuses bytes for the same hash across calls", async () => { - await withDB(async (db) => { - writeFileSync(db, "/seed.bin", new Uint8Array(CHUNK_SIZE).fill(7), {}, () => 1); - // Pull the chunk hash out of vfs_chunks so we can hit the - // cache helper directly without going through readFile. - const row = db.one<{ hash: Uint8Array }>( - "SELECT hash FROM vfs_chunks WHERE inode = (SELECT child_inode FROM vfs_dirents WHERE name = ?)", - "seed.bin", - ); - expect(row).toBeDefined(); - const hash = row?.hash as Uint8Array; - - // First call populates the cache; the second returns the - // exact same Uint8Array reference rather than re-querying. - const first = getBlobBytes(db, hash); - expect(first).toBeInstanceOf(Uint8Array); - const second = getBlobBytes(db, hash); - expect(second).toBe(first); - }); - }); - - it("evicts the least-recently-used entry once 17 distinct hashes are cached", async () => { - await withDB(async (db) => { - // Stage 17 files with distinct content so each one gets a - // unique blob hash. The cache holds 16; the 17th fetch must - // evict the least-recently-used, which is the first hash. - const hashes: Uint8Array[] = []; - for (let i = 0; i < 17; i++) { - const path = `/distinct-${i}.bin`; - writeFileSync(db, path, new TextEncoder().encode(`payload-${i}`), {}, () => 1); - const row = db.one<{ hash: Uint8Array }>( - "SELECT hash FROM vfs_chunks WHERE inode = (SELECT child_inode FROM vfs_dirents WHERE name = ?)", - `distinct-${i}.bin`, - ); - hashes.push(row?.hash as Uint8Array); - } - clearBlobCache(db); - - // First 16 fetches populate the cache. - for (let i = 0; i < 16; i++) getBlobBytes(db, hashes[i]); - // 17th fetch evicts the LRU (entry 0); spy to confirm the - // 18th fetch of hash 0 is a fresh SQL lookup, but hash 16 is - // cached and the 18th fetch of hash 1 (still the LRU) hits - // SQL too. - getBlobBytes(db, hashes[16]); - - const spy = vi.spyOn(db, "one"); - getBlobBytes(db, hashes[0]); // evicted, must hit SQL - getBlobBytes(db, hashes[16]); // hot, must NOT hit SQL - const lookups = spy.mock.calls.filter( - ([query]) => typeof query === "string" && query.includes("vfs_blob_bytes"), - ).length; - spy.mockRestore(); - - expect(lookups).toBe(1); - }); - }); - - it("moves a touched entry to most-recent so it survives eviction", async () => { - await withDB(async (db) => { - const hashes: Uint8Array[] = []; - for (let i = 0; i < 17; i++) { - const path = `/touched-${i}.bin`; - writeFileSync(db, path, new TextEncoder().encode(`touched-${i}`), {}, () => 1); - const row = db.one<{ hash: Uint8Array }>( - "SELECT hash FROM vfs_chunks WHERE inode = (SELECT child_inode FROM vfs_dirents WHERE name = ?)", - `touched-${i}.bin`, - ); - hashes.push(row?.hash as Uint8Array); - } - clearBlobCache(db); - - for (let i = 0; i < 16; i++) getBlobBytes(db, hashes[i]); - // Touch hash 0 so it becomes most-recent. - getBlobBytes(db, hashes[0]); - // The 17th fetch should now evict hash 1, not hash 0. - getBlobBytes(db, hashes[16]); - - const spy = vi.spyOn(db, "one"); - getBlobBytes(db, hashes[0]); // touched, still cached - getBlobBytes(db, hashes[1]); // evicted, must hit SQL - const lookups = spy.mock.calls.filter( - ([query]) => typeof query === "string" && query.includes("vfs_blob_bytes"), - ).length; - spy.mockRestore(); - - expect(lookups).toBe(1); - }); - }); - - it("readRangeSync avoids repeating vfs_blob_bytes lookups on sequential reads", async () => { - await withDB(async (db) => { - // 4 MiB of repeated content → one dedup'd blob in the store. - // A sequential read in 128 KiB windows used to issue one - // SELECT bytes per window, even though every window came out - // of the same blob. - const payload = new Uint8Array(CHUNK_SIZE * 8).fill(3); - writeFileSync(db, "/big.bin", payload, {}, () => 1); - clearBlobCache(db); - - const spy = vi.spyOn(db, "one"); - const window = 128 * 1024; - for (let offset = 0; offset < payload.byteLength; offset += window) { - readRangeSync(db, "/big.bin", offset, window); - } - const blobLookups = spy.mock.calls.filter( - ([query]) => typeof query === "string" && query.includes("vfs_blob_bytes"), - ).length; - spy.mockRestore(); - - // 8 distinct chunks were written and they all share one - // hash; we should fetch the blob bytes at most once. - expect(blobLookups).toBeLessThanOrEqual(1); - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/blobCache.ts b/spikes/349-dofs/vendor/dofs/src/fs/blobCache.ts deleted file mode 100644 index b997ee82..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/blobCache.ts +++ /dev/null @@ -1,87 +0,0 @@ -// In-process LRU cache of vfs_blob_bytes payloads, keyed by hash. -// -// FUSE reads up to 128 KiB at a time (the kernel's default max_read); -// our chunk size is 512 KiB. A sequential read of a chunk-backed file -// re-fetches the same blob 4x by default. Worse, a 64 MiB file of -// repeated content (e.g. `dd if=/dev/zero`) deduplicates to a single -// blob in vfs_blobs, and we then re-fetch that one blob 512 times -// over the lifetime of one read pass. -// -// vfs_blob_bytes is content-addressed. The normal write path -// (upsertChunkBlob) uses ON CONFLICT DO NOTHING, so a correct -// (hash, bytes) pair is never overwritten and the cache stays valid -// for it. The one exception is repair: stageBlob (the sync receiver -// path) uses ON CONFLICT DO UPDATE SET bytes to replace an incomplete -// or size-mismatched payload left by an interrupted or corrupt write, -// and clears this cache afterward so a stale payload is never served -// after a repair. -// -// The cache is bounded (CHUNK_CACHE_MAX_ENTRIES) and per-Database so -// independent test databases don't pollute each other. Eviction is -// LRU; access moves an entry to the most-recent position. - -import type { Database } from "../storage.js"; - -// Number of distinct blob payloads kept in memory per Database. -// At 512 KiB per blob this caps the cache at ~8 MiB, large enough -// to hold a handful of hot chunks for sequential reads of large -// files without dominating process memory. -const CHUNK_CACHE_MAX_ENTRIES = 16; - -const caches = new WeakMap>(); - -function cacheFor(db: Database): Map { - let cache = caches.get(db); - if (cache === undefined) { - cache = new Map(); - caches.set(db, cache); - } - return cache; -} - -// Stringify a 32-byte hash so it can key a JS Map. Latin-1 -// preserves every byte exactly and avoids the allocation cost of -// hex encoding for what is a very hot path. -function hashKey(hash: Uint8Array): string { - let out = ""; - for (let i = 0; i < hash.byteLength; i++) { - out += String.fromCharCode(hash[i]); - } - return out; -} - -// Look up blob bytes by hash. Cache hit returns the cached -// Uint8Array directly (callers must not mutate it). Cache miss -// queries vfs_blob_bytes and stores the result. Returns undefined -// if the blob isn't in the store. -export function getBlobBytes(db: Database, hash: Uint8Array): Uint8Array | undefined { - const cache = cacheFor(db); - const key = hashKey(hash); - const cached = cache.get(key); - if (cached !== undefined) { - // Reinsert to move to the most-recent position. Map iteration - // order is insertion order, so this gives us LRU eviction for - // free without a separate doubly-linked list. - cache.delete(key); - cache.set(key, cached); - return cached; - } - const row = db.one<{ bytes: Uint8Array }>( - "SELECT bytes FROM vfs_blob_bytes WHERE hash = ?", - hash, - ); - if (row === undefined) return undefined; - cache.set(key, row.bytes); - while (cache.size > CHUNK_CACHE_MAX_ENTRIES) { - const first = cache.keys().next(); - if (first.done === true) break; - cache.delete(first.value); - } - return row.bytes; -} - -// Reset the cache for `db`. Tests use this to keep cache state from -// leaking between cases that share a Database constructor pattern. -export function clearBlobCache(db: Database): void { - caches.delete(db); -} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/chmod.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/chmod.test.ts deleted file mode 100644 index 68cd7d08..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/chmod.test.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { chmod } from "./chmod.js"; -import { mkdir } from "./mkdir.js"; -import { invalidateReadOnlyMountCache } from "./mount-guard.js"; -import { resolveInode } from "./resolve.js"; -import { symlink } from "./symlink.js"; -import { withDB } from "./with-db.js"; -import { writeFileSync } from "./writeFile.js"; - -const utf8 = (s: string): Uint8Array => new TextEncoder().encode(s); - -describe("chmod", () => { - it("updates the mode on a regular file", async () => { - await withDB((db) => { - writeFileSync(db, "/a.txt", utf8("hi"), {}, () => 1000); - chmod(db, "/a.txt", 0o600, () => 2000); - expect(resolveInode(db, "/a.txt")?.mode).toBe(0o600); - }); - }); - - it("updates the mode on a directory", async () => { - await withDB((db) => { - mkdir(db, "/d", { mode: 0o755 }, () => 1000); - chmod(db, "/d", 0o700, () => 2000); - expect(resolveInode(db, "/d")?.mode).toBe(0o700); - }); - }); - - it("masks the supplied mode to twelve bits", async () => { - // POSIX chmod truncates anything above 07777. Mirror that here - // so callers can hand us a Node-style stat.mode (which carries - // file-type bits in the upper byte) without corrupting the - // stored permissions. - await withDB((db) => { - writeFileSync(db, "/a", utf8("hi"), {}, () => 0); - chmod(db, "/a", 0o100644, () => 0); - expect(resolveInode(db, "/a")?.mode).toBe(0o644); - }); - }); - - it("bumps rev and stamps it onto the node", async () => { - await withDB((db) => { - writeFileSync(db, "/a", utf8("hi"), {}, () => 0); - const beforeRev = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'"); - chmod(db, "/a", 0o600, () => 0); - const afterRev = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'"); - expect(afterRev).toBe((beforeRev ?? 0) + 1); - const nodeRev = db.scalar( - "SELECT n.rev FROM vfs_nodes n JOIN vfs_dirents d ON d.child_inode = n.inode WHERE name = ?", - "a", - ); - expect(nodeRev).toBe(afterRev); - }); - }); - - it("updates mtime to now()", async () => { - await withDB((db) => { - writeFileSync(db, "/a", utf8("hi"), {}, () => 1000); - chmod(db, "/a", 0o600, () => 5000); - expect(resolveInode(db, "/a")?.mtime).toBe(5000); - }); - }); - - it("follows symlinks by default — POSIX chmod semantics", async () => { - // chmod("/link") changes the mode of the target, not the link. - // POSIX symlinks carry mode 0o777 and chmod against the link - // itself is a no-op on most kernels. We mirror that. - await withDB((db) => { - writeFileSync(db, "/target", utf8("hi"), {}, () => 0); - symlink(db, "/target", "/link", () => 0); - chmod(db, "/link", 0o600, () => 0); - expect(resolveInode(db, "/target")?.mode).toBe(0o600); - // The symlink node itself stays at 0o777. - expect(resolveInode(db, "/link", { followSymlinks: false })?.mode).toBe(0o777); - }); - }); - - it("rejects ENOENT for a missing path", async () => { - await withDB((db) => { - expect(() => chmod(db, "/missing", 0o600, () => 0)).toThrowError( - expect.objectContaining({ code: "ENOENT" }), - ); - }); - }); - - it("rejects EROFS when the path overlaps a read-only mount root", async () => { - await withDB((db) => { - // Stage the read-only mount through the same SQL the indexer - // uses, then materialise a directory under it so chmod has - // something to land on. invalidateReadOnlyMountCache mirrors - // what the indexer does after writing _vfs_mounts. - db.run("INSERT INTO _vfs_mounts (root, kind, mode) VALUES ('/mnt', 'r2', 'read-only')"); - invalidateReadOnlyMountCache(db); - // Directly stage a directory under the mount root via SQL - // since mkdir() would itself reject under the read-only - // guard. - db.run("INSERT INTO vfs_nodes (type, mode, mtime, rev) VALUES ('dir', 493, 0, 0)"); - const inode = db.scalar("SELECT last_insert_rowid()"); - db.run( - "INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (1, 'mnt', ?)", - inode, - ); - expect(() => chmod(db, "/mnt", 0o755, () => 0)).toThrowError( - expect.objectContaining({ code: "EROFS" }), - ); - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/chmod.ts b/spikes/349-dofs/vendor/dofs/src/fs/chmod.ts deleted file mode 100644 index 5c0800fb..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/chmod.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { createWorkspaceError } from "../errors.js"; -import { canonicalizePath } from "../path.js"; -import { incrementRev } from "../rev.js"; -import type { Database } from "../storage.js"; -import { assertNotReadOnly } from "./mount-guard.js"; -import { resolveInode } from "./resolve.js"; - -// Change the file mode bits of a path. Follows symlinks like POSIX -// chmod — the change lands on the target, not the link. Bumps rev -// and mtime so the sync protocol carries the change. -// -// The supplied mode is masked to 12 bits (the permission bits and -// the setuid / setgid / sticky bits). Callers that pass a Node-style -// stat.mode with file-type bits in the upper byte get only the -// permission half stored. -export function chmod(db: Database, path: string, mode: number, now: () => number): void { - const { path: canonical } = canonicalizePath(path); - assertNotReadOnly(db, canonical); - - db.transactionSync(() => { - const node = resolveInode(db, canonical); - if (node === null) { - throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); - } - const rev = incrementRev(db); - db.run( - "UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ? WHERE inode = ?", - mode & 0o7777, - now(), - rev, - node.inode, - ); - }); -} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/filesystem.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/filesystem.test.ts deleted file mode 100644 index 3e2842d3..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/filesystem.test.ts +++ /dev/null @@ -1,174 +0,0 @@ -// Smoke tests for the WorkspaceFilesystem class wrapper. -// -// The class is a thin forward to the free fs/* functions, so per-op -// behaviour is already covered by neighbouring tests -// (stat.test.ts, readdir.test.ts, ...). What's tested here is the -// wrapper itself: methods land on the right free function, the -// (db, now) pair gets threaded through, and the documented shape -// at the class boundary matches the free functions. - -import { describe, expect, it } from "vitest"; - -import { initializeSchema } from "../schema/index.js"; -import { Database } from "../storage.js"; -import { SQLiteTestStorage } from "../testing.js"; -import { WorkspaceFilesystem } from "./filesystem.js"; - -async function withFs( - fn: (fs: WorkspaceFilesystem) => T | Promise, - now: () => number = () => 1234, -): Promise { - const storage = new SQLiteTestStorage(); - const db = new Database(storage); - initializeSchema(db, now); - const fs = new WorkspaceFilesystem(db, { now }); - try { - return await fn(fs); - } finally { - storage.close(); - } -} - -describe("WorkspaceFilesystem", () => { - it("writeFile and readFile round-trip utf8", async () => { - await withFs(async (fs) => { - await fs.writeFile("/hello.txt", "hello world"); - expect(await fs.readFile("/hello.txt", "utf8")).toBe("hello world"); - }); - }); - - it("readFile returns a stream by default", async () => { - await withFs(async (fs) => { - await fs.writeFile("/bin", new Uint8Array([1, 2, 3, 4])); - const stream = await fs.readFile("/bin"); - expect(stream).toBeInstanceOf(ReadableStream); - const buf = new Uint8Array(await new Response(stream).arrayBuffer()); - expect(Array.from(buf)).toEqual([1, 2, 3, 4]); - }); - }); - - it("stat returns the documented shape for a file", async () => { - await withFs(async (fs) => { - await fs.writeFile("/a.txt", "ab"); - const s = await fs.stat("/a.txt"); - expect(s).toMatchObject({ - name: "a.txt", - size: 2, - isFile: true, - isDirectory: false, - }); - }); - }); - - it("stat throws ENOENT for a missing path", async () => { - await withFs(async (fs) => { - await expect(fs.stat("/missing")).rejects.toMatchObject({ code: "ENOENT" }); - }); - }); - - it("mkdir creates a directory that stat recognises", async () => { - await withFs(async (fs) => { - await fs.mkdir("/d"); - const s = await fs.stat("/d"); - expect(s.isDirectory).toBe(true); - }); - }); - - it("readdir lists immediate children only", async () => { - await withFs(async (fs) => { - await fs.mkdir("/d"); - await fs.writeFile("/d/a.txt", "a"); - await fs.writeFile("/d/b.txt", "b"); - await fs.mkdir("/d/sub"); - await fs.writeFile("/d/sub/deep.txt", "deep"); - const names = (await fs.readdir("/d")).map((e) => e.name).sort(); - expect(names).toEqual(["a.txt", "b.txt", "sub"]); - }); - }); - - it("find walks subtrees and ls flattens paths", async () => { - await withFs(async (fs) => { - await fs.mkdir("/p"); - await fs.writeFile("/p/a.txt", "a"); - await fs.mkdir("/p/q"); - await fs.writeFile("/p/q/b.txt", "b"); - - const found = (await fs.find("/p")).map((e) => e.path).sort(); - expect(found).toContain("/p/a.txt"); - expect(found).toContain("/p/q/b.txt"); - - const flat = (await fs.ls("/p")).sort(); - expect(flat).toContain("/p/a.txt"); - expect(flat).toContain("/p/q/b.txt"); - }); - }); - - it("grep finds matching lines", async () => { - await withFs(async (fs) => { - await fs.writeFile("/notes.txt", "alpha\nbeta\ngamma\n"); - const hits = await fs.grep("beta", "/notes.txt"); - expect(hits).toHaveLength(1); - expect(hits[0]?.text).toBe("beta"); - }); - }); - - it("rm removes a file; rm with recursive removes a directory tree", async () => { - await withFs(async (fs) => { - await fs.writeFile("/x.txt", "x"); - await fs.rm("/x.txt"); - await expect(fs.stat("/x.txt")).rejects.toMatchObject({ code: "ENOENT" }); - - await fs.mkdir("/tree"); - await fs.writeFile("/tree/inner.txt", "i"); - await fs.rm("/tree", { recursive: true }); - await expect(fs.stat("/tree")).rejects.toMatchObject({ code: "ENOENT" }); - }); - }); - - it("chmod updates the stored mode", async () => { - await withFs(async (fs) => { - await fs.writeFile("/a", "hi"); - await fs.chmod("/a", 0o600); - expect((await fs.stat("/a")).mode).toBe(0o600); - }); - }); - - it("symlink + readlink round-trip", async () => { - await withFs(async (fs) => { - await fs.writeFile("/target", "hi"); - await fs.symlink("/target", "/link"); - expect(await fs.readlink("/link")).toBe("/target"); - }); - }); - - it("stat follows symlinks; lstat reports the link itself", async () => { - await withFs(async (fs) => { - await fs.writeFile("/target", "hello"); - await fs.symlink("/target", "/link"); - const s = await fs.stat("/link"); - expect(s.isFile).toBe(true); - expect(s.isSymbolicLink).toBe(false); - const l = await fs.lstat("/link"); - expect(l.isSymbolicLink).toBe(true); - expect(l.isFile).toBe(false); - expect(l.size).toBe("/target".length); - }); - }); - - it("threads the injected clock through writeFile", async () => { - let t = 5000; - await withFs( - async (fs) => { - await fs.writeFile("/clock.txt", "c"); - const s = await fs.stat("/clock.txt"); - expect(s.mtime).toBe(5000); - - t = 9000; - await fs.writeFile("/clock.txt", "c2"); - const s2 = await fs.stat("/clock.txt"); - expect(s2.mtime).toBe(9000); - }, - () => t, - ); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/filesystem.ts b/spikes/349-dofs/vendor/dofs/src/fs/filesystem.ts deleted file mode 100644 index 82ea588c..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/filesystem.ts +++ /dev/null @@ -1,128 +0,0 @@ -// WorkspaceFilesystem — class wrapper that binds a Database and a -// clock to the free fs/* functions. -// -// Every method here is a thin forward to the matching free -// function. The class exists so callers (host-side Workspace, -// in-container tools, tests) get a single instance to thread -// through their code rather than passing (db, now) pairs into -// every call. -// -// Free functions remain exported for internal callers — the -// apply paths in sync/* operate on a Database directly, and the -// in-package tests skip the class wrapper when they only need a -// single op. - -import type { Database } from "../storage.js"; - -import { chmod } from "./chmod.js"; -import { find, type WorkspaceFoundEntry } from "./find.js"; -import { type GrepOptions, grep, type WorkspaceGrepMatch } from "./grep.js"; -import { ls } from "./ls.js"; -import { type MkdirOptions, mkdir } from "./mkdir.js"; -import { type ReaddirOptions, readdir, type WorkspaceDirentResult } from "./readdir.js"; -import { type ReadFileOptions, readFile } from "./readFile.js"; -import { readlink } from "./readlink.js"; -import { type RmOptions, rm } from "./rm.js"; -import { lstat, stat, type WorkspaceStatResult } from "./stat.js"; -import { symlink } from "./symlink.js"; -import { type WriteFileContent, type WriteFileOptions, writeFile } from "./writeFile.js"; - -export interface WorkspaceFilesystemOptions { - // Clock used for mtime / last_seen. Defaults to Date.now. - // Override for deterministic tests. - now?: () => number; -} - -export class WorkspaceFilesystem { - readonly db: Database; - readonly now: () => number; - - constructor(db: Database, options: WorkspaceFilesystemOptions = {}) { - this.db = db; - this.now = options.now ?? Date.now; - } - - // --- Reads ------------------------------------------------------- - - readFile(path: string): Promise>; - readFile(path: string, encoding: "utf8"): Promise; - readFile(path: string, options: ReadFileOptions): Promise>; - readFile( - path: string, - optionsOrEncoding?: "utf8" | ReadFileOptions, - ): Promise> { - // Forward through the free function's overload set. The - // individual overloads above let callers see the precise - // return type for each input shape. - // Cast through the union overload of the free function; - // the class's overloads above carry the precise return type - // for each input shape back to the caller. - return readFile(this.db, path, optionsOrEncoding as ReadFileOptions); - } - - async stat(path: string): Promise { - return stat(this.db, path); - } - - // POSIX lstat — like stat, but doesn't follow a trailing symlink. - // Use when the caller wants to inspect the link itself: readlink - // / unlink under a Node-style fs surface, or just-bash's adapter - // routing lstat through to the workspace. - async lstat(path: string): Promise { - return lstat(this.db, path); - } - - // Return the stored target of a symlink. EINVAL when path is - // not a symlink; ENOENT when path is missing. - async readlink(path: string): Promise { - return readlink(this.db, path); - } - - async readdir(path: string, options: ReaddirOptions = {}): Promise { - return readdir(this.db, path, options); - } - - async find(directory: string, pattern?: string): Promise { - return find(this.db, directory, pattern); - } - - async ls(prefix: string): Promise { - return ls(this.db, prefix); - } - - grep(pattern: string, path: string, options: GrepOptions = {}): Promise { - return grep(this.db, pattern, path, options); - } - - // --- Mutations --------------------------------------------------- - - writeFile( - path: string, - content: WriteFileContent, - options: WriteFileOptions = {}, - ): Promise { - return writeFile(this.db, path, content, options, this.now); - } - - async mkdir(path: string, options: MkdirOptions = {}): Promise { - mkdir(this.db, path, options, this.now); - } - - async rm(path: string, options: RmOptions = {}): Promise { - rm(this.db, path, options); - } - - // Change the permission bits on a path. Follows symlinks like - // POSIX chmod — the change lands on the target, not the link. - // The supplied mode is masked to twelve bits. - async chmod(path: string, mode: number): Promise { - chmod(this.db, path, mode, this.now); - } - - // Create a symbolic link at `path` pointing at `target`. The - // target is stored verbatim; it can be relative or absolute and - // is allowed to dangle. - async symlink(target: string, path: string): Promise { - symlink(this.db, target, path, this.now); - } -} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/find.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/find.test.ts deleted file mode 100644 index f2e8d064..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/find.test.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { find } from "./find.js"; -import { mkdir } from "./mkdir.js"; -import { withDB } from "./with-db.js"; -import { writeFile } from "./writeFile.js"; - -describe("find", () => { - it("returns nothing for an empty directory", async () => { - await withDB((db) => { - expect(find(db, "/")).toEqual([]); - }); - }); - - it("walks every entry without a pattern", async () => { - await withDB(async (db) => { - mkdir(db, "/a/b", { recursive: true }, () => 0); - await writeFile(db, "/a/x.ts", "", {}, () => 0); - await writeFile(db, "/a/b/y.md", "", {}, () => 0); - const entries = find(db, "/").sort((p, q) => p.path.localeCompare(q.path)); - expect(entries).toEqual([ - { path: "/a", type: "dir" }, - { path: "/a/b", type: "dir" }, - { path: "/a/b/y.md", type: "file" }, - { path: "/a/x.ts", type: "file" }, - ]); - }); - }); - - it("treats an empty pattern like no pattern and returns every entry", async () => { - await withDB(async (db) => { - mkdir(db, "/a/b", { recursive: true }, () => 0); - await writeFile(db, "/a/x.ts", "", {}, () => 0); - await writeFile(db, "/a/b/y.md", "", {}, () => 0); - const entries = find(db, "/", "").sort((p, q) => p.path.localeCompare(q.path)); - expect(entries).toEqual([ - { path: "/a", type: "dir" }, - { path: "/a/b", type: "dir" }, - { path: "/a/b/y.md", type: "file" }, - { path: "/a/x.ts", type: "file" }, - ]); - }); - }); - - it("matches a single-level glob *.ts within the directory only", async () => { - await withDB(async (db) => { - mkdir(db, "/a/b", { recursive: true }, () => 0); - await writeFile(db, "/a/x.ts", "", {}, () => 0); - await writeFile(db, "/a/b/y.ts", "", {}, () => 0); - const paths = find(db, "/a", "*.ts") - .map((e) => e.path) - .sort(); - expect(paths).toEqual(["/a/x.ts"]); - }); - }); - - it("matches ** recursively", async () => { - await withDB(async (db) => { - mkdir(db, "/a/b/c", { recursive: true }, () => 0); - await writeFile(db, "/a/x.md", "", {}, () => 0); - await writeFile(db, "/a/b/y.md", "", {}, () => 0); - await writeFile(db, "/a/b/c/z.md", "", {}, () => 0); - const paths = find(db, "/a", "**/*.md") - .map((e) => e.path) - .sort(); - expect(paths).toEqual(["/a/b/c/z.md", "/a/b/y.md", "/a/x.md"]); - }); - }); - - it("walks from a nested directory", async () => { - await withDB(async (db) => { - mkdir(db, "/a/b/c", { recursive: true }, () => 0); - await writeFile(db, "/a/b/y.ts", "", {}, () => 0); - await writeFile(db, "/a/b/c/z.ts", "", {}, () => 0); - const paths = find(db, "/a/b", "**/*.ts") - .map((e) => e.path) - .sort(); - expect(paths).toEqual(["/a/b/c/z.ts", "/a/b/y.ts"]); - }); - }); - - it("does not match files outside the start directory even with **", async () => { - await withDB(async (db) => { - mkdir(db, "/a", {}, () => 0); - mkdir(db, "/b", {}, () => 0); - await writeFile(db, "/a/x.ts", "", {}, () => 0); - await writeFile(db, "/b/x.ts", "", {}, () => 0); - const paths = find(db, "/a", "**/*.ts").map((e) => e.path); - expect(paths).toEqual(["/a/x.ts"]); - }); - }); - - it("throws ENOENT when the directory is missing", async () => { - await withDB((db) => { - expect(() => find(db, "/missing")).toThrowError(expect.objectContaining({ code: "ENOENT" })); - }); - }); - - it("throws ENOTDIR when called on a file", async () => { - await withDB(async (db) => { - await writeFile(db, "/file.txt", "x", {}, () => 0); - expect(() => find(db, "/file.txt")).toThrowError( - expect.objectContaining({ code: "ENOTDIR" }), - ); - }); - }); - - it("escapes regex metacharacters in literal segments of a pattern", async () => { - await withDB(async (db) => { - mkdir(db, "/a", {}, () => 0); - await writeFile(db, "/a/file.ts", "", {}, () => 0); - // The dot in `*.ts` is a regex metacharacter; make sure we don't match - // any other single character against it. - await writeFile(db, "/a/fileXts", "", {}, () => 0); - const paths = find(db, "/a", "*.ts").map((e) => e.path); - expect(paths).toEqual(["/a/file.ts"]); - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/find.ts b/spikes/349-dofs/vendor/dofs/src/fs/find.ts deleted file mode 100644 index 67e1f747..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/find.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { createWorkspaceError } from "../errors.js"; -import { canonicalizePath } from "../path.js"; -import type { Database } from "../storage.js"; -import { resolveInode } from "./resolve.js"; - -export interface WorkspaceFoundEntry { - path: string; - type: "file" | "dir"; -} - -interface ChildRow { - name: string; - child_inode: number; - type: "file" | "dir"; -} - -export function find(db: Database, directory: string, pattern?: string): WorkspaceFoundEntry[] { - const { path: canonical } = canonicalizePath(directory); - const node = resolveInode(db, canonical); - if (node === null) { - throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); - } - if (node.type !== "dir") { - throw createWorkspaceError("ENOTDIR", `not a directory: ${canonical}`, canonical); - } - - const out: WorkspaceFoundEntry[] = []; - // An empty pattern is equivalent to no pattern: walk and return - // everything rather than compiling it into `^$`, which would match - // only empty relative paths and yield no results. - const regex = pattern ? compileGlob(pattern) : undefined; - - walk(db, node.inode, canonical, out); - - if (regex === undefined) { - return out; - } - // Glob matches against the path relative to the start directory. - const prefix = canonical === "/" ? "/" : `${canonical}/`; - return out.filter((entry) => { - if (!entry.path.startsWith(prefix)) return false; - const rel = entry.path.slice(prefix.length); - return regex.test(rel); - }); -} - -function walk(db: Database, parentInode: number, parentPath: string, out: WorkspaceFoundEntry[]) { - const children = db.all( - `SELECT d.name AS name, d.child_inode AS child_inode, n.type AS type - FROM vfs_dirents d - JOIN vfs_nodes n ON n.inode = d.child_inode - WHERE d.parent_inode = ? - ORDER BY d.name`, - parentInode, - ); - for (const child of children) { - const childPath = parentPath === "/" ? `/${child.name}` : `${parentPath}/${child.name}`; - out.push({ path: childPath, type: child.type }); - if (child.type === "dir") { - walk(db, child.child_inode, childPath, out); - } - } -} - -// Compile a simple glob into a regex. Supported: -// * matches any run of characters except '/' -// ** matches any run of characters including '/' -// Anything else is a literal. Regex metacharacters in literals are -// escaped so '.' in '*.ts' doesn't match an arbitrary character. -function compileGlob(pattern: string): RegExp { - let re = ""; - let i = 0; - while (i < pattern.length) { - const ch = pattern[i]; - if (ch === "*") { - if (pattern[i + 1] === "*") { - // '**/' matches zero or more path segments. Without the slash, '**' - // matches any run including slashes. - if (pattern[i + 2] === "/") { - re += "(?:.*/)?"; - i += 3; - } else { - re += ".*"; - i += 2; - } - } else { - re += "[^/]*"; - i += 1; - } - continue; - } - if (REGEX_METACHARS.has(ch)) { - re += `\\${ch}`; - } else { - re += ch; - } - i += 1; - } - return new RegExp(`^${re}$`); -} - -const REGEX_METACHARS = new Set([".", "+", "?", "^", "$", "(", ")", "[", "]", "{", "}", "|", "\\"]); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/gc.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/gc.test.ts deleted file mode 100644 index 64cce055..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/gc.test.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import type { Database } from "../storage.js"; -import { gc } from "./gc.js"; -import { rm } from "./rm.js"; -import { withDB } from "./with-db.js"; -import { writeFile } from "./writeFile.js"; - -function blobCount(db: Database): number { - return db.scalar("SELECT COUNT(*) FROM vfs_blobs") ?? 0; -} - -function blobBytesCount(db: Database): number { - return db.scalar("SELECT COUNT(*) FROM vfs_blob_bytes") ?? 0; -} - -describe("gc", () => { - it("returns { blobsFreed: 0, manifestsFreed: 0 } on an empty FS", async () => { - await withDB((db) => { - expect(gc(db, { now: () => 1_000_000 })).toEqual({ blobsFreed: 0, manifestsFreed: 0 }); - }); - }); - - it("does not free blobs that are still referenced", async () => { - await withDB(async (db) => { - await writeFile(db, "/a.txt", "live content", {}, () => 1000); - expect(blobCount(db)).toBe(1); - expect(gc(db, { now: () => 999_999_999 })).toEqual({ - blobsFreed: 0, - manifestsFreed: 0, - }); - expect(blobCount(db)).toBe(1); - }); - }); - - it("frees orphan blobs left behind by overwrite", async () => { - await withDB(async (db) => { - await writeFile(db, "/x.txt", "first", {}, () => 1000); - await writeFile(db, "/x.txt", "second", {}, () => 1000); - // Both blobs exist; the 'first' content is orphaned. - expect(blobCount(db)).toBe(2); - - const result = gc(db, { now: () => 2000, safetyWindowMs: 0 }); - expect(result.blobsFreed).toBe(1); - expect(blobCount(db)).toBe(1); - // The remaining blob is the one referenced by the current chunk. - const referenced = db.scalar( - "SELECT COUNT(*) FROM vfs_blobs b WHERE EXISTS (SELECT 1 FROM vfs_chunks c WHERE c.hash = b.hash)", - ); - expect(referenced).toBe(1); - }); - }); - - it("cascades the delete to vfs_blob_bytes", async () => { - await withDB(async (db) => { - await writeFile(db, "/x.txt", "first", {}, () => 1000); - await writeFile(db, "/x.txt", "second", {}, () => 1000); - expect(blobBytesCount(db)).toBe(2); - gc(db, { now: () => 2000, safetyWindowMs: 0 }); - expect(blobBytesCount(db)).toBe(1); - }); - }); - - it("frees orphan blobs left behind by rm", async () => { - await withDB(async (db) => { - await writeFile(db, "/a.txt", "unique-content", {}, () => 1000); - expect(blobCount(db)).toBe(1); - rm(db, "/a.txt", {}); - expect(blobCount(db)).toBe(1); // rm doesn't sweep blobs - const result = gc(db, { now: () => 2000, safetyWindowMs: 0 }); - expect(result.blobsFreed).toBe(1); - expect(blobCount(db)).toBe(0); - }); - }); - - it("respects the safety window", async () => { - await withDB(async (db) => { - await writeFile(db, "/x.txt", "first", {}, () => 1000); - await writeFile(db, "/x.txt", "second", {}, () => 1000); - // Inside the safety window the orphan stays. - expect(gc(db, { now: () => 1500, safetyWindowMs: 1_000 })).toEqual({ - blobsFreed: 0, - manifestsFreed: 0, - }); - expect(blobCount(db)).toBe(2); - // Outside the window it gets swept. - expect(gc(db, { now: () => 5000, safetyWindowMs: 1_000 })).toEqual({ - blobsFreed: 1, - manifestsFreed: 1, - }); - expect(blobCount(db)).toBe(1); - }); - }); - - it("uses a conservative default safety window when none is provided", async () => { - await withDB(async (db) => { - await writeFile(db, "/x.txt", "first", {}, () => 1000); - await writeFile(db, "/x.txt", "second", {}, () => 1000); - // The default is conservative enough that a small elapsed time - // does not sweep anything. - expect(gc(db, { now: () => 1500 })).toEqual({ blobsFreed: 0, manifestsFreed: 0 }); - expect(blobCount(db)).toBe(2); - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/gc.ts b/spikes/349-dofs/vendor/dofs/src/fs/gc.ts deleted file mode 100644 index 4229fee9..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/gc.ts +++ /dev/null @@ -1,55 +0,0 @@ -import type { Database } from "../storage.js"; - -export interface GcOptions { - // Wall-clock at the moment GC runs. Defaults to Date.now so callers - // can pin it from tests. - now?: () => number; - // Blobs whose last_seen is younger than now() - safetyWindowMs are - // never swept. The default is generous (1 hour) so a misconfigured - // GC pass cannot wipe blobs the application is actively writing. - safetyWindowMs?: number; -} - -export interface GcResult { - blobsFreed: number; - manifestsFreed: number; -} - -const DEFAULT_SAFETY_WINDOW_MS = 60 * 60 * 1000; // 1 hour - -export function gc(db: Database, options: GcOptions = {}): GcResult { - const now = (options.now ?? Date.now)(); - const safety = options.safetyWindowMs ?? DEFAULT_SAFETY_WINDOW_MS; - const cutoff = now - safety; - - return db.transactionSync(() => { - // Sweep orphan blobs: no row in vfs_chunks references the hash and - // last_seen is older than the safety cutoff. vfs_blob_bytes - // cascades on delete via the foreign key, so the bytes row goes - // with its parent. - db.run( - `DELETE FROM vfs_blobs - WHERE last_seen < ? - AND NOT EXISTS (SELECT 1 FROM vfs_chunks c WHERE c.hash = vfs_blobs.hash)`, - cutoff, - ); - const blobsFreed = db.scalar("SELECT changes()") ?? 0; - - // Manifests share the blob safety window: a writer might have - // inserted a manifest row but not yet linked it from vfs_nodes. - // Inside the same transactionSync block this can't happen, but - // keep the window as defence in depth for sync-layer code that - // might stage manifests before linking nodes. - db.run( - `DELETE FROM vfs_manifests - WHERE last_seen < ? - AND NOT EXISTS ( - SELECT 1 FROM vfs_nodes n WHERE n.manifest_hash = vfs_manifests.hash - )`, - cutoff, - ); - const manifestsFreed = db.scalar("SELECT changes()") ?? 0; - - return { blobsFreed, manifestsFreed }; - }); -} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/grep.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/grep.test.ts deleted file mode 100644 index aef8fbcf..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/grep.test.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { grep } from "./grep.js"; -import { mkdir } from "./mkdir.js"; -import { withDB } from "./with-db.js"; -import { CHUNK_SIZE, writeFile } from "./writeFile.js"; - -describe("grep", () => { - it("returns no matches for an empty file", async () => { - await withDB(async (db) => { - await writeFile(db, "/empty", "", {}, () => 0); - expect(await grep(db, "TODO", "/empty")).toEqual([]); - }); - }); - - it("finds a single match in a single file", async () => { - await withDB(async (db) => { - await writeFile(db, "/a.txt", "line one\nTODO fix me\nline three\n", {}, () => 0); - expect(await grep(db, "TODO", "/a.txt")).toEqual([ - { path: "/a.txt", line: 2, text: "TODO fix me" }, - ]); - }); - }); - - it("returns multiple matches in one file", async () => { - await withDB(async (db) => { - await writeFile(db, "/a.txt", "a TODO here\nplain\nb TODO there\n", {}, () => 0); - expect(await grep(db, "TODO", "/a.txt")).toEqual([ - { path: "/a.txt", line: 1, text: "a TODO here" }, - { path: "/a.txt", line: 3, text: "b TODO there" }, - ]); - }); - }); - - it("walks a directory recursively", async () => { - await withDB(async (db) => { - mkdir(db, "/d/sub", { recursive: true }, () => 0); - await writeFile(db, "/d/a.txt", "TODO root\n", {}, () => 0); - await writeFile(db, "/d/sub/b.txt", "TODO nested\n", {}, () => 0); - await writeFile(db, "/d/sub/c.txt", "no match\n", {}, () => 0); - const matches = await grep(db, "TODO", "/d"); - expect(matches.map((m) => m.path).sort()).toEqual(["/d/a.txt", "/d/sub/b.txt"]); - }); - }); - - it("respects ignoreCase", async () => { - await withDB(async (db) => { - await writeFile(db, "/a.txt", "todo\nTODO\nTodo\n", {}, () => 0); - expect((await grep(db, "TODO", "/a.txt", { ignoreCase: true })).length).toBe(3); - expect((await grep(db, "TODO", "/a.txt")).length).toBe(1); - }); - }); - - it("matches across a chunk boundary", async () => { - await withDB(async (db) => { - // Lay out a file whose line straddles the 512KiB chunk boundary. - const pad = "a".repeat(CHUNK_SIZE - 5); - const content = `${pad}TODO straddle\nafter\n`; - await writeFile(db, "/big.txt", content, {}, () => 0); - const matches = await grep(db, "TODO", "/big.txt"); - expect(matches).toHaveLength(1); - expect(matches[0].path).toBe("/big.txt"); - expect(matches[0].text.endsWith("TODO straddle")).toBe(true); - }); - }); - - it("returns lines 1-indexed", async () => { - await withDB(async (db) => { - await writeFile(db, "/a.txt", "first\nsecond\nthird\n", {}, () => 0); - expect(await grep(db, "first", "/a.txt")).toEqual([ - { path: "/a.txt", line: 1, text: "first" }, - ]); - expect(await grep(db, "third", "/a.txt")).toEqual([ - { path: "/a.txt", line: 3, text: "third" }, - ]); - }); - }); - - it("rejects ENOENT when the path is missing", async () => { - await withDB(async (db) => { - await expect(grep(db, "x", "/missing")).rejects.toMatchObject({ code: "ENOENT" }); - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/grep.ts b/spikes/349-dofs/vendor/dofs/src/fs/grep.ts deleted file mode 100644 index fd28471c..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/grep.ts +++ /dev/null @@ -1,107 +0,0 @@ -import { createWorkspaceError } from "../errors.js"; -import { canonicalizePath } from "../path.js"; -import type { Database } from "../storage.js"; -import { find } from "./find.js"; -import { readFile } from "./readFile.js"; -import { resolveInode } from "./resolve.js"; - -export interface WorkspaceGrepMatch { - path: string; - line: number; - text: string; -} - -export interface GrepOptions { - ignoreCase?: boolean; -} - -export async function grep( - db: Database, - pattern: string, - path: string, - options: GrepOptions = {}, -): Promise { - const { path: canonical } = canonicalizePath(path); - const node = resolveInode(db, canonical); - if (node === null) { - throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); - } - - const filePaths = - node.type === "file" - ? [canonical] - : find(db, canonical) - .filter((entry) => entry.type === "file") - .map((entry) => entry.path); - - const matches: WorkspaceGrepMatch[] = []; - for (const filePath of filePaths) { - await scanFile(db, filePath, pattern, options, matches); - } - return matches; -} - -// Stream the file in chunks so very large files don't load fully into -// memory. Carry a partial-line tail between chunks (everything after -// the last '\n') so a line that straddles a chunk boundary still -// matches as one line. Line numbers are 1-indexed. -async function scanFile( - db: Database, - path: string, - pattern: string, - options: GrepOptions, - out: WorkspaceGrepMatch[], -): Promise { - const stream = await readFile(db, path); - const reader = stream.getReader(); - const decoder = new TextDecoder("utf-8", { fatal: false }); - const needle = options.ignoreCase ? pattern.toUpperCase() : pattern; - - let tail = ""; - let lineNo = 1; - while (true) { - const { value, done } = await reader.read(); - if (done) break; - if (value === undefined) continue; - const text = tail + decoder.decode(value, { stream: true }); - const newlineIdx = text.lastIndexOf("\n"); - const ready = newlineIdx === -1 ? "" : text.slice(0, newlineIdx); - tail = newlineIdx === -1 ? text : text.slice(newlineIdx + 1); - if (ready.length > 0) { - lineNo = scanLines(ready, lineNo, needle, options.ignoreCase === true, path, out); - } - } - // Drain the decoder and scan whatever's left (final line without a - // trailing newline). - tail += decoder.decode(); - if (tail.length > 0) { - scanLines(tail, lineNo, needle, options.ignoreCase === true, path, out); - } -} - -// Walk `block` line-by-line, push matches into `out`, return the next -// 1-indexed line number to use for the following block. -function scanLines( - block: string, - startLine: number, - needle: string, - ignoreCase: boolean, - path: string, - out: WorkspaceGrepMatch[], -): number { - let line = startLine; - let cursor = 0; - while (cursor <= block.length) { - const next = block.indexOf("\n", cursor); - const end = next === -1 ? block.length : next; - const text = block.slice(cursor, end); - const haystack = ignoreCase ? text.toUpperCase() : text; - if (haystack.includes(needle)) { - out.push({ path, line, text }); - } - line += 1; - if (next === -1) break; - cursor = next + 1; - } - return line; -} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/link.ts b/spikes/349-dofs/vendor/dofs/src/fs/link.ts deleted file mode 100644 index c5e21d17..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/link.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { createWorkspaceError } from "../errors.js"; -import { canonicalizePath } from "../path.js"; -import { incrementRev } from "../rev.js"; -import { ROOT_INODE } from "../schema/index.js"; -import type { Database } from "../storage.js"; -import { assertNotReadOnly } from "./mount-guard.js"; -import { resolveInode } from "./resolve.js"; -import { invalidateResolveExact } from "./resolveCache.js"; - -function resolveParent(db: Database, parts: string[], canonical: string): number { - let parentInode = ROOT_INODE; - for (let i = 0; i < parts.length - 1; i++) { - const child = db.one<{ child_inode: number }>( - "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", - parentInode, - parts[i], - ); - if (child === undefined) { - throw createWorkspaceError("ENOENT", `parent directory missing: ${canonical}`, canonical); - } - const next = db.one<{ inode: number; type: "file" | "dir" | "symlink" }>( - "SELECT inode, type FROM vfs_nodes WHERE inode = ?", - child.child_inode, - ); - if (next === undefined) { - throw createWorkspaceError("ENOENT", `dangling dirent: ${canonical}`, canonical); - } - if (next.type !== "dir") { - throw createWorkspaceError( - "ENOTDIR", - `parent path segment is not a directory: ${canonical}`, - canonical, - ); - } - parentInode = next.inode; - } - return parentInode; -} - -export function link(db: Database, existingPath: string, newPath: string): void { - const { parts, path: canonicalNew } = canonicalizePath(newPath); - if (parts.length === 0) { - throw createWorkspaceError("EEXIST", "cannot link onto root", canonicalNew); - } - - assertNotReadOnly(db, canonicalNew); - - db.transactionSync(() => { - const source = resolveInode(db, existingPath); - if (source === null) { - throw createWorkspaceError("ENOENT", `no such file: ${existingPath}`, existingPath); - } - if (source.type !== "file") { - throw createWorkspaceError( - "EPERM", - `cannot hardlink non-file: ${existingPath}`, - existingPath, - ); - } - - const parentInode = resolveParent(db, parts, canonicalNew); - const leafName = parts[parts.length - 1]; - const existing = db.one<{ child_inode: number }>( - "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", - parentInode, - leafName, - ); - if (existing !== undefined) { - throw createWorkspaceError("EEXIST", `path exists: ${canonicalNew}`, canonicalNew); - } - - db.run( - "INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", - parentInode, - leafName, - source.inode, - ); - const rev = incrementRev(db); - db.run("UPDATE vfs_nodes SET rev = ? WHERE inode = ?", rev, source.inode); - // A new hardlink name for an existing file: a leaf with no - // descendants, so drop just the (possibly negative) entry for it. - invalidateResolveExact(db, canonicalNew); - }); -} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/ls.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/ls.test.ts deleted file mode 100644 index 108c94ee..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/ls.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { ls } from "./ls.js"; -import { mkdir } from "./mkdir.js"; -import { symlink } from "./symlink.js"; -import { withDB } from "./with-db.js"; -import { writeFile } from "./writeFile.js"; - -describe("ls", () => { - it("returns an empty array when nothing matches", async () => { - await withDB((db) => { - expect(ls(db, "/")).toEqual([]); - }); - }); - - it("lists every file under root sorted by path", async () => { - await withDB(async (db) => { - mkdir(db, "/a/b", { recursive: true }, () => 0); - await writeFile(db, "/a/b/x.ts", "", {}, () => 0); - await writeFile(db, "/a/y.ts", "", {}, () => 0); - await writeFile(db, "/z.ts", "", {}, () => 0); - expect(ls(db, "/")).toEqual(["/a/b/x.ts", "/a/y.ts", "/z.ts"]); - }); - }); - - it("returns only files, not directory entries", async () => { - await withDB(async (db) => { - mkdir(db, "/a/b", { recursive: true }, () => 0); - await writeFile(db, "/a/b/x", "", {}, () => 0); - expect(ls(db, "/")).toEqual(["/a/b/x"]); - }); - }); - - it("matches an exact subtree by prefix", async () => { - await withDB(async (db) => { - mkdir(db, "/wsp", {}, () => 0); - mkdir(db, "/workspace", {}, () => 0); - await writeFile(db, "/wsp/a", "", {}, () => 0); - await writeFile(db, "/workspace/b", "", {}, () => 0); - await writeFile(db, "/workspace/c", "", {}, () => 0); - expect(ls(db, "/workspace")).toEqual(["/workspace/b", "/workspace/c"]); - }); - }); - - it("returns just the file path when the prefix is a file", async () => { - await withDB(async (db) => { - mkdir(db, "/a", {}, () => 0); - await writeFile(db, "/a/x.ts", "hi", {}, () => 0); - expect(ls(db, "/a/x.ts")).toEqual(["/a/x.ts"]); - }); - }); - - it("returns an empty array for a missing prefix", async () => { - await withDB((db) => { - expect(ls(db, "/no/such/prefix")).toEqual([]); - }); - }); - - it("lists a nested subdirectory without scanning sibling subtrees", async () => { - await withDB(async (db) => { - mkdir(db, "/a/deep", { recursive: true }, () => 0); - mkdir(db, "/b", { recursive: true }, () => 0); - await writeFile(db, "/a/deep/x.ts", "", {}, () => 0); - await writeFile(db, "/a/deep/y.ts", "", {}, () => 0); - await writeFile(db, "/a/top.ts", "", {}, () => 0); - await writeFile(db, "/b/other.ts", "", {}, () => 0); - expect(ls(db, "/a/deep")).toEqual(["/a/deep/x.ts", "/a/deep/y.ts"]); - }); - }); - - it("does not follow a symlink prefix", async () => { - await withDB(async (db) => { - mkdir(db, "/real", {}, () => 0); - await writeFile(db, "/real/f.ts", "", {}, () => 0); - symlink(db, "/real", "/link", () => 0); - // A symlink is a leaf with no dirents, so listing at or through - // it yields nothing; the real directory still lists normally. - expect(ls(db, "/link")).toEqual([]); - expect(ls(db, "/link/f.ts")).toEqual([]); - expect(ls(db, "/real")).toEqual(["/real/f.ts"]); - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/ls.ts b/spikes/349-dofs/vendor/dofs/src/fs/ls.ts deleted file mode 100644 index 6a5c2387..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/ls.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { canonicalizePath } from "../path.js"; -import { ROOT_INODE } from "../schema/index.js"; -import type { Database } from "../storage.js"; - -interface PathRow { - path: string; -} - -// Recursive CTE that materializes the file paths under one listing -// root. Files only (no directory entries) because that's the -// documented "flat list of file paths" semantics. -// -// The walk is seeded at the listing root's inode: each row is -// (inode, path, type), built by concatenating dirent names with '/' -// separators onto the seed path. Scoping the seed to the requested -// directory keeps the walk O(subtree) instead of O(whole tree). -const LS_QUERY = ` - WITH RECURSIVE walk(inode, path, type) AS ( - SELECT inode, ?, type FROM vfs_nodes WHERE inode = ? - UNION ALL - SELECT n.inode, w.path || '/' || d.name, n.type - FROM walk w - JOIN vfs_dirents d ON d.parent_inode = w.inode - JOIN vfs_nodes n ON n.inode = d.child_inode - ) - SELECT path FROM walk - WHERE type = 'file' - ORDER BY path -`; - -// Walk dirents from the root to `parts` without following symlinks, so -// the seed matches the CTE's structural view: a symlink component has -// no dirents and thus lists nothing, and a missing or non-directory -// component resolves to null (an empty listing). Returns the root -// inode for an empty path. -function resolvePrefixInode(db: Database, parts: string[]): number | null { - let inode = ROOT_INODE; - for (const name of parts) { - const child = db.one<{ child_inode: number }>( - "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", - inode, - name, - ); - if (child === undefined) return null; - inode = child.child_inode; - } - return inode; -} - -export function ls(db: Database, prefix: string): string[] { - const { parts, path: canonical } = canonicalizePath(prefix); - const inode = resolvePrefixInode(db, parts); - if (inode === null) return []; - // Root contributes the empty string so its children start with '/'; - // a non-root prefix seeds its own path so descendants read as - // absolute paths. - const seedPath = canonical === "/" ? "" : canonical; - return db.all(LS_QUERY, seedPath, inode).map((row) => row.path); -} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/mkdir.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/mkdir.test.ts deleted file mode 100644 index e1fe0221..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/mkdir.test.ts +++ /dev/null @@ -1,140 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { ROOT_INODE } from "../schema/index.js"; -import { mkdir } from "./mkdir.js"; -import { resolveInode } from "./resolve.js"; -import { withDB } from "./with-db.js"; - -describe("mkdir", () => { - it("creates a top-level directory with the default mode", async () => { - await withDB((db) => { - mkdir(db, "/a", {}, () => 2000); - const resolved = resolveInode(db, "/a"); - expect(resolved).toMatchObject({ type: "dir", mode: 0o755, mtime: 2000 }); - }); - }); - - it("honors the supplied mode", async () => { - await withDB((db) => { - mkdir(db, "/locked", { mode: 0o700 }, () => 0); - expect(resolveInode(db, "/locked")?.mode).toBe(0o700); - }); - }); - - it("bumps rev and stamps it onto the new node", async () => { - await withDB((db) => { - const beforeRev = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'"); - mkdir(db, "/a", {}, () => 0); - const afterRev = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'"); - expect(afterRev).toBe((beforeRev ?? 0) + 1); - - const nodeRev = db.scalar( - "SELECT n.rev FROM vfs_nodes n JOIN vfs_dirents d ON d.child_inode = n.inode WHERE d.parent_inode = ? AND d.name = ?", - ROOT_INODE, - "a", - ); - expect(nodeRev).toBe(afterRev); - }); - }); - - it("rejects when the parent directory is missing", async () => { - await withDB((db) => { - expect(() => mkdir(db, "/no/such/parent", {}, () => 0)).toThrowError( - expect.objectContaining({ code: "ENOENT" }), - ); - }); - }); - - it("rejects when the parent path segment is a file (ENOTDIR)", async () => { - await withDB((db) => { - db.run("INSERT INTO vfs_nodes (type, mode, mtime, rev) VALUES ('file', 420, 0, 0)"); - const inode = db.scalar("SELECT last_insert_rowid()"); - db.run( - "INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", - ROOT_INODE, - "a", - inode, - ); - expect(() => mkdir(db, "/a/b", {}, () => 0)).toThrowError( - expect.objectContaining({ code: "ENOTDIR" }), - ); - }); - }); - - it("rejects EEXIST when the path already exists as a directory without recursive", async () => { - await withDB((db) => { - mkdir(db, "/a", {}, () => 0); - expect(() => mkdir(db, "/a", {}, () => 0)).toThrowError( - expect.objectContaining({ code: "EEXIST" }), - ); - }); - }); - - it("rejects EEXIST when the path already exists as a file", async () => { - await withDB((db) => { - db.run("INSERT INTO vfs_nodes (type, mode, mtime, rev) VALUES ('file', 420, 0, 0)"); - const inode = db.scalar("SELECT last_insert_rowid()"); - db.run( - "INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", - ROOT_INODE, - "a", - inode, - ); - expect(() => mkdir(db, "/a", {}, () => 0)).toThrowError( - expect.objectContaining({ code: "EEXIST" }), - ); - }); - }); - - it("recursive: creates missing ancestors", async () => { - await withDB((db) => { - mkdir(db, "/x/y/z", { recursive: true }, () => 1234); - expect(resolveInode(db, "/x")?.type).toBe("dir"); - expect(resolveInode(db, "/x/y")?.type).toBe("dir"); - expect(resolveInode(db, "/x/y/z")?.type).toBe("dir"); - }); - }); - - it("recursive: is idempotent when the target already exists as a dir", async () => { - await withDB((db) => { - mkdir(db, "/a/b", { recursive: true }, () => 0); - expect(() => mkdir(db, "/a/b", { recursive: true }, () => 0)).not.toThrow(); - }); - }); - - it("recursive: still rejects EEXIST when the target exists as a file", async () => { - await withDB((db) => { - mkdir(db, "/a", {}, () => 0); - db.run("INSERT INTO vfs_nodes (type, mode, mtime, rev) VALUES ('file', 420, 0, 0)"); - const inode = db.scalar("SELECT last_insert_rowid()"); - const aInode = resolveInode(db, "/a")?.inode; - db.run( - "INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", - aInode, - "b", - inode, - ); - expect(() => mkdir(db, "/a/b", { recursive: true }, () => 0)).toThrowError( - expect.objectContaining({ code: "EEXIST" }), - ); - }); - }); - - it("rejects EEXIST when creating root", async () => { - await withDB((db) => { - expect(() => mkdir(db, "/", {}, () => 0)).toThrowError( - expect.objectContaining({ code: "EEXIST" }), - ); - }); - }); - - it("accepts recursive: false for node:fs/promises parity", async () => { - await withDB((db) => { - // boolean false should be accepted by the type and behave as default. - mkdir(db, "/dir", { recursive: false }, () => 0); - expect(() => mkdir(db, "/dir", { recursive: false }, () => 0)).toThrowError( - expect.objectContaining({ code: "EEXIST" }), - ); - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/mkdir.ts b/spikes/349-dofs/vendor/dofs/src/fs/mkdir.ts deleted file mode 100644 index 4d664c61..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/mkdir.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { createWorkspaceError } from "../errors.js"; -import { canonicalizePath } from "../path.js"; -import { incrementRev } from "../rev.js"; -import { ROOT_INODE } from "../schema/index.js"; -import type { Database } from "../storage.js"; -import { assertNotReadOnly } from "./mount-guard.js"; -import { invalidateResolveExact } from "./resolveCache.js"; - -export interface MkdirOptions { - recursive?: boolean; - mode?: number; -} - -interface ResolvedSegment { - inode: number; - type: "file" | "dir"; -} - -// Look up a child by name under a parent directory. Returns undefined -// when there's no dirent. The caller decides whether that's an error. -function lookupChild(db: Database, parentInode: number, name: string): ResolvedSegment | undefined { - const row = db.one<{ child_inode: number }>( - "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", - parentInode, - name, - ); - if (row === undefined) { - return undefined; - } - const node = db.one<{ inode: number; type: "file" | "dir" }>( - "SELECT inode, type FROM vfs_nodes WHERE inode = ?", - row.child_inode, - ); - if (node === undefined) { - return undefined; - } - return node; -} - -// Create one directory entry under `parentInode`, returning the new -// inode. The caller has already verified the name is not taken. -function createDir( - db: Database, - parentInode: number, - name: string, - mode: number, - mtime: number, - rev: number, -): number { - // RETURNING folds the rowid read into the INSERT. - const row = db.one<{ inode: number }>( - "INSERT INTO vfs_nodes (type, mode, mtime, rev) VALUES ('dir', ?, ?, ?) RETURNING inode", - mode, - mtime, - rev, - ); - if (row === undefined) { - throw createWorkspaceError("EIO", "failed to allocate inode"); - } - const inode = row.inode; - db.run( - "INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", - parentInode, - name, - inode, - ); - return inode; -} - -export function mkdir(db: Database, path: string, options: MkdirOptions, now: () => number): void { - const { parts, path: canonical } = canonicalizePath(path); - const recursive = options.recursive === true; - const mode = (options.mode ?? 0o755) & 0o7777; - - if (parts.length === 0) { - // Root always exists post-initializeSchema; mkdir("/") is EEXIST - // even with recursive (matches Node fs.mkdir's "EEXIST on root" - // behaviour for non-recursive; for recursive Node returns - // undefined, but our docs treat mkdir("/") as nonsensical). - throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); - } - assertNotReadOnly(db, canonical); - - db.transactionSync(() => { - const rev = incrementRev(db); - const mtime = now(); - - let parentInode = ROOT_INODE; - // Walk all but the final segment. Each must already exist as a - // directory; if `recursive`, we create missing ones. - for (let i = 0; i < parts.length - 1; i++) { - const name = parts[i]; - const existing = lookupChild(db, parentInode, name); - if (existing === undefined) { - if (!recursive) { - throw createWorkspaceError("ENOENT", `parent directory missing: ${canonical}`, canonical); - } - parentInode = createDir(db, parentInode, name, 0o755, mtime, rev); - // A newly created directory is empty, so a cached negative for - // its own path is the only stale entry possible; drop it exact. - invalidateResolveExact(db, `/${parts.slice(0, i + 1).join("/")}`); - continue; - } - if (existing.type !== "dir") { - throw createWorkspaceError( - "ENOTDIR", - `parent path segment is not a directory: ${canonical}`, - canonical, - ); - } - parentInode = existing.inode; - } - - // Final segment. - const leafName = parts[parts.length - 1]; - const existing = lookupChild(db, parentInode, leafName); - if (existing !== undefined) { - // EEXIST is correct for both "already a directory" and - // "already a file" per docs/04. Recursive only swallows the - // already-a-directory case. - if (recursive && existing.type === "dir") { - return; - } - throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); - } - - createDir(db, parentInode, leafName, mode, mtime, rev); - invalidateResolveExact(db, canonical); - }); -} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/mount-guard.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/mount-guard.test.ts deleted file mode 100644 index cc729a48..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/mount-guard.test.ts +++ /dev/null @@ -1,318 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import type { Database } from "../storage.js"; -import { mkdir } from "./mkdir.js"; -import { - assertNotReadOnly, - getReadOnlyMountRoots, - invalidateReadOnlyMountCache, -} from "./mount-guard.js"; -import { rename } from "./rename.js"; -import { resolveInode } from "./resolve.js"; -import { rm } from "./rm.js"; -import { symlink } from "./symlink.js"; -import { withDB } from "./with-db.js"; -import { writeFile, writeFileSync } from "./writeFile.js"; - -// Stage a read-only mount the way the workspace-side indexer -// eventually will: a row in `_vfs_mounts` plus an actual subtree -// stamped with `mount_root`. Tests that want the cache to pick this -// up should invalidate it after staging. -function stageMount(db: Database, root: string, mode: "read-only" | "read-write"): void { - db.run( - "INSERT INTO _vfs_mounts (root, kind, indexed, mode) VALUES (?, ?, 1, ?)", - root, - "test", - mode, - ); - invalidateReadOnlyMountCache(db); -} - -// Create a stub directory hierarchy at the mount root, stamped with -// `mount_root`. The guard only consults `_vfs_mounts`, so for the -// rm tests we materialise enough of the subtree for the walk to -// find something to delete. -async function materialiseRootDir(db: Database, root: string, now: () => number): Promise { - mkdir(db, root, { recursive: true }, now); - // Stamp the inode so a later "drop the workspace.mount_root - // column" sweep would notice if anything else relies on it. - db.run( - "UPDATE vfs_nodes SET mount_root = ? WHERE inode = (SELECT child_inode FROM vfs_dirents WHERE name = ? AND parent_inode = 1)", - root, - root.slice(1), - ); -} - -describe("mount-guard helpers", () => { - it("caches read-only roots per database and reloads after invalidation", async () => { - await withDB(async (db) => { - // Cold cache: empty list, no rows. - expect(getReadOnlyMountRoots(db)).toEqual([]); - - // Stage a row without invalidating; cache stays empty. - db.run( - "INSERT INTO _vfs_mounts (root, kind, indexed, mode) VALUES (?, ?, 1, 'read-only')", - "/workspace/r2", - "r2", - ); - expect(getReadOnlyMountRoots(db)).toEqual([]); - - // After invalidation the next call re-reads. - invalidateReadOnlyMountCache(db); - expect(getReadOnlyMountRoots(db)).toEqual(["/workspace/r2"]); - - // A read-write row stays out of the read-only set. - db.run( - "INSERT INTO _vfs_mounts (root, kind, indexed, mode) VALUES (?, ?, 1, 'read-write')", - "/workspace/scratch", - "r2", - ); - invalidateReadOnlyMountCache(db); - expect(getReadOnlyMountRoots(db)).toEqual(["/workspace/r2"]); - }); - }); - - it("assertNotReadOnly is a no-op when no read-only mounts are registered", async () => { - await withDB(async (db) => { - expect(() => assertNotReadOnly(db, "/anywhere")).not.toThrow(); - }); - }); - - it("assertNotReadOnly throws EROFS for paths under, at, or above a read-only root", async () => { - await withDB(async (db) => { - stageMount(db, "/workspace/r2", "read-only"); - - // Direct paths inside. - expect(() => assertNotReadOnly(db, "/workspace/r2/hello.txt")).toThrow(/EROFS|read-only/); - // Path equal to the mount root. - expect(() => assertNotReadOnly(db, "/workspace/r2")).toThrow(/EROFS|read-only/); - // Ancestor of the mount root (the rm-the-whole-workspace - // shape). - expect(() => assertNotReadOnly(db, "/workspace")).toThrow(/EROFS|read-only/); - - // Paths outside the mount are fine. - expect(() => assertNotReadOnly(db, "/workspace/r2-sibling")).not.toThrow(); - expect(() => assertNotReadOnly(db, "/scratch/elsewhere")).not.toThrow(); - }); - }); - - it("read-write mounts do not register as read-only", async () => { - await withDB(async (db) => { - stageMount(db, "/workspace/rw", "read-write"); - expect(getReadOnlyMountRoots(db)).toEqual([]); - expect(() => assertNotReadOnly(db, "/workspace/rw/file")).not.toThrow(); - }); - }); -}); - -describe("writeFile under a read-only mount", () => { - it("rejects a streaming write under the mount root with EROFS", async () => { - await withDB(async (db) => { - // Materialise the directory before flipping the mount to - // read-only so the guard doesn't block our own setup. - mkdir(db, "/workspace/r2", { recursive: true }, () => 0); - stageMount(db, "/workspace/r2", "read-only"); - - const source = new ReadableStream({ - start(c) { - c.enqueue(new TextEncoder().encode("blocked")); - c.close(); - }, - }); - await expect( - writeFile(db, "/workspace/r2/hello.txt", source, {}, () => 0), - ).rejects.toMatchObject({ code: "EROFS" }); - - // The reject happens before we stage blobs, so no orphan - // rows land. - const blobs = db.scalar("SELECT COUNT(*) FROM vfs_blobs") ?? 0; - expect(blobs).toBe(0); - }); - }); - - it("rejects writeFileSync under the mount root with EROFS", async () => { - await withDB(async (db) => { - mkdir(db, "/workspace/r2", { recursive: true }, () => 0); - stageMount(db, "/workspace/r2", "read-only"); - - expect(() => - writeFileSync(db, "/workspace/r2/hello.txt", new Uint8Array([1, 2, 3]), {}, () => 0), - ).toThrow(/EROFS|read-only/); - }); - }); - - it("allows writes under a read-write mount", async () => { - await withDB(async (db) => { - mkdir(db, "/workspace/rw", { recursive: true }, () => 0); - stageMount(db, "/workspace/rw", "read-write"); - - // No throw; the bytes land in vfs_nodes. - writeFileSync(db, "/workspace/rw/ok.txt", new TextEncoder().encode("hi"), {}, () => 0); - const inode = db.scalar( - "SELECT inode FROM vfs_nodes WHERE manifest_hash IS NOT NULL", - ); - expect(inode).toBeDefined(); - }); - }); -}); - -describe("mkdir under a read-only mount", () => { - it("rejects mkdir under the mount root with EROFS", async () => { - await withDB(async (db) => { - mkdir(db, "/workspace/r2", { recursive: true }, () => 0); - stageMount(db, "/workspace/r2", "read-only"); - - expect(() => mkdir(db, "/workspace/r2/sub", { recursive: true }, () => 0)).toThrow( - /EROFS|read-only/, - ); - }); - }); - - it("rejects mkdir of a read-only mount root that doesn't exist yet", async () => { - await withDB(async (db) => { - stageMount(db, "/workspace/r2", "read-only"); - expect(() => mkdir(db, "/workspace/r2", { recursive: true }, () => 0)).toThrow( - /EROFS|read-only/, - ); - }); - }); -}); - -describe("rm under a read-only mount", () => { - it("rejects rm of a path inside the mount", async () => { - await withDB(async (db) => { - // Stage the row, materialise the subtree before stamping - // read-only so writeFile can land a file. - stageMount(db, "/workspace/r2", "read-write"); - await materialiseRootDir(db, "/workspace/r2", () => 0); - writeFileSync(db, "/workspace/r2/hello.txt", new Uint8Array([1]), {}, () => 0); - - // Flip to read-only. - db.run("UPDATE _vfs_mounts SET mode = 'read-only' WHERE root = ?", "/workspace/r2"); - invalidateReadOnlyMountCache(db); - - expect(() => rm(db, "/workspace/r2/hello.txt", {})).toThrow(/EROFS|read-only/); - }); - }); - - it("rejects rm of the mount root itself", async () => { - await withDB(async (db) => { - await materialiseRootDir(db, "/workspace/r2", () => 0); - stageMount(db, "/workspace/r2", "read-only"); - - expect(() => rm(db, "/workspace/r2", { recursive: true, force: true })).toThrow( - /EROFS|read-only/, - ); - }); - }); - - it("rejects rm of an ancestor whose subtree contains a read-only mount", async () => { - await withDB(async (db) => { - mkdir(db, "/workspace/r2", { recursive: true }, () => 0); - stageMount(db, "/workspace/r2", "read-only"); - - // The ancestor path /workspace overlaps the read-only root - // via the symmetric check; rm with recursive/force must - // reject before deleting anything. - expect(() => rm(db, "/workspace", { recursive: true, force: true })).toThrow( - /EROFS|read-only/, - ); - - // The mount root inode is still present. - const remaining = db.scalar("SELECT COUNT(*) FROM vfs_dirents WHERE name = ?", "r2"); - expect(remaining).toBeGreaterThan(0); - }); - }); - - it("allows rm of a path outside any mount", async () => { - await withDB(async (db) => { - mkdir(db, "/scratch", { recursive: true }, () => 0); - writeFileSync(db, "/scratch/file.txt", new Uint8Array([1]), {}, () => 0); - stageMount(db, "/workspace/r2", "read-only"); - - expect(() => rm(db, "/scratch/file.txt", {})).not.toThrow(); - }); - }); - - it("allows rm under a read-write mount", async () => { - await withDB(async (db) => { - mkdir(db, "/workspace/rw", { recursive: true }, () => 0); - writeFileSync(db, "/workspace/rw/hi.txt", new Uint8Array([1]), {}, () => 0); - stageMount(db, "/workspace/rw", "read-write"); - - expect(() => rm(db, "/workspace/rw/hi.txt", {})).not.toThrow(); - }); - }); - - it("rejects rm through a symlinked parent that resolves into a read-only mount", async () => { - await withDB((db) => { - mkdir(db, "/mnt", { recursive: true }, () => 0); - writeFileSync(db, "/mnt/file.txt", new Uint8Array([1]), {}, () => 0); - symlink(db, "/mnt", "/link", () => 0); - stageMount(db, "/mnt", "read-only"); - const before = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; - - expect(() => rm(db, "/link/file.txt", {})).toThrowError( - expect.objectContaining({ code: "EROFS" }), - ); - - expect(resolveInode(db, "/mnt/file.txt")).not.toBeNull(); - expect(db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'")).toBe(before); - }); - }); - - it("rejects recursive rm of a directory inside a read-only mount via a symlinked parent", async () => { - await withDB((db) => { - mkdir(db, "/mnt/dir", { recursive: true }, () => 0); - writeFileSync(db, "/mnt/dir/file.txt", new Uint8Array([1]), {}, () => 0); - symlink(db, "/mnt", "/link", () => 0); - stageMount(db, "/mnt", "read-only"); - const before = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; - - expect(() => rm(db, "/link/dir", { recursive: true })).toThrowError( - expect.objectContaining({ code: "EROFS" }), - ); - - expect(resolveInode(db, "/mnt/dir/file.txt")).not.toBeNull(); - expect(db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'")).toBe(before); - }); - }); -}); - -describe("rename under a read-only mount", () => { - it("rejects rename from a symlinked parent that resolves into a read-only mount", async () => { - await withDB((db) => { - mkdir(db, "/mnt", { recursive: true }, () => 0); - writeFileSync(db, "/mnt/file.txt", new Uint8Array([1]), {}, () => 0); - symlink(db, "/mnt", "/link", () => 0); - stageMount(db, "/mnt", "read-only"); - const before = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; - - expect(() => rename(db, "/link/file.txt", "/moved.txt")).toThrowError( - expect.objectContaining({ code: "EROFS" }), - ); - - expect(resolveInode(db, "/mnt/file.txt")).not.toBeNull(); - expect(resolveInode(db, "/moved.txt")).toBeNull(); - expect(db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'")).toBe(before); - }); - }); - - it("rejects rename to a symlinked parent that resolves into a read-only mount", async () => { - await withDB((db) => { - mkdir(db, "/mnt", { recursive: true }, () => 0); - writeFileSync(db, "/src.txt", new Uint8Array([1]), {}, () => 0); - symlink(db, "/mnt", "/link", () => 0); - stageMount(db, "/mnt", "read-only"); - const before = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; - - expect(() => rename(db, "/src.txt", "/link/file.txt")).toThrowError( - expect.objectContaining({ code: "EROFS" }), - ); - - expect(resolveInode(db, "/src.txt")).not.toBeNull(); - expect(resolveInode(db, "/mnt/file.txt")).toBeNull(); - expect(db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'")).toBe(before); - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/mount-guard.ts b/spikes/349-dofs/vendor/dofs/src/fs/mount-guard.ts deleted file mode 100644 index 6a2aadd3..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/mount-guard.ts +++ /dev/null @@ -1,82 +0,0 @@ -// Read-only mount guard. -// -// Every dofs mutating entry point (writeFile, mkdir, rm, and the -// apply path in sync/apply.ts) consults this module to reject -// writes that fall under a registered read-only mount root. The -// guard lives at the data layer so container-side writes that -// arrive via pullOnce -> applyChanges are caught too — the -// workspace-side surface wrapper alone cannot see them. -// -// The set of read-only roots is small (one row per registered -// mount per workspace, typically <10) and changes only at indexer -// write time. Cache it per Database in a WeakMap so repeat lookups -// don't hit SQLite. The mount indexer in @cloudflare/computer -// invalidates the cache via `invalidateReadOnlyMountCache(db)` after -// it writes _vfs_mounts. - -import { createWorkspaceError } from "../errors.js"; -import type { Database } from "../storage.js"; - -// undefined sentinel = "not loaded yet"; an empty array means -// "loaded, no read-only mounts registered". The two are not the -// same: the empty case must skip the SQL lookup on every check. -const cache = new WeakMap(); - -// Public so the workspace-side indexer can drop the cache after it -// writes a new _vfs_mounts row. Tests also call it when they stage -// a mount fixture by direct SQL. -export function invalidateReadOnlyMountCache(db: Database): void { - cache.delete(db); -} - -function loadReadOnlyRoots(db: Database): readonly string[] { - const rows = db.all<{ root: string }>("SELECT root FROM _vfs_mounts WHERE mode = 'read-only'"); - const roots = rows.map((r) => r.root); - cache.set(db, roots); - return roots; -} - -export function getReadOnlyMountRoots(db: Database): readonly string[] { - const cached = cache.get(db); - if (cached !== undefined) return cached; - return loadReadOnlyRoots(db); -} - -// Symmetric overlap check between a candidate write path and a -// mount root. Either: -// - `path` is at or below `root` (a direct write or rm under the -// mount root), OR -// - `root` is below `path` (an ancestor rm that would recurse -// through the mount). -// Both shapes must be blocked so a read-only mount survives both -// vectors. Mirrors the predicate that lived in -// GuardedWorkspaceFilesystem before the data-layer move. -function overlapsRoot(path: string, root: string): boolean { - return path === root || path.startsWith(`${root}/`) || root.startsWith(`${path}/`); -} - -// Throws EROFS when the path overlaps any read-only mount root. -// Callers should invoke this before any DB mutation. The error -// shape matches the existing createWorkspaceError contract so -// surface callers see a normal WorkspaceFsError. -export function assertNotReadOnly(db: Database, path: string): void { - const roots = getReadOnlyMountRoots(db); - if (roots.length === 0) return; - for (const root of roots) { - if (overlapsRoot(path, root)) { - throw createWorkspaceError("EROFS", `read-only mount at ${root}: cannot modify`, path); - } - } -} - -// Variant for callers that already know the path is canonicalised -// and want to reject a single descendant during a recursive walk -// (rm's walkPostOrder). Returns the matching root or undefined; the -// caller decides whether to throw, log, or skip. -export function readOnlyRootFor(db: Database, path: string): string | undefined { - const roots = getReadOnlyMountRoots(db); - for (const root of roots) { - if (overlapsRoot(path, root)) return root; - } - return undefined; -} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/readFile.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/readFile.test.ts deleted file mode 100644 index c8d80c56..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/readFile.test.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { mkdir } from "./mkdir.js"; -import { readFile } from "./readFile.js"; -import { withDB } from "./with-db.js"; -import { CHUNK_SIZE, writeFile } from "./writeFile.js"; - -async function drain(stream: ReadableStream): Promise { - const reader = stream.getReader(); - const parts: Uint8Array[] = []; - let total = 0; - while (true) { - const { value, done } = await reader.read(); - if (done) break; - if (value !== undefined) { - parts.push(value); - total += value.byteLength; - } - } - const out = new Uint8Array(total); - let offset = 0; - for (const part of parts) { - out.set(part, offset); - offset += part.byteLength; - } - return out; -} - -describe("readFile", () => { - it("returns a ReadableStream by default", async () => { - await withDB(async (db) => { - await writeFile(db, "/a.txt", "hello workspace", {}, () => 0); - const stream = await readFile(db, "/a.txt"); - expect(stream).toBeInstanceOf(ReadableStream); - expect(new TextDecoder().decode(await drain(stream))).toBe("hello workspace"); - }); - }); - - it("returns a string when encoding is 'utf8'", async () => { - await withDB(async (db) => { - await writeFile(db, "/a.txt", "hello", {}, () => 0); - expect(await readFile(db, "/a.txt", "utf8")).toBe("hello"); - }); - }); - - it("accepts the object-form encoding option", async () => { - await withDB(async (db) => { - await writeFile(db, "/a.txt", "hello", {}, () => 0); - expect(await readFile(db, "/a.txt", { encoding: "utf8" })).toBe("hello"); - }); - }); - - it("streams a multi-chunk file in chunk-sized pieces", async () => { - await withDB(async (db) => { - const bytes = new Uint8Array(CHUNK_SIZE + 100); - bytes.fill(0x41); - for (let i = CHUNK_SIZE; i < bytes.byteLength; i++) bytes[i] = 0x42; - await writeFile(db, "/big", bytes, {}, () => 0); - - const stream = await readFile(db, "/big"); - const reader = stream.getReader(); - const first = await reader.read(); - expect(first.done).toBe(false); - expect(first.value?.byteLength).toBe(CHUNK_SIZE); - expect(first.value?.[0]).toBe(0x41); - const second = await reader.read(); - expect(second.done).toBe(false); - expect(second.value?.byteLength).toBe(100); - expect(second.value?.[0]).toBe(0x42); - const end = await reader.read(); - expect(end.done).toBe(true); - }); - }); - - it("returns an empty stream for an empty file", async () => { - await withDB(async (db) => { - await writeFile(db, "/empty", "", {}, () => 0); - const stream = await readFile(db, "/empty"); - const bytes = await drain(stream); - expect(bytes.byteLength).toBe(0); - expect(await readFile(db, "/empty", "utf8")).toBe(""); - }); - }); - - it("does not modify vfs_blobs.last_seen when chunks are read", async () => { - await withDB(async (db) => { - const bytes = new Uint8Array(CHUNK_SIZE + 1); - bytes.fill(0x61); - await writeFile(db, "/x.txt", bytes, {}, () => 100); - expect(db.scalar("SELECT MIN(last_seen) FROM vfs_blobs")).toBe(100); - - // String form reads every chunk and must leave last_seen alone. - await readFile(db, "/x.txt", "utf8"); - expect(db.scalar("SELECT MIN(last_seen) FROM vfs_blobs")).toBe(100); - - // Stream form: drain it so every chunk is pulled, then confirm - // no restamp happened in the pull callback. - const stream = await readFile(db, "/x.txt"); - const reader = stream.getReader(); - while (true) { - const { done } = await reader.read(); - if (done) break; - } - expect(db.scalar("SELECT MIN(last_seen) FROM vfs_blobs")).toBe(100); - }); - }); - - it("rejects ENOENT when the path does not exist", async () => { - await withDB(async (db) => { - await expect(readFile(db, "/missing")).rejects.toMatchObject({ code: "ENOENT" }); - await expect(readFile(db, "/missing", "utf8")).rejects.toMatchObject({ code: "ENOENT" }); - }); - }); - - it("rejects EISDIR when the path is a directory", async () => { - await withDB(async (db) => { - mkdir(db, "/d", {}, () => 0); - await expect(readFile(db, "/d")).rejects.toMatchObject({ code: "EISDIR" }); - await expect(readFile(db, "/d", "utf8")).rejects.toMatchObject({ code: "EISDIR" }); - }); - }); - - it("rejects ENOENT when an intermediate segment is missing", async () => { - await withDB(async (db) => { - await expect(readFile(db, "/no/such/file")).rejects.toMatchObject({ code: "ENOENT" }); - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/readFile.ts b/spikes/349-dofs/vendor/dofs/src/fs/readFile.ts deleted file mode 100644 index 7f89de40..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/readFile.ts +++ /dev/null @@ -1,194 +0,0 @@ -import { createWorkspaceError } from "../errors.js"; -import { canonicalizePath } from "../path.js"; -import type { Database } from "../storage.js"; -import { getBlobBytes } from "./blobCache.js"; -import { resolveInode } from "./resolve.js"; -import { getPendingWriteBufferByPath, getWriteBuffer } from "./writeBuffer.js"; -import { CHUNK_SIZE } from "./writeFile.js"; - -export interface ReadFileOptions { - encoding?: "utf8"; -} - -interface ChunkRow { - hash: Uint8Array; - size: number; -} - -// Overloads match docs/04_filesystem_interface.md exactly. -export function readFile(db: Database, path: string): Promise>; -export function readFile(db: Database, path: string, encoding: "utf8"): Promise; -export function readFile( - db: Database, - path: string, - options: ReadFileOptions, -): Promise>; -export async function readFile( - db: Database, - path: string, - optionsOrEncoding?: "utf8" | ReadFileOptions, -): Promise> { - const wantString = - optionsOrEncoding === "utf8" || - (typeof optionsOrEncoding === "object" && optionsOrEncoding?.encoding === "utf8"); - - // Pending-create files surface through the path-keyed buffer. - const { path: canonical } = canonicalizePath(path); - const pending = getPendingWriteBufferByPath(db, canonical); - if (pending !== undefined) { - const snapshot = new Uint8Array(pending.size); - snapshot.set(pending.buf.subarray(0, pending.size)); - if (wantString) return new TextDecoder().decode(snapshot); - return new ReadableStream({ - start(controller) { - controller.enqueue(snapshot); - controller.close(); - }, - }); - } - - // Resolve up front so we surface ENOENT/EISDIR before doing any - // streaming work. - const node = resolveInode(db, path); - if (node === null) { - throw createWorkspaceError("ENOENT", `no such file: ${path}`, path); - } - if (node.type !== "file") { - throw createWorkspaceError("EISDIR", `path is a directory: ${path}`, path); - } - - // While a write buffer is open for this inode it is the source of - // truth. Skip the chunk store and serve the buffered bytes. - const buffered = getWriteBuffer(db, node.inode); - if (buffered?.dirty) { - const snapshot = new Uint8Array(buffered.size); - snapshot.set(buffered.buf.subarray(0, buffered.size)); - if (wantString) return new TextDecoder().decode(snapshot); - return new ReadableStream({ - start(controller) { - controller.enqueue(snapshot); - controller.close(); - }, - }); - } - - const chunks = db.all( - "SELECT hash, size FROM vfs_chunks WHERE inode = ? ORDER BY idx", - node.inode, - ); - - if (wantString) { - // Fast path — concatenate everything and decode once. Matches the - // node:fs/promises.readFile semantics for an encoding argument: - // memory cost = whole file. - const totalSize = chunks.reduce((acc, c) => acc + c.size, 0); - const out = new Uint8Array(totalSize); - let offset = 0; - for (const chunk of chunks) { - const bytes = getBlobBytes(db, chunk.hash); - if (bytes === undefined) { - throw createWorkspaceError("EIO", `missing blob bytes for ${path}`, path); - } - out.set(bytes, offset); - offset += bytes.byteLength; - } - return new TextDecoder().decode(out); - } - - // Stream form. We enqueue one Uint8Array per chunk, lazily pulled. - // Reads resolve bytes by hash and never restamp last_seen: a chunk - // being read is already linked to a node, so gc's orphan gate keeps - // it. last_seen only guards blobs staged but not yet linked. - let i = 0; - return new ReadableStream({ - pull(controller) { - if (i >= chunks.length) { - controller.close(); - return; - } - const chunk = chunks[i++]; - const bytes = getBlobBytes(db, chunk.hash); - if (bytes === undefined) { - controller.error(createWorkspaceError("EIO", `missing blob bytes for ${path}`, path)); - return; - } - controller.enqueue(bytes); - }, - }); -} - -// Positional read primitive. Walks only the chunk rows that overlap -// [offset, offset+length), so the FUSE driver can serve a kernel -// read without materializing the whole file. -export function readRangeSync( - db: Database, - path: string, - offset: number, - length: number, -): Uint8Array { - if (!Number.isInteger(offset) || offset < 0) { - throw createWorkspaceError("EINVAL", `invalid read offset: ${offset}`, path); - } - if (!Number.isInteger(length) || length < 0) { - throw createWorkspaceError("EINVAL", `invalid read length: ${length}`, path); - } - // Pending-create files have no inode yet. Serve reads from the - // path-keyed buffer until release commits the row. - const { path: canonical } = canonicalizePath(path); - const pending = getPendingWriteBufferByPath(db, canonical); - if (pending !== undefined) { - if (length === 0) return new Uint8Array(); - if (offset >= pending.size) return new Uint8Array(); - const end = Math.min(offset + length, pending.size); - return pending.buf.subarray(offset, end); - } - const node = resolveInode(db, path); - if (node === null) { - throw createWorkspaceError("ENOENT", `no such file: ${path}`, path); - } - if (node.type !== "file") { - throw createWorkspaceError("EISDIR", `path is a directory: ${path}`, path); - } - if (length === 0) return new Uint8Array(); - - // If a write buffer is open for this inode, it is the source of - // truth: pending writes have not yet committed to vfs_chunks. - // Reading from SQLite here would return stale bytes. - const buffered = getWriteBuffer(db, node.inode); - if (buffered?.dirty) { - if (offset >= buffered.size) return new Uint8Array(); - const end = Math.min(offset + length, buffered.size); - return buffered.buf.subarray(offset, end); - } - - // node.size is the cached value resolveInode just loaded. - const totalSize = node.size; - if (offset >= totalSize) return new Uint8Array(); - const end = Math.min(offset + length, totalSize); - const firstIdx = Math.floor(offset / CHUNK_SIZE); - const lastIdx = Math.floor((end - 1) / CHUNK_SIZE); - // Pull every overlapping chunk in one indexed range scan. Missing - // indices (a sparse file) simply don't come back, so the assembly - // below compacts around the gaps exactly as a per-index walk would. - const chunks = db.all<{ idx: number; hash: Uint8Array }>( - "SELECT idx, hash FROM vfs_chunks WHERE inode = ? AND idx BETWEEN ? AND ? ORDER BY idx", - node.inode, - firstIdx, - lastIdx, - ); - const out = new Uint8Array(end - offset); - let written = 0; - for (const { idx, hash } of chunks) { - const start = idx * CHUNK_SIZE; - const bytes = getBlobBytes(db, hash); - if (bytes === undefined) { - throw createWorkspaceError("EIO", `missing blob bytes for ${path}`, path); - } - const srcStart = Math.max(0, offset - start); - const srcEnd = Math.min(bytes.byteLength, end - start); - if (srcEnd <= srcStart) continue; - out.set(bytes.subarray(srcStart, srcEnd), written); - written += srcEnd - srcStart; - } - return written === out.byteLength ? out : out.subarray(0, written); -} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/readRange.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/readRange.test.ts deleted file mode 100644 index 626b5ab9..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/readRange.test.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { readRangeSync } from "./readFile.js"; -import { resolveInode } from "./resolve.js"; -import { withDB } from "./with-db.js"; -import { CHUNK_SIZE, writeFileSync } from "./writeFile.js"; - -describe("readRangeSync", () => { - it("reads small chunk-backed files at non-zero offset", async () => { - await withDB((db) => { - writeFileSync(db, "/inline.txt", new TextEncoder().encode("hello world"), {}, () => 1); - - const slice = readRangeSync(db, "/inline.txt", 6, 5); - expect(new TextDecoder().decode(slice)).toBe("world"); - }); - }); - - it("clamps the read at end of file", async () => { - await withDB((db) => { - writeFileSync(db, "/inline.txt", new TextEncoder().encode("abc"), {}, () => 1); - - expect(readRangeSync(db, "/inline.txt", 0, 100).byteLength).toBe(3); - expect(readRangeSync(db, "/inline.txt", 2, 100).byteLength).toBe(1); - expect(readRangeSync(db, "/inline.txt", 3, 100).byteLength).toBe(0); - }); - }); - - it("reads a single chunk window without materializing other chunks", async () => { - await withDB((db) => { - const original = new Uint8Array(CHUNK_SIZE * 3); - original.fill(1, 0, CHUNK_SIZE); - original.fill(2, CHUNK_SIZE, CHUNK_SIZE * 2); - original.fill(3, CHUNK_SIZE * 2); - writeFileSync(db, "/large.bin", original, {}, () => 1); - - const slice = readRangeSync(db, "/large.bin", CHUNK_SIZE + 10, 5); - expect(Array.from(slice)).toEqual([2, 2, 2, 2, 2]); - }); - }); - - it("reads across a chunk boundary", async () => { - await withDB((db) => { - const original = new Uint8Array(CHUNK_SIZE + 100); - original.fill(1, 0, CHUNK_SIZE); - original.fill(2, CHUNK_SIZE); - writeFileSync(db, "/large.bin", original, {}, () => 1); - - const slice = readRangeSync(db, "/large.bin", CHUNK_SIZE - 2, 4); - expect(Array.from(slice)).toEqual([1, 1, 2, 2]); - }); - }); - - it("returns an empty view past the end of a chunk-backed file", async () => { - await withDB((db) => { - const original = new Uint8Array(CHUNK_SIZE + 1); - original.fill(7); - writeFileSync(db, "/large.bin", original, {}, () => 1); - - expect(readRangeSync(db, "/large.bin", CHUNK_SIZE + 1, 10).byteLength).toBe(0); - }); - }); - - it("assembles a read spanning multiple chunks with partial ends", async () => { - await withDB((db) => { - const original = new Uint8Array(CHUNK_SIZE * 3); - original.fill(1, 0, CHUNK_SIZE); - original.fill(2, CHUNK_SIZE, CHUNK_SIZE * 2); - original.fill(3, CHUNK_SIZE * 2); - writeFileSync(db, "/large.bin", original, {}, () => 1); - - // Start inside chunk 0 and end inside chunk 2, so the range query - // returns all three rows and they must assemble in idx order with - // correct partial-chunk trimming. - const start = CHUNK_SIZE - 3; - const len = CHUNK_SIZE + 6; - const slice = readRangeSync(db, "/large.bin", start, len); - expect(slice.byteLength).toBe(len); - expect(equalBytes(slice, original.subarray(start, start + len))).toBe(true); - }); - }); - - it("reads an entire multi-chunk file byte-for-byte", async () => { - await withDB((db) => { - const original = new Uint8Array(CHUNK_SIZE * 2 + 50); - for (let i = 0; i < original.byteLength; i++) original[i] = i % 251; - writeFileSync(db, "/large.bin", original, {}, () => 1); - - const slice = readRangeSync(db, "/large.bin", 0, original.byteLength); - expect(equalBytes(slice, original)).toBe(true); - }); - }); - - it("compacts around a missing chunk row rather than zero-filling", async () => { - await withDB((db) => { - const original = new Uint8Array(CHUNK_SIZE * 3); - original.fill(1, 0, CHUNK_SIZE); - original.fill(2, CHUNK_SIZE, CHUNK_SIZE * 2); - original.fill(3, CHUNK_SIZE * 2); - writeFileSync(db, "/large.bin", original, {}, () => 1); - const node = resolveInode(db, "/large.bin"); - // Drop the middle chunk row (node.size still reports three - // chunks). The read elides the gap and returns the present - // chunks concatenated, trimmed to what was actually read. - db.run("DELETE FROM vfs_chunks WHERE inode = ? AND idx = 1", node?.inode ?? 0); - - const slice = readRangeSync(db, "/large.bin", 0, CHUNK_SIZE * 3); - expect(slice.byteLength).toBe(CHUNK_SIZE * 2); - expect(slice[0]).toBe(1); - expect(slice[CHUNK_SIZE - 1]).toBe(1); - expect(slice[CHUNK_SIZE]).toBe(3); - expect(slice[CHUNK_SIZE * 2 - 1]).toBe(3); - }); - }); - - it("throws EIO when a referenced chunk's blob bytes are gone", async () => { - await withDB((db) => { - writeFileSync(db, "/inline.txt", new TextEncoder().encode("hello"), {}, () => 1); - // vfs_chunks still references the hash, but the bytes are gone - // (cascade from vfs_blobs) — a read must surface EIO. - db.run("DELETE FROM vfs_blobs"); - - expect(() => readRangeSync(db, "/inline.txt", 0, 5)).toThrowError( - expect.objectContaining({ code: "EIO" }), - ); - }); - }); -}); - -function equalBytes(a: Uint8Array, b: Uint8Array): boolean { - if (a.byteLength !== b.byteLength) return false; - for (let i = 0; i < a.byteLength; i++) { - if (a[i] !== b[i]) return false; - } - return true; -} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/readdir.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/readdir.test.ts deleted file mode 100644 index 77b69ea6..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/readdir.test.ts +++ /dev/null @@ -1,126 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { mkdir } from "./mkdir.js"; -import { readdir } from "./readdir.js"; -import { withDB } from "./with-db.js"; -import { writeFile } from "./writeFile.js"; - -describe("readdir", () => { - it("returns an empty array for an empty directory", async () => { - await withDB((db) => { - expect(readdir(db, "/")).toEqual([]); - }); - }); - - it("lists files and directories with dirent shape", async () => { - await withDB(async (db) => { - mkdir(db, "/sub", {}, () => 0); - await writeFile(db, "/file.txt", "x", {}, () => 0); - - const entries = readdir(db, "/"); - expect(entries).toHaveLength(2); - expect(entries).toContainEqual({ - name: "file.txt", - parentPath: "/", - isFile: true, - isDirectory: false, - isSymbolicLink: false, - }); - expect(entries).toContainEqual({ - name: "sub", - parentPath: "/", - isFile: false, - isDirectory: true, - isSymbolicLink: false, - }); - }); - }); - - it("sorts entries by name", async () => { - await withDB(async (db) => { - await writeFile(db, "/b", "", {}, () => 0); - await writeFile(db, "/a", "", {}, () => 0); - await writeFile(db, "/c", "", {}, () => 0); - expect(readdir(db, "/").map((e) => e.name)).toEqual(["a", "b", "c"]); - }); - }); - - it("limits committed entries before materializing the result", async () => { - await withDB(async (db) => { - for (const name of ["a", "b", "c"]) await writeFile(db, `/${name}`, "", {}, () => 0); - expect(readdir(db, "/", { limit: 2 }).map((entry) => entry.name)).toEqual(["a", "b"]); - }); - }); - - it("uses the canonical parent path for nested directories", async () => { - await withDB(async (db) => { - mkdir(db, "/a/b", { recursive: true }, () => 0); - await writeFile(db, "/a/b/leaf.txt", "x", {}, () => 0); - - const entries = readdir(db, "/a/b"); - expect(entries).toEqual([ - { - name: "leaf.txt", - parentPath: "/a/b", - isFile: true, - isDirectory: false, - isSymbolicLink: false, - }, - ]); - }); - }); - - it("canonicalizes the parentPath even when called with a non-canonical input", async () => { - await withDB(async (db) => { - mkdir(db, "/a", {}, () => 0); - await writeFile(db, "/a/x", "", {}, () => 0); - const entries = readdir(db, "/a//."); - expect(entries[0]).toMatchObject({ parentPath: "/a" }); - }); - }); - - it("throws ENOENT for a missing path", async () => { - await withDB((db) => { - expect(() => readdir(db, "/missing")).toThrowError( - expect.objectContaining({ code: "ENOENT" }), - ); - }); - }); - - it("throws ENOENT when an intermediate segment is missing", async () => { - await withDB((db) => { - expect(() => readdir(db, "/no/such/path")).toThrowError( - expect.objectContaining({ code: "ENOENT" }), - ); - }); - }); - - it("includes symlink entries with isSymbolicLink set", async () => { - // resolveInode + readdir originally only filtered file and dir - // rows; symlinks were invisible. The dirent shape now carries - // an explicit isSymbolicLink flag so just-bash and other - // adapters can branch on the type without a follow-up lstat. - const { symlink } = await import("./symlink.js"); - await withDB(async (db) => { - await writeFile(db, "/target", "x", {}, () => 0); - symlink(db, "/target", "/link", () => 0); - const entries = readdir(db, "/"); - const link = entries.find((e) => e.name === "link"); - expect(link).toMatchObject({ - name: "link", - isFile: false, - isDirectory: false, - isSymbolicLink: true, - }); - }); - }); - - it("throws ENOTDIR when called on a file", async () => { - await withDB(async (db) => { - await writeFile(db, "/file.txt", "x", {}, () => 0); - expect(() => readdir(db, "/file.txt")).toThrowError( - expect.objectContaining({ code: "ENOTDIR" }), - ); - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/readdir.ts b/spikes/349-dofs/vendor/dofs/src/fs/readdir.ts deleted file mode 100644 index dddfa8d1..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/readdir.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { createWorkspaceError } from "../errors.js"; -import { canonicalizePath } from "../path.js"; -import type { Database } from "../storage.js"; -import { resolveInode } from "./resolve.js"; -import { listPendingByParent } from "./writeBuffer.js"; - -export interface WorkspaceDirentResult { - name: string; - parentPath: string; - isFile: boolean; - isDirectory: boolean; - isSymbolicLink: boolean; -} - -interface DirentRow { - name: string; - type: "file" | "dir" | "symlink"; -} - -export interface ReaddirOptions { - /** Maximum committed entries to materialize. Pending entries may extend the result. */ - limit?: number; -} - -export function readdir( - db: Database, - path: string, - options: ReaddirOptions = {}, -): WorkspaceDirentResult[] { - const { path: canonical } = canonicalizePath(path); - const node = resolveInode(db, canonical); - if (node === null) { - throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); - } - if (node.type !== "dir") { - throw createWorkspaceError("ENOTDIR", `not a directory: ${canonical}`, canonical); - } - - const limit = options.limit; - if (limit !== undefined && (!Number.isSafeInteger(limit) || limit < 0)) { - throw new TypeError("readdir limit must be a non-negative safe integer"); - } - const rows = db.all( - `SELECT d.name AS name, n.type AS type - FROM vfs_dirents d - JOIN vfs_nodes n ON n.inode = d.child_inode - WHERE d.parent_inode = ? - ORDER BY d.name - ${limit === undefined ? "" : "LIMIT ?"}`, - ...(limit === undefined ? [node.inode] : [node.inode, limit]), - ); - - const entries = rows.map((row) => ({ - name: row.name, - parentPath: canonical, - isFile: row.type === "file", - isDirectory: row.type === "dir", - isSymbolicLink: row.type === "symlink", - })); - - // Merge in pending-create buffers parented under this directory so - // a `readdir` between FUSE create and release still surfaces the - // file. Skip any whose name already appears in the SQL rows (in - // case a concurrent commit just landed it). - const pending = listPendingByParent(db, node.inode); - if (pending.length > 0) { - const seen = new Set(entries.map((e) => e.name)); - for (const entry of pending) { - if (entry.pending === undefined) continue; - const { leafName } = entry.pending; - if (seen.has(leafName)) continue; - entries.push({ - name: leafName, - parentPath: canonical, - isFile: true, - isDirectory: false, - isSymbolicLink: false, - }); - } - entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); - } - - return entries; -} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/readlink.ts b/spikes/349-dofs/vendor/dofs/src/fs/readlink.ts deleted file mode 100644 index 8b9b5048..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/readlink.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { createWorkspaceError } from "../errors.js"; -import type { Database } from "../storage.js"; -import { resolveInode } from "./resolve.js"; - -// Return the stored target of a symlink. Does not follow the link. -// Mirrors POSIX semantics: ENOENT for a missing path, EINVAL when -// the path resolves to something that isn't a symlink. -export function readlink(db: Database, path: string): string { - const node = resolveInode(db, path, { followSymlinks: false }); - if (node === null) { - throw createWorkspaceError("ENOENT", `no such path: ${path}`, path); - } - if (node.type !== "symlink" || node.linkTarget === undefined) { - throw createWorkspaceError("EINVAL", `not a symlink: ${path}`, path); - } - return node.linkTarget; -} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/rename.ts b/spikes/349-dofs/vendor/dofs/src/fs/rename.ts deleted file mode 100644 index c2692b1e..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/rename.ts +++ /dev/null @@ -1,230 +0,0 @@ -import { createWorkspaceError } from "../errors.js"; -import { canonicalizePath } from "../path.js"; -import { incrementRev } from "../rev.js"; -import type { Database } from "../storage.js"; -import { recordDelete } from "../sync/changes.js"; -import { pathOf } from "../sync/paths.js"; -import { assertNotReadOnly } from "./mount-guard.js"; -import { resolveInode } from "./resolve.js"; -import { invalidateResolveExact, invalidateResolveSubtree } from "./resolveCache.js"; -import { unlinkDirent } from "./unlink.js"; - -type NodeType = "file" | "dir" | "symlink"; - -export function rename(db: Database, oldPath: string, newPath: string): void { - const { path: oldCanonical } = canonicalizePath(oldPath); - const { parts: newParts, path: newCanonical } = canonicalizePath(newPath); - - if (oldCanonical === "/") { - throw createWorkspaceError("EINVAL", "cannot rename root", oldCanonical); - } - if (newParts.length === 0) { - throw createWorkspaceError("EINVAL", "cannot rename onto root", newCanonical); - } - - assertNotReadOnly(db, oldCanonical); - assertNotReadOnly(db, newCanonical); - - db.transactionSync(() => { - const source = resolveInode(db, oldCanonical, { followSymlinks: false }); - if (source === null) { - throw createWorkspaceError("ENOENT", `no such path: ${oldCanonical}`, oldCanonical); - } - - // Resolve the source's real parent dirent. The parent path is - // resolved with symlinks followed so a request through a symlinked - // directory lands on the real container; the inode is then - // identified by (parent_inode, name) rather than by child_inode so - // a hardlinked source touches only the requested name. - const { parts: oldParts } = canonicalizePath(oldCanonical); - const oldName = oldParts[oldParts.length - 1]; - const oldParentPath = oldParts.length === 1 ? "/" : `/${oldParts.slice(0, -1).join("/")}`; - const oldParent = resolveInode(db, oldParentPath); - if (oldParent === null || oldParent.type !== "dir") { - throw createWorkspaceError("ENOENT", `no such path: ${oldCanonical}`, oldCanonical); - } - const oldParentReal = pathOf(db, oldParent.inode); - if (oldParentReal === null) { - throw createWorkspaceError("ENOENT", `no such path: ${oldCanonical}`, oldCanonical); - } - const oldRealPath = oldParentReal === "/" ? `/${oldName}` : `${oldParentReal}/${oldName}`; - assertNotReadOnly(db, oldRealPath); - - if (oldCanonical === newCanonical) return; - - const newName = newParts[newParts.length - 1]; - const newParentPath = newParts.length === 1 ? "/" : `/${newParts.slice(0, -1).join("/")}`; - const newParent = resolveInode(db, newParentPath); - if (newParent === null || newParent.type !== "dir") { - throw createWorkspaceError( - "ENOENT", - `parent directory missing: ${newCanonical}`, - newCanonical, - ); - } - const newParentReal = pathOf(db, newParent.inode); - if (newParentReal === null) { - throw createWorkspaceError( - "ENOENT", - `parent directory missing: ${newCanonical}`, - newCanonical, - ); - } - const newRealPath = newParentReal === "/" ? `/${newName}` : `${newParentReal}/${newName}`; - assertNotReadOnly(db, newRealPath); - - // A rename whose source and destination resolve to the very same - // dirent (same real parent and name, e.g. through a symlinked path) - // is a true no-op: leave the tree and the change stream untouched. - // This is distinct from renaming one hardlink onto another, where - // the names differ and the source link must still be removed. - if (oldParent.inode === newParent.inode && oldName === newName) return; - - const existing = db.one<{ child_inode: number; type: "file" | "dir" | "symlink" }>( - `SELECT d.child_inode AS child_inode, n.type AS type - FROM vfs_dirents d - JOIN vfs_nodes n ON n.inode = d.child_inode - WHERE d.parent_inode = ? AND d.name = ?`, - newParent.inode, - newName, - ); - - // Authoritative directory self-move guard. It tests the *resolved* - // destination parent inode against the source subtree, so it catches - // a symlinked destination that lands inside the source and allows one - // that resolves outside it. A textual prefix test on the unresolved - // path could do neither and is intentionally absent. - if ( - source.type === "dir" && - renamedSubtreeContains(db, source.inode, oldRealPath, newParent.inode) - ) { - throw createWorkspaceError( - "EINVAL", - `cannot rename a directory into itself: ${oldRealPath}`, - newCanonical, - ); - } - - if (existing !== undefined) { - assertCompatibleOverwrite(source.type, existing.type, newCanonical); - if (existing.type === "dir") { - const childCount = db.scalar( - "SELECT COUNT(*) FROM vfs_dirents WHERE parent_inode = ?", - existing.child_inode, - ); - if ((childCount ?? 0) > 0) { - throw createWorkspaceError("ENOTEMPTY", `not empty: ${newCanonical}`, newCanonical); - } - } - // Displace only the destination name. The displaced inode may - // carry other hardlinks (or be the source inode itself), so reap - // its chunks and node row only once the final link disappears. - // Order matters: displace before unlinking the source so a - // hardlink-onto-hardlink rename never momentarily drops to zero - // links and reaps the inode it is about to re-point. - unlinkDirent(db, newParent.inode, newName, existing.child_inode, existing.type); - } - - // Unlink only the source name; a hardlinked source keeps its other - // names alive. - db.run("DELETE FROM vfs_dirents WHERE parent_inode = ? AND name = ?", oldParent.inode, oldName); - db.run( - "INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", - newParent.inode, - newName, - source.inode, - ); - - const rev = incrementRev(db); - // Rename is represented on the wire as old-path tombstones plus - // live entries for the moved inode subtree, so stamp only that - // subtree with the shared rev. Parent directory mtimes are left - // unchanged on purpose; this diverges from POSIX rename(2), but - // avoids treating the old and new parents as content changes. A - // directory move stamps and tombstones its whole subtree in two - // set-based statements; a file or symlink touches one inode and - // one path. - if (source.type === "dir") { - stampRenamedSubtree(db, source.inode, oldRealPath, rev); - } else { - db.run("UPDATE vfs_nodes SET rev = ? WHERE inode = ?", rev, source.inode); - recordDelete(db, rev, oldRealPath); - } - - // Drop cached resolutions for both endpoints. A directory move - // changes every descendant's path, so both sides need a subtree - // drop; a file/symlink move only touches the two leaf paths. The - // destination drop also covers any entry displaced by an overwrite. - if (source.type === "dir") { - invalidateResolveSubtree(db, oldRealPath); - invalidateResolveSubtree(db, newRealPath); - } else { - invalidateResolveExact(db, oldRealPath); - invalidateResolveExact(db, newRealPath); - } - }); -} - -function assertCompatibleOverwrite( - sourceType: NodeType, - existingType: NodeType, - path: string, -): void { - if (sourceType === "dir" && existingType === "dir") return; - if (existingType === "dir") { - throw createWorkspaceError("EISDIR", `cannot overwrite directory: ${path}`, path); - } - if (sourceType === "dir") { - throw createWorkspaceError("ENOTDIR", `cannot overwrite non-directory: ${path}`, path); - } -} - -// Recursive walk of a directory subtree seeded at an inode and its -// path. Descends through directory dirents only, so files and symlinks -// are leaves and each hardlink name yields its own row (matching the -// per-component collection it replaces). Bound as a reusable WITH -// clause whose two placeholders are the seed inode and path; callers -// append their own projection. -const SUBTREE_CTE = `WITH RECURSIVE subtree(inode, type, path) AS ( - SELECT ?, 'dir', ? - UNION ALL - SELECT n.inode, n.type, - CASE WHEN s.path = '/' THEN '/' || d.name ELSE s.path || '/' || d.name END - FROM subtree s - JOIN vfs_dirents d ON d.parent_inode = s.inode - JOIN vfs_nodes n ON n.inode = d.child_inode - WHERE s.type = 'dir' -)`; - -function renamedSubtreeContains( - db: Database, - rootInode: number, - rootPath: string, - targetInode: number, -): boolean { - const hit = db.one<{ hit: number }>( - `${SUBTREE_CTE} SELECT 1 AS hit FROM subtree WHERE inode = ? LIMIT 1`, - rootInode, - rootPath, - targetInode, - ); - return hit !== undefined; -} - -// Stamp the shared rev on every inode in the moved subtree and record -// an old-path tombstone for each entry, in two set-based statements -// over the same walk. -function stampRenamedSubtree(db: Database, rootInode: number, rootPath: string, rev: number): void { - db.run( - `${SUBTREE_CTE} UPDATE vfs_nodes SET rev = ? WHERE inode IN (SELECT inode FROM subtree)`, - rootInode, - rootPath, - rev, - ); - db.run( - `${SUBTREE_CTE} INSERT INTO vfs_changes (rev, path, op) SELECT ?, path, 'delete' FROM subtree ORDER BY path`, - rootInode, - rootPath, - rev, - ); -} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/resolve.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/resolve.test.ts deleted file mode 100644 index c8a939e0..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/resolve.test.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { ROOT_INODE } from "../schema/index.js"; -import type { Database } from "../storage.js"; -import { resolveInode } from "./resolve.js"; -import { withDB } from "./with-db.js"; - -// Convenience: insert a node row and a dirent under a given parent. -// Returns the new inode. type defaults to 'file' so directory tests are -// explicit. -function addNode( - db: Database, - parentInode: number, - name: string, - options: { type?: "file" | "dir"; mode?: number; mtime?: number } = {}, -): number { - const type = options.type ?? "file"; - const mode = options.mode ?? (type === "dir" ? 0o755 : 0o644); - const mtime = options.mtime ?? 0; - db.run("INSERT INTO vfs_nodes (type, mode, mtime, rev) VALUES (?, ?, ?, 0)", type, mode, mtime); - const inode = db.scalar("SELECT last_insert_rowid()"); - if (inode === undefined) { - throw new Error("failed to allocate inode"); - } - db.run( - "INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", - parentInode, - name, - inode, - ); - return inode; -} - -describe("resolveInode", () => { - it("resolves the root", async () => { - await withDB( - (db) => { - expect(resolveInode(db, "/")).toEqual({ - inode: ROOT_INODE, - type: "dir", - mode: 0o755, - mtime: 0, - size: 0, - }); - }, - { now: () => 0 }, - ); - }); - - it("resolves a top-level file", async () => { - await withDB((db) => { - const inode = addNode(db, ROOT_INODE, "hello.txt", { type: "file", mode: 0o644, mtime: 99 }); - expect(resolveInode(db, "/hello.txt")).toEqual({ - inode, - type: "file", - mode: 0o644, - mtime: 99, - size: 0, - }); - }); - }); - - it("resolves a nested directory", async () => { - await withDB((db) => { - const dir = addNode(db, ROOT_INODE, "a", { type: "dir" }); - const sub = addNode(db, dir, "b", { type: "dir" }); - const leaf = addNode(db, sub, "c.txt", { type: "file", mtime: 7 }); - expect(resolveInode(db, "/a/b/c.txt")).toEqual({ - inode: leaf, - type: "file", - mode: 0o644, - mtime: 7, - size: 0, - }); - }); - }); - - it("returns null when the final segment is missing", async () => { - await withDB((db) => { - addNode(db, ROOT_INODE, "a", { type: "dir" }); - expect(resolveInode(db, "/a/missing")).toBeNull(); - }); - }); - - it("returns null when an intermediate segment is missing", async () => { - await withDB((db) => { - expect(resolveInode(db, "/no/such/path")).toBeNull(); - }); - }); - - it("returns null when an intermediate segment is a file (not a dir)", async () => { - await withDB((db) => { - addNode(db, ROOT_INODE, "a", { type: "file" }); - expect(resolveInode(db, "/a/b")).toBeNull(); - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/resolve.ts b/spikes/349-dofs/vendor/dofs/src/fs/resolve.ts deleted file mode 100644 index defaa66c..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/resolve.ts +++ /dev/null @@ -1,264 +0,0 @@ -import { createWorkspaceError } from "../errors.js"; -import { canonicalizePath } from "../path.js"; -import { ROOT_INODE } from "../schema/index.js"; -import type { Database } from "../storage.js"; -import { lookupResolveCache, storeResolveCache } from "./resolveCache.js"; - -export interface ResolvedInode { - inode: number; - type: "file" | "dir" | "symlink"; - mode: number; - mtime: number; - // Cached file size from vfs_nodes.size. Always 0 for directories - // and symlinks; for files this matches SUM(vfs_chunks.size) for - // the inode. Stat callers consume it directly instead of doing a - // separate aggregate query. - size: number; - // Populated only when type === "symlink". Higher layers (readlink, - // lstat) consume this; resolveInode follows it transparently unless - // the caller asks otherwise. - linkTarget?: string; -} - -export interface ResolveOptions { - // Default true. Pass false to land on a symlink itself — the - // lstat / readlink code paths rely on this. Loops are still - // detected when following. - followSymlinks?: boolean; -} - -interface NodeRow { - inode: number; - type: "file" | "dir" | "symlink"; - mode: number; - mtime: number; - size: number; - link_target: string | null; -} - -interface ChildRow { - child_inode: number; -} - -// Cap the total number of symlinks resolved across a single -// resolveInode() call. Matches Linux's default SYMLOOP_MAX of 40. -const MAX_SYMLINK_FOLLOWS = 40; - -// Walk vfs_dirents from ROOT_INODE down to `path`. Returns null when -// any segment is missing, when an intermediate segment is a file -// (which a real filesystem would surface as ENOTDIR — callers map -// the `null` to the appropriate POSIX code), or when a final-segment -// symlink dangles. Throws ELOOP when a cycle is detected. -// -// `path` is canonicalized internally so callers can pass user input -// directly. Pre-canonicalized paths are also accepted and incur the -// same trivial re-canonicalization cost. -export function resolveInode( - db: Database, - path: string, - options: ResolveOptions = {}, -): ResolvedInode | null { - const followFinal = options.followSymlinks !== false; - const { parts, path: canonical } = canonicalizePath(path); - - // Cache + single-statement CTE serve only cache-eligible reads: - // follow-symlinks resolutions outside a transaction. Everything else - // uses the per-component loop: - // * followSymlinks:false (lstat / readlink / the provider's - // pre-mutation captures) — not cached, and the loop is cheaper - // for these shallow one-shot resolves than the recursive CTE. - // * inside a transaction (every mutation path) — resolves are - // shallow and hot, the CTE competes with the mutation's own - // statements for the plan cache (recompiling it is far dearer - // than the loop), and the cache must not be populated - // mid-transaction anyway (rollback safety). - // Mutations still invalidate the cache; that is independent of this. - if (!followFinal || db.inTransaction) { - return resolveParts(db, parts, followFinal, 0); - } - - // Repeat reads of the same path are served from the per-Database - // cache. Only the path -> inode mapping is cached; re-read the node - // row so mode/size/mtime/type are always current. A stale mapping - // (inode reaped without invalidation) reads back null and falls - // through to a full resolve that re-populates the cache. - const hit = lookupResolveCache(db, canonical); - if (hit !== undefined) { - if (hit.kind === "negative") { - return null; - } - const node = readNode(db, hit.inode); - if (node !== null) { - return toResolved(node); - } - } - - // One recursive-CTE statement resolves the common symlink-free - // case. Any symlink on the path falls back to the per-component loop, - // which follows links and enforces ELOOP; those resolutions are not - // cached (a followed path is an alias whose invalidation can't be - // reasoned about structurally). - const cte = resolveViaCte(db, parts); - if (cte.kind === "symlink") { - return resolveParts(db, parts, followFinal, 0); - } - storeResolveCache(db, canonical, cte.node === null ? null : cte.node.inode); - return cte.node; -} - -interface CteRow { - level: number; - inode: number; - type: "file" | "dir" | "symlink"; - mode: number; - mtime: number; - size: number; - link_target: string | null; -} - -type CteResolution = - // Walk completed with no symlink on the path: `node` is the resolved - // final node, or null when a segment was missing or an intermediate - // was not a directory (both map to null, exactly like the loop). - | { kind: "resolved"; node: ResolvedInode | null } - // A symlink was encountered anywhere on the path (intermediate or - // final). The CTE can't follow links, so the caller must fall back to - // the loop for byte-identical follow / ELOOP / dangling behaviour. - | { kind: "symlink" }; - -// Single-statement path walk. Binds the canonical path segments as a -// JSON array and walks vfs_dirents -> vfs_nodes from ROOT_INODE, one -// level per segment. Descends only through directories (WHERE -// w.type = 'dir'), so a file intermediate stalls the walk (ENOTDIR) -// and a missing dirent produces no row (ENOENT) — both surface as a -// missing level-D row, matching the loop's null. Every node the walk -// touches is returned so the caller can detect any symlink and fall -// back. -function resolveViaCte(db: Database, parts: string[]): CteResolution { - const rows = db.all( - `WITH RECURSIVE - segs(level, name) AS ( - SELECT key, value FROM json_each(?) - ), - walk(level, inode, type, mode, mtime, size, link_target) AS ( - SELECT 0, n.inode, n.type, n.mode, n.mtime, n.size, n.link_target - FROM vfs_nodes n - WHERE n.inode = ? - UNION ALL - SELECT w.level + 1, n.inode, n.type, n.mode, n.mtime, n.size, n.link_target - FROM walk w - JOIN segs s ON s.level = w.level - JOIN vfs_dirents d ON d.parent_inode = w.inode AND d.name = s.name - JOIN vfs_nodes n ON n.inode = d.child_inode - WHERE w.type = 'dir' - ) - SELECT level, inode, type, mode, mtime, size, link_target - FROM walk - ORDER BY level`, - JSON.stringify(parts), - ROOT_INODE, - ); - - const depth = parts.length; - let target: CteRow | undefined; - for (const row of rows) { - // Any symlink on the walk (root is level 0 and always a dir) means - // the loop must take over to follow it. - if (row.level >= 1 && row.type === "symlink") { - return { kind: "symlink" }; - } - if (row.level === depth) { - target = row; - } - } - return { - kind: "resolved", - node: target === undefined ? null : toResolved(target), - }; -} - -function toResolved(node: NodeRow): ResolvedInode { - return { - inode: node.inode, - type: node.type, - mode: node.mode, - mtime: node.mtime, - size: node.size, - linkTarget: node.link_target ?? undefined, - }; -} - -function resolveParts( - db: Database, - parts: string[], - followFinal: boolean, - follows: number, -): ResolvedInode | null { - const root = readNode(db, ROOT_INODE); - if (root === null) { - return null; - } - - let current: NodeRow = root; - for (let i = 0; i < parts.length; i++) { - const isFinal = i === parts.length - 1; - if (current.type !== "dir") { - return null; - } - const child = db.one( - "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", - current.inode, - parts[i], - ); - if (child === undefined) { - return null; - } - const next = readNode(db, child.child_inode); - if (next === null) { - return null; - } - // Intermediate symlinks always get followed; final-segment symlinks - // are only followed when the caller wants. A dangling intermediate - // is the same as a missing intermediate (return null). - if (next.type === "symlink" && (!isFinal || followFinal)) { - follows += 1; - if (follows > MAX_SYMLINK_FOLLOWS) { - throw createWorkspaceError("ELOOP", "too many symlinks resolving path"); - } - const target = next.link_target ?? ""; - const resolved = resolveParts(db, canonicalizePath(target).parts, true, follows); - if (resolved === null) { - return null; - } - // Replace the current dirent-resolved node with the followed - // result, then keep walking remaining segments (if any). - current = { - inode: resolved.inode, - type: resolved.type, - mode: resolved.mode, - mtime: resolved.mtime, - size: resolved.size, - link_target: resolved.linkTarget ?? null, - }; - continue; - } - current = next; - } - - return { - inode: current.inode, - type: current.type, - mode: current.mode, - mtime: current.mtime, - size: current.size, - linkTarget: current.link_target ?? undefined, - }; -} - -function readNode(db: Database, inode: number): NodeRow | null { - const row = db.one( - "SELECT inode, type, mode, mtime, size, link_target FROM vfs_nodes WHERE inode = ?", - inode, - ); - return row ?? null; -} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/resolveCache.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/resolveCache.test.ts deleted file mode 100644 index b3e61b52..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/resolveCache.test.ts +++ /dev/null @@ -1,248 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { applyChangesSync } from "../sync/apply.js"; -import { link } from "./link.js"; -import { mkdir } from "./mkdir.js"; -import { rename } from "./rename.js"; -import { resolveInode } from "./resolve.js"; -import { invalidateResolveExact } from "./resolveCache.js"; -import { rm } from "./rm.js"; -import { symlink } from "./symlink.js"; -import { withDB } from "./with-db.js"; -import { writeFileSync } from "./writeFile.js"; - -const NOW = (): number => 1000; - -function bytesOf(text: string): Uint8Array { - return new TextEncoder().encode(text); -} - -// These prove the path->inode cache never serves a stale result across -// every mutation shape, and that the recursive-CTE resolve's symlink -// fallback matches the per-component loop. They run under both backends -// (node:sqlite and real DO SqlStorage) via withDB, so cache correctness -// is exercised on the shipping storage. -describe("resolve cache + CTE resolve", () => { - it("drops a negative entry the instant the path is created", async () => { - await withDB(async (db) => { - mkdir(db, "/d", { recursive: true }, NOW); - // Prime a negative entry. - expect(resolveInode(db, "/d/f")).toBeNull(); - writeFileSync(db, "/d/f", bytesOf("x"), {}, NOW); - // Must resolve to the new inode, not a stale ENOENT. - expect(resolveInode(db, "/d/f")?.type).toBe("file"); - }); - }); - - it("invalidates a positive entry on unlink, rename, and rmdir", async () => { - await withDB(async (db) => { - mkdir(db, "/p", { recursive: true }, NOW); - - writeFileSync(db, "/p/a", bytesOf("a"), {}, NOW); - expect(resolveInode(db, "/p/a")?.inode).toBeGreaterThan(0); // prime - rm(db, "/p/a", {}); - expect(resolveInode(db, "/p/a")).toBeNull(); - - writeFileSync(db, "/p/b", bytesOf("b"), {}, NOW); - expect(resolveInode(db, "/p/b")).not.toBeNull(); // prime - rename(db, "/p/b", "/p/c"); - expect(resolveInode(db, "/p/b")).toBeNull(); - expect(resolveInode(db, "/p/c")).not.toBeNull(); - - mkdir(db, "/p/sub", { recursive: true }, NOW); - expect(resolveInode(db, "/p/sub")).not.toBeNull(); // prime - rm(db, "/p/sub", {}); - expect(resolveInode(db, "/p/sub")).toBeNull(); - }); - }); - - it("invalidates every descendant path on a directory rename", async () => { - await withDB(async (db) => { - mkdir(db, "/src/inner", { recursive: true }, NOW); - writeFileSync(db, "/src/inner/deep.txt", bytesOf("d"), {}, NOW); - // Prime positives for the whole chain. - const deepInode = resolveInode(db, "/src/inner/deep.txt")?.inode; - expect(resolveInode(db, "/src/inner")).not.toBeNull(); - expect(deepInode).toBeGreaterThan(0); - - rename(db, "/src", "/dst"); - - // Old descendant paths must be gone, not stale positives. - expect(resolveInode(db, "/src/inner/deep.txt")).toBeNull(); - expect(resolveInode(db, "/src/inner")).toBeNull(); - expect(resolveInode(db, "/src")).toBeNull(); - // New paths resolve; the moved file keeps its inode. - expect(resolveInode(db, "/dst/inner/deep.txt")?.inode).toBe(deepInode); - }); - }); - - it("resolves a hardlink's second name to the shared inode", async () => { - await withDB(async (db) => { - mkdir(db, "/h", { recursive: true }, NOW); - writeFileSync(db, "/h/a", bytesOf("shared"), {}, NOW); - const inode = resolveInode(db, "/h/a")?.inode; - // Prime a negative for the not-yet-existing link path. - expect(resolveInode(db, "/h/b")).toBeNull(); - link(db, "/h/a", "/h/b"); - expect(resolveInode(db, "/h/b")?.inode).toBe(inode); - }); - }); - - it("reflects a sync-applied change through cached negative and positive paths", async () => { - await withDB(async (db) => { - mkdir(db, "/s", { recursive: true }, NOW); - writeFileSync(db, "/s/existing", bytesOf("e"), {}, NOW); - // Prime: negative for a dir we will create, positive for a file we - // will delete. - expect(resolveInode(db, "/s/newdir")).toBeNull(); - expect(resolveInode(db, "/s/existing")).not.toBeNull(); - - // applyChangesSync funnels through mkdir (create) and rm (delete), - // both of which invalidate the cache. - applyChangesSync( - db, - [ - { kind: "dir", rev: 1, path: "/s/newdir", mode: 0o755, mtime: 1000 }, - { kind: "delete", rev: 2, path: "/s/existing" }, - ], - new Map(), - ); - - expect(resolveInode(db, "/s/newdir")?.type).toBe("dir"); - expect(resolveInode(db, "/s/existing")).toBeNull(); - - // Apply a symlink over a POPULATED directory: exercises apply's own - // conflict-cleanup branch (removeReplaceableFinalEntry -> - // removeInodeTreeAtPath subtree invalidation) plus symlink-create - // invalidation, with a primed descendant positive. - mkdir(db, "/s/dir/child", { recursive: true }, NOW); - expect(resolveInode(db, "/s/dir/child")).not.toBeNull(); - expect(resolveInode(db, "/s/dir")?.type).toBe("dir"); - applyChangesSync( - db, - [ - { - kind: "symlink", - rev: 3, - path: "/s/dir", - target: "/elsewhere", - mode: 0o777, - mtime: 1000, - }, - ], - new Map(), - ); - // /s/dir is now a symlink; the former descendant no longer resolves. - expect(resolveInode(db, "/s/dir", { followSymlinks: false })?.type).toBe("symlink"); - expect(resolveInode(db, "/s/dir/child")).toBeNull(); - }); - }); - - it("leaves no stale entry after a rolled-back write", async () => { - await withDB(async (db) => { - mkdir(db, "/r", { recursive: true }, NOW); - const rDir = resolveInode(db, "/r")?.inode ?? 0; - writeFileSync(db, "/r/keep", bytesOf("k"), {}, NOW); - const keepInode = resolveInode(db, "/r/keep")?.inode; // prime positive - expect(keepInode).toBeGreaterThan(0); - - // A mutation is (structural write + cache invalidation) inside one - // transaction. This drives that shape with raw statements at a - // single transaction level — the DO backend forbids the nested - // savepoints an outer db.transactionSync around an fs op would use, - // and the cache's rollback-safety is backend-independent anyway. - - // Rolled-back delete: the invalidation drops the entry mid-txn; the - // rollback restores the dirent; the recompute finds it alive again - // (no stale ENOENT). - expect(() => - db.transactionSync(() => { - db.run("DELETE FROM vfs_dirents WHERE parent_inode = ? AND name = ?", rDir, "keep"); - invalidateResolveExact(db, "/r/keep"); - throw new Error("boom-delete"); - }), - ).toThrow("boom-delete"); - expect(resolveInode(db, "/r/keep")?.inode).toBe(keepInode); - - // Rolled-back create + inode reuse: population is gated inside a - // transaction, so the doomed inode is never cached for /r/new. A - // broken gate would cache it and, after AUTOINCREMENT reuses the - // number on the next committed create, alias /r/new to /r/other. - expect(resolveInode(db, "/r/new")).toBeNull(); // prime negative - expect(() => - db.transactionSync(() => { - db.run("INSERT INTO vfs_nodes (type, mode, mtime, rev) VALUES ('file', 420, 0, 0)"); - const inode = db.scalar("SELECT last_insert_rowid() AS v") ?? 0; - db.run( - "INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", - rDir, - "new", - inode, - ); - invalidateResolveExact(db, "/r/new"); - // Reading inside the txn must not populate the cache (the gate). - expect(resolveInode(db, "/r/new")).not.toBeNull(); - throw new Error("boom-create"); - }), - ).toThrow("boom-create"); - writeFileSync(db, "/r/other", bytesOf("o"), {}, NOW); - expect(resolveInode(db, "/r/new")).toBeNull(); - expect(resolveInode(db, "/r/other")).not.toBeNull(); - }); - }); - - it("falls back to the loop for symlinks, resolving identically", async () => { - await withDB(async (db) => { - mkdir(db, "/target/sub", { recursive: true }, NOW); - writeFileSync(db, "/target/sub/file.txt", bytesOf("hello"), {}, NOW); - const realInode = resolveInode(db, "/target/sub/file.txt")?.inode; - - // Intermediate symlink: /link -> /target. - symlink(db, "/target", "/link", NOW); - // Following through the link reaches the real file inode. - expect(resolveInode(db, "/link/sub/file.txt")?.inode).toBe(realInode); - // Following the link itself lands on the directory it points at. - expect(resolveInode(db, "/link")?.type).toBe("dir"); - // lstat (no follow) lands on the symlink node itself. - const withoutFollow = resolveInode(db, "/link", { followSymlinks: false }); - expect(withoutFollow?.type).toBe("symlink"); - expect(withoutFollow?.linkTarget).toBe("/target"); - - // Dangling symlink: ENOENT when followed, symlink node when not. - symlink(db, "/nope", "/dangling", NOW); - expect(resolveInode(db, "/dangling")).toBeNull(); - expect(resolveInode(db, "/dangling", { followSymlinks: false })?.type).toBe("symlink"); - }); - }); - - it("drops a negative primed beneath a path that a new symlink makes resolvable", async () => { - await withDB(async (db) => { - mkdir(db, "/target", { recursive: true }, NOW); - writeFileSync(db, "/target/x", bytesOf("x"), {}, NOW); - const realInode = resolveInode(db, "/target/x")?.inode; - // Prime a negative for a path beneath where the link will land - // (no symlink on the path yet, so the negative is cached). - expect(resolveInode(db, "/link/x")).toBeNull(); - - // Creating the symlink makes "/link/x" resolvable through it; the - // subtree invalidation must drop the stale negative beneath it. - symlink(db, "/target", "/link", NOW); - expect(resolveInode(db, "/link/x")?.inode).toBe(realInode); - }); - }); - - it("does not cache a negative when a symlink on the path forces the loop", async () => { - await withDB(async (db) => { - mkdir(db, "/target", { recursive: true }, NOW); - symlink(db, "/target", "/link", NOW); - // Resolve through the link before the leaf exists: the CTE bails - // to the loop and must cache nothing for the aliased path. - expect(resolveInode(db, "/link/x")).toBeNull(); - - // Create the real leaf. This invalidates "/target/x", not the - // "/link/x" alias — so a negative wrongly cached by the bail would - // linger and this read would still see null. - writeFileSync(db, "/target/x", bytesOf("x"), {}, NOW); - expect(resolveInode(db, "/link/x")?.type).toBe("file"); - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/resolveCache.ts b/spikes/349-dofs/vendor/dofs/src/fs/resolveCache.ts deleted file mode 100644 index 0b40f054..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/resolveCache.ts +++ /dev/null @@ -1,129 +0,0 @@ -// Per-Database path -> inode resolution cache. -// -// Maps a canonical absolute path (as produced by -// canonicalizePath().path) to the inode that resolveInode(path, -// { followSymlinks: true }) lands on, or a NEGATIVE marker when the -// path does not resolve. It turns repeat stat/exists/read of the same -// path into an O(1) lookup instead of an O(depth) walk. -// -// Deliberately narrow, for correctness: -// -// * Only the path -> inode MAPPING is cached. resolveInode always -// re-reads the node row on a hit, so content changes (chmod, -// size/mtime, type) are never served stale — only structural -// mutations that move a dirent can invalidate an entry. -// -// * Only symlink-free resolutions are cached. Following a symlink -// makes the cached path an alias of the target whose invalidation -// can't be reasoned about from the path alone, so resolveInode -// stores nothing when a symlink was traversed. -// -// * Population is gated on Database.inTransaction: entries are only -// written outside a transaction, so a rolled-back mutation can -// never leave a positive/negative entry reflecting uncommitted -// state. Mutations invalidate (drop) freely — dropping is safe -// under rollback because the worst case is a recompute. -// -// The cache is per-Database (WeakMap) and bounded (LRU by Map -// insertion order, same discipline as blobCache). - -import type { Database } from "../storage.js"; - -// Sentinel value for "this path resolves to nothing" (ENOENT/ENOTDIR). -const NEGATIVE = -1; - -// Upper bound on cached paths per Database. Entries are tiny (a string -// key and a number), so this caps memory at a few MB while covering -// the working set of a busy tree. -const MAX_ENTRIES = 8192; - -// Keyed by the Database instance, so correctness assumes exactly one -// Database wraps each SqlStorage. Two Databases over the same storage -// would hold independent caches and could serve each other stale -// results; the DO owns a single Database, which upholds this. -const caches = new WeakMap>(); - -function cacheFor(db: Database): Map { - let cache = caches.get(db); - if (cache === undefined) { - cache = new Map(); - caches.set(db, cache); - } - return cache; -} - -export type ResolveCacheHit = { kind: "inode"; inode: number } | { kind: "negative" }; - -// Look up a canonical path. Returns undefined on a miss, a positive -// inode hit, or a negative (known-absent) hit. Bumps LRU recency. -export function lookupResolveCache( - db: Database, - canonicalPath: string, -): ResolveCacheHit | undefined { - const cache = cacheFor(db); - const value = cache.get(canonicalPath); - if (value === undefined) { - return undefined; - } - // Move to most-recent position for LRU eviction. - cache.delete(canonicalPath); - cache.set(canonicalPath, value); - return value === NEGATIVE ? { kind: "negative" } : { kind: "inode", inode: value }; -} - -// Cache a resolution. `inode === null` records a negative entry. No-op -// while a transaction is active so the cache never reflects -// uncommitted state (rollback safety). -export function storeResolveCache(db: Database, canonicalPath: string, inode: number | null): void { - if (db.inTransaction) { - return; - } - const cache = cacheFor(db); - cache.set(canonicalPath, inode === null ? NEGATIVE : inode); - while (cache.size > MAX_ENTRIES) { - const oldest = cache.keys().next(); - if (oldest.done === true) { - break; - } - cache.delete(oldest.value); - } -} - -// Drop the entry for exactly `canonicalPath`. Use after a mutation -// that changes a single leaf's existence without affecting anything -// beneath it: creating/removing a file, symlink, hardlink, or an -// empty directory. O(1). -export function invalidateResolveExact(db: Database, canonicalPath: string): void { - const cache = caches.get(db); - cache?.delete(canonicalPath); -} - -// Drop `canonicalPath` and every entry beneath it (keys prefixed -// `canonicalPath + "/"`). Use when a mutation changes a whole subtree's -// resolution: a recursive delete, any directory rename (every -// descendant's path changes), a structural subtree replacement, or a -// symlink create (paths *through* the new link become resolvable, so -// stale negatives beneath it must go). Root ("/") clears everything. -export function invalidateResolveSubtree(db: Database, canonicalPath: string): void { - const cache = caches.get(db); - if (cache === undefined || cache.size === 0) { - return; - } - if (canonicalPath === "/") { - cache.clear(); - return; - } - cache.delete(canonicalPath); - const prefix = `${canonicalPath}/`; - for (const key of cache.keys()) { - if (key.startsWith(prefix)) { - cache.delete(key); - } - } -} - -// Drop the entire cache for a Database. Used by tests and available as -// a blunt reset. -export function clearResolveCache(db: Database): void { - caches.get(db)?.clear(); -} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/rm.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/rm.test.ts deleted file mode 100644 index 5acd002d..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/rm.test.ts +++ /dev/null @@ -1,256 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import type { Database } from "../storage.js"; -import { mkdir } from "./mkdir.js"; -import { readdir } from "./readdir.js"; -import { readFile } from "./readFile.js"; -import { resolveInode } from "./resolve.js"; -import { rm } from "./rm.js"; -import { symlink } from "./symlink.js"; -import { withDB } from "./with-db.js"; -import { writeFile } from "./writeFile.js"; - -interface ChangeRow { - rev: number; - path: string; - op: string; -} - -function listChanges(db: Database): ChangeRow[] { - return db.all("SELECT rev, path, op FROM vfs_changes ORDER BY rev"); -} - -function countBlobs(db: Database): number { - return db.scalar("SELECT COUNT(*) FROM vfs_blobs") ?? 0; -} - -describe("rm", () => { - it("removes a single file", async () => { - await withDB(async (db) => { - await writeFile(db, "/a.txt", "hi", {}, () => 0); - rm(db, "/a.txt", {}); - expect(resolveInode(db, "/a.txt")).toBeNull(); - expect(readdir(db, "/")).toEqual([]); - }); - }); - - it("records a tombstone for the removed path", async () => { - await withDB(async (db) => { - await writeFile(db, "/a.txt", "hi", {}, () => 0); - rm(db, "/a.txt", {}); - expect(listChanges(db)).toEqual([expect.objectContaining({ path: "/a.txt", op: "delete" })]); - }); - }); - - it("records tombstones at the resolved path through intermediate symlinks", async () => { - await withDB(async (db) => { - mkdir(db, "/real", {}, () => 0); - await writeFile(db, "/real/file.txt", "content", {}, () => 0); - symlink(db, "/real", "/link", () => 0); - - rm(db, "/link/file.txt", {}); - - expect(listChanges(db)).toContainEqual( - expect.objectContaining({ path: "/real/file.txt", op: "delete" }), - ); - expect(listChanges(db)).not.toContainEqual( - expect.objectContaining({ path: "/link/file.txt", op: "delete" }), - ); - }); - }); - - it("bumps rev once per call", async () => { - await withDB(async (db) => { - await writeFile(db, "/a.txt", "hi", {}, () => 0); - const before = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; - rm(db, "/a.txt", {}); - const after = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; - expect(after).toBe(before + 1); - }); - }); - - it("leaves orphan blob rows alive for gc()", async () => { - await withDB(async (db) => { - await writeFile(db, "/a.txt", "unique-content", {}, () => 0); - const before = countBlobs(db); - expect(before).toBe(1); - rm(db, "/a.txt", {}); - expect(countBlobs(db)).toBe(1); - }); - }); - - it("removes a symlink itself rather than its target", async () => { - await withDB(async (db) => { - await writeFile(db, "/target.txt", "still here", {}, () => 0); - symlink(db, "/target.txt", "/link.txt", () => 0); - - rm(db, "/link.txt", {}); - - expect(resolveInode(db, "/link.txt", { followSymlinks: false })).toBeNull(); - expect(resolveInode(db, "/target.txt")).not.toBeNull(); - }); - }); - - it("recursive rm does not follow symlinks out of the removed tree", async () => { - await withDB(async (db) => { - await writeFile(db, "/outside.txt", "still here", {}, () => 0); - mkdir(db, "/d", {}, () => 0); - symlink(db, "/outside.txt", "/d/link.txt", () => 0); - - rm(db, "/d", { recursive: true }); - - expect(resolveInode(db, "/d", { followSymlinks: false })).toBeNull(); - expect(resolveInode(db, "/outside.txt")).not.toBeNull(); - }); - }); - - it("removes a dangling symlink", async () => { - await withDB((db) => { - symlink(db, "/missing", "/dangling", () => 0); - - rm(db, "/dangling", {}); - - expect(resolveInode(db, "/dangling", { followSymlinks: false })).toBeNull(); - }); - }); - - it("rejects ENOENT for a missing path", async () => { - await withDB((db) => { - expect(() => rm(db, "/missing", {})).toThrowError( - expect.objectContaining({ code: "ENOENT" }), - ); - }); - }); - - it("force swallows ENOENT", async () => { - await withDB((db) => { - expect(() => rm(db, "/missing", { force: true })).not.toThrow(); - expect(listChanges(db)).toEqual([]); - }); - }); - - it("rejects EPERM on root", async () => { - await withDB((db) => { - expect(() => rm(db, "/", {})).toThrowError(expect.objectContaining({ code: "EPERM" })); - expect(() => rm(db, "/", { recursive: true })).toThrowError( - expect.objectContaining({ code: "EPERM" }), - ); - expect(() => rm(db, "/", { recursive: true, force: true })).toThrowError( - expect.objectContaining({ code: "EPERM" }), - ); - }); - }); - - it("removes an empty directory without recursive", async () => { - await withDB((db) => { - mkdir(db, "/d", {}, () => 0); - rm(db, "/d", {}); - expect(resolveInode(db, "/d")).toBeNull(); - }); - }); - - it("rejects ENOTEMPTY on a non-empty directory without recursive", async () => { - await withDB(async (db) => { - mkdir(db, "/d", {}, () => 0); - await writeFile(db, "/d/a", "x", {}, () => 0); - expect(() => rm(db, "/d", {})).toThrowError(expect.objectContaining({ code: "ENOTEMPTY" })); - }); - }); - - it("recursive removes a directory tree", async () => { - await withDB(async (db) => { - mkdir(db, "/d/e/f", { recursive: true }, () => 0); - await writeFile(db, "/d/a", "x", {}, () => 0); - await writeFile(db, "/d/e/b", "y", {}, () => 0); - await writeFile(db, "/d/e/f/c", "z", {}, () => 0); - rm(db, "/d", { recursive: true }); - expect(resolveInode(db, "/d")).toBeNull(); - expect(resolveInode(db, "/d/a")).toBeNull(); - expect(resolveInode(db, "/d/e/f/c")).toBeNull(); - expect(readdir(db, "/")).toEqual([]); - }); - }); - - it("recursive removes a symlink to a directory without deleting its target", async () => { - await withDB(async (db) => { - mkdir(db, "/target/sub", { recursive: true }, () => 0); - await writeFile(db, "/target/sub/file.txt", "content", {}, () => 0); - symlink(db, "/target", "/link", () => 0); - - rm(db, "/link", { recursive: true }); - - expect(resolveInode(db, "/link", { followSymlinks: false })).toBeNull(); - expect(await readFile(db, "/target/sub/file.txt", "utf8")).toBe("content"); - }); - }); - - it("recursive records one tombstone per removed path", async () => { - await withDB(async (db) => { - mkdir(db, "/d", {}, () => 0); - await writeFile(db, "/d/a", "x", {}, () => 0); - await writeFile(db, "/d/b", "y", {}, () => 0); - rm(db, "/d", { recursive: true }); - const paths = listChanges(db) - .map((r) => r.path) - .sort(); - expect(paths).toEqual(["/d", "/d/a", "/d/b"]); - }); - }); - - it("recursive records resolved subtree tombstones through intermediate symlinks", async () => { - await withDB(async (db) => { - mkdir(db, "/real/dir", { recursive: true }, () => 0); - await writeFile(db, "/real/dir/a", "x", {}, () => 0); - await writeFile(db, "/real/dir/b", "y", {}, () => 0); - symlink(db, "/real", "/link", () => 0); - - rm(db, "/link/dir", { recursive: true }); - - const paths = listChanges(db) - .map((r) => r.path) - .sort(); - expect(paths).toEqual(expect.arrayContaining(["/real/dir", "/real/dir/a", "/real/dir/b"])); - expect(paths).not.toEqual( - expect.arrayContaining(["/link/dir", "/link/dir/a", "/link/dir/b"]), - ); - }); - }); - - it("recursive still bumps rev only once for the whole tree", async () => { - await withDB(async (db) => { - mkdir(db, "/d", {}, () => 0); - await writeFile(db, "/d/a", "x", {}, () => 0); - await writeFile(db, "/d/b", "y", {}, () => 0); - const before = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; - rm(db, "/d", { recursive: true }); - const after = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; - expect(after).toBe(before + 1); - }); - }); - - it("recursive cleans up chunk rows for removed files", async () => { - await withDB(async (db) => { - mkdir(db, "/d", {}, () => 0); - await writeFile(db, "/d/a", "first", {}, () => 0); - await writeFile(db, "/d/b", "second", {}, () => 0); - rm(db, "/d", { recursive: true }); - const chunkRows = db.scalar("SELECT COUNT(*) FROM vfs_chunks") ?? 0; - expect(chunkRows).toBe(0); - }); - }); - - it("force is idempotent on missing intermediate segments", async () => { - await withDB((db) => { - expect(() => rm(db, "/no/such/path", { force: true })).not.toThrow(); - }); - }); - - it("accepts recursive: false / force: false for node:fs/promises parity", async () => { - await withDB(async (db) => { - await writeFile(db, "/a.txt", "hello", {}, () => 0); - // boolean false should be accepted by the type and behave as default. - rm(db, "/a.txt", { recursive: false, force: false }); - expect(resolveInode(db, "/a.txt")).toBeNull(); - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/rm.ts b/spikes/349-dofs/vendor/dofs/src/fs/rm.ts deleted file mode 100644 index cf82fb8e..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/rm.ts +++ /dev/null @@ -1,181 +0,0 @@ -import { createWorkspaceError } from "../errors.js"; -import { canonicalizePath } from "../path.js"; -import { incrementRev } from "../rev.js"; -import type { Database } from "../storage.js"; -import { recordDelete } from "../sync/changes.js"; -import { pathOf } from "../sync/paths.js"; -import { assertNotReadOnly } from "./mount-guard.js"; -import { resolveInode } from "./resolve.js"; -import { invalidateResolveExact, invalidateResolveSubtree } from "./resolveCache.js"; -import { unlinkDirent } from "./unlink.js"; - -export interface RmOptions { - recursive?: boolean; - force?: boolean; -} - -interface DirChild { - name: string; - child_inode: number; - type: "file" | "dir" | "symlink"; -} - -// Walk a directory subtree post-order so we delete leaves before -// parents. Yields each node together with the parent inode and name -// the walk already knows, so the caller can unlink the dirent by -// (parent, name) without re-resolving the parent from root. The caller -// appends one tombstone per yielded path and clears vfs_chunks for -// file inodes. -function* walkPostOrder( - db: Database, - rootInode: number, - rootPath: string, - rootParentInode: number, - rootName: string, -): Generator<{ - path: string; - inode: number; - type: "file" | "dir" | "symlink"; - parentInode: number; - name: string; -}> { - // Stack-based DFS to avoid recursion limits on deep trees. - type Frame = { - inode: number; - path: string; - type: "file" | "dir" | "symlink"; - parentInode: number; - name: string; - expanded: boolean; - }; - const stack: Frame[] = [ - { - inode: rootInode, - path: rootPath, - type: "dir", - parentInode: rootParentInode, - name: rootName, - expanded: false, - }, - ]; - - while (stack.length > 0) { - const top = stack[stack.length - 1]; - if (top.type !== "dir" || top.expanded) { - stack.pop(); - yield { - path: top.path, - inode: top.inode, - type: top.type, - parentInode: top.parentInode, - name: top.name, - }; - continue; - } - top.expanded = true; - const children = db.all( - `SELECT d.name AS name, d.child_inode AS child_inode, n.type AS type - FROM vfs_dirents d - JOIN vfs_nodes n ON n.inode = d.child_inode - WHERE d.parent_inode = ? - ORDER BY d.name`, - top.inode, - ); - for (const child of children) { - const childPath = top.path === "/" ? `/${child.name}` : `${top.path}/${child.name}`; - stack.push({ - inode: child.child_inode, - path: childPath, - type: child.type, - parentInode: top.inode, - name: child.name, - expanded: false, - }); - } - } -} - -export function rm(db: Database, path: string, options: RmOptions): void { - const { parts, path: canonical } = canonicalizePath(path); - - if (parts.length === 0) { - // The workspace root is structural; refuse to delete it even with - // recursive+force. Matches the doc's example. - throw createWorkspaceError("EPERM", `cannot remove the root directory`, canonical); - } - - // assertNotReadOnly uses the symmetric overlap predicate, so a - // recursive rm of an ancestor whose subtree contains a read-only - // mount root is caught here without walking the tree. - assertNotReadOnly(db, canonical); - - const force = options.force === true; - const recursive = options.recursive === true; - - db.transactionSync(() => { - const node = resolveInode(db, canonical, { followSymlinks: false }); - if (node === null) { - if (force) return; - throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); - } - - if (node.type === "dir" && !recursive) { - const childCount = db.scalar( - "SELECT COUNT(*) FROM vfs_dirents WHERE parent_inode = ?", - node.inode, - ); - if ((childCount ?? 0) > 0) { - throw createWorkspaceError("ENOTEMPTY", `directory not empty: ${canonical}`, canonical); - } - } - - // Resolve the entry's real path from its parent rather than from - // the inode: a hardlinked file has several names, and pathOf would - // pick an arbitrary one. Following symlinks on the parent lets a - // request through a symlinked directory land on the real container - // while still removing exactly the requested name. - const name = parts[parts.length - 1]; - const parentPath = parts.length === 1 ? "/" : `/${parts.slice(0, -1).join("/")}`; - const parent = resolveInode(db, parentPath); - if (parent === null || parent.type !== "dir") { - throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); - } - const parentReal = pathOf(db, parent.inode); - if (parentReal === null) { - throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); - } - const realPath = parentReal === "/" ? `/${name}` : `${parentReal}/${name}`; - assertNotReadOnly(db, realPath); - - const rev = incrementRev(db); - - if (node.type !== "dir" || !recursive) { - // Single entry removal — file, symlink, or empty directory. A - // file inode may have multiple dirents (hardlinks), so remove - // only the requested name and reap chunks/node after the final - // link disappears. `parent` is already resolved above, so unlink - // by (parent, name) directly rather than re-resolving. The - // tombstone is recorded at the resolved real path so sync sees - // the move-aware location. - unlinkDirent(db, parent.inode, name, node.inode, node.type); - recordDelete(db, rev, realPath); - // A single removed entry is a file, symlink, or empty directory: - // no cached descendants to worry about, so drop it exact. - invalidateResolveExact(db, realPath); - return; - } - - // Recursive directory removal. Walk leaves first so each delete - // sees an empty parent by the time we get to it. File entries may - // be hardlinked outside this subtree, so delete by path rather - // than by child inode. The walk carries each node's parent inode - // and name, so unlinkDirent needs no per-node re-resolve from root. - for (const entry of walkPostOrder(db, node.inode, realPath, parent.inode, name)) { - unlinkDirent(db, entry.parentInode, entry.name, entry.inode, entry.type); - recordDelete(db, rev, entry.path); - } - // The whole subtree under realPath is gone; one subtree drop covers - // every descendant's cached resolution. - invalidateResolveSubtree(db, realPath); - }); -} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/stat.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/stat.test.ts deleted file mode 100644 index 8878bfd9..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/stat.test.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { mkdir } from "./mkdir.js"; -import { lstat, stat } from "./stat.js"; -import { symlink } from "./symlink.js"; -import { withDB } from "./with-db.js"; -import { writeFileSync } from "./writeFile.js"; - -const utf8 = (s: string): Uint8Array => new TextEncoder().encode(s); - -describe("stat", () => { - it("reports a regular file", async () => { - await withDB((db) => { - writeFileSync(db, "/a.txt", utf8("hello"), { mode: 0o644 }, () => 1234); - const s = stat(db, "/a.txt"); - expect(s).toMatchObject({ - name: "a.txt", - mode: 0o644, - size: 5, - isFile: true, - isDirectory: false, - isSymbolicLink: false, - mtime: 1234, - }); - }); - }); - - it("reports a directory", async () => { - await withDB((db) => { - mkdir(db, "/d", { mode: 0o700 }, () => 0); - const s = stat(db, "/d"); - expect(s).toMatchObject({ - name: "d", - mode: 0o700, - size: 0, - isFile: false, - isDirectory: true, - isSymbolicLink: false, - }); - }); - }); - - it("follows symlinks", async () => { - // stat() on a symlink reports the target. The link itself is - // observable only via lstat(). - await withDB((db) => { - writeFileSync(db, "/target", utf8("hello"), { mode: 0o600 }, () => 0); - symlink(db, "/target", "/link", () => 0); - const s = stat(db, "/link"); - expect(s.isFile).toBe(true); - expect(s.isSymbolicLink).toBe(false); - expect(s.mode).toBe(0o600); - expect(s.size).toBe(5); - }); - }); - - it("throws ENOENT for a missing path", async () => { - await withDB((db) => { - expect(() => stat(db, "/missing")).toThrowError(expect.objectContaining({ code: "ENOENT" })); - }); - }); -}); - -describe("lstat", () => { - it("reports a symlink without following it", async () => { - // POSIX lstat: size is the byte length of the stored target, - // mode is the symlink node's own mode (always 0o777 today). - await withDB((db) => { - writeFileSync(db, "/target", utf8("hello world"), {}, () => 0); - symlink(db, "/target", "/link", () => 0); - const s = lstat(db, "/link"); - expect(s.isSymbolicLink).toBe(true); - expect(s.isFile).toBe(false); - expect(s.isDirectory).toBe(false); - expect(s.size).toBe("/target".length); - expect(s.mode).toBe(0o777); - }); - }); - - it("matches stat for non-symlink nodes", async () => { - await withDB((db) => { - writeFileSync(db, "/a.txt", utf8("hi"), {}, () => 0); - const s = stat(db, "/a.txt"); - const l = lstat(db, "/a.txt"); - expect(l).toEqual(s); - }); - }); - - it("throws ENOENT for a missing path", async () => { - await withDB((db) => { - expect(() => lstat(db, "/missing")).toThrowError(expect.objectContaining({ code: "ENOENT" })); - }); - }); - - it("returns the dangling symlink itself when the target is missing", async () => { - await withDB((db) => { - symlink(db, "/nowhere", "/dangling", () => 0); - const s = lstat(db, "/dangling"); - expect(s.isSymbolicLink).toBe(true); - expect(s.size).toBe("/nowhere".length); - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/stat.ts b/spikes/349-dofs/vendor/dofs/src/fs/stat.ts deleted file mode 100644 index 4ce2734f..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/stat.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { createWorkspaceError } from "../errors.js"; -import { canonicalizePath } from "../path.js"; -import type { Database } from "../storage.js"; -import { resolveInode } from "./resolve.js"; -import { getPendingWriteBufferByPath, getWriteBuffer } from "./writeBuffer.js"; - -export interface WorkspaceStatResult { - name: string; - // Inode of the resolved node, or 0 for a pending-create file that - // has no inode yet. Exposed so provider stat surfaces can read the - // inode from the same resolve instead of walking the path twice. - inode: number; - mode: number; - mtime: number; - size: number; - isFile: boolean; - isDirectory: boolean; - // True when the result describes a symlink itself rather than - // its target. Only lstat() can produce a true value here; stat() - // follows links and reports the final node. - isSymbolicLink: boolean; -} - -export function stat(db: Database, path: string): WorkspaceStatResult { - return statShared(db, path, true); -} - -// Like stat, but does not follow a trailing symlink. Mirrors POSIX -// lstat: the returned size for a symlink is the byte length of the -// stored target, and mode is the symlink node's own mode. -export function lstat(db: Database, path: string): WorkspaceStatResult { - return statShared(db, path, false); -} - -function statShared(db: Database, path: string, followFinal: boolean): WorkspaceStatResult { - const { name, path: canonical } = canonicalizePath(path); - // Pending-create files have no inode yet; serve the buffer state - // so callers between create and release see the file as it stands. - // Pending creates never apply to symlinks, so this is safe to run - // even on the lstat path — a hit here always corresponds to a - // file mid-open. - const pending = getPendingWriteBufferByPath(db, canonical); - if (pending !== undefined && pending.pending !== undefined) { - return { - name, - // A pending create has no inode until releaseWriteBufferSync - // commits it; report 0, which yields nlink 1 in the provider. - inode: 0, - mode: pending.mode & 0o7777, - mtime: pending.pending.mtime, - size: pending.size, - isFile: true, - isDirectory: false, - isSymbolicLink: false, - }; - } - const node = resolveInode(db, path, { followSymlinks: followFinal }); - if (node === null) { - throw createWorkspaceError("ENOENT", `no such path: ${path}`, path); - } - - const isDirectory = node.type === "dir"; - const isFile = node.type === "file"; - const isSymbolicLink = node.type === "symlink"; - let size = 0; - if (isFile) { - // Prefer the in-memory buffer when an open file has unflushed - // writes; otherwise read the cached size off vfs_nodes that - // resolveInode just loaded for us, no extra SQL. - const buffered = getWriteBuffer(db, node.inode); - size = buffered?.dirty ? buffered.size : node.size; - } else if (isSymbolicLink) { - size = (node.linkTarget ?? "").length; - } - - return { - name, - inode: node.inode, - mode: node.mode, - mtime: node.mtime, - size, - isFile, - isDirectory, - isSymbolicLink, - }; -} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/symlink.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/symlink.test.ts deleted file mode 100644 index f4317a3e..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/symlink.test.ts +++ /dev/null @@ -1,150 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { mkdir } from "./mkdir.js"; -import { invalidateReadOnlyMountCache } from "./mount-guard.js"; -import { readlink } from "./readlink.js"; -import { resolveInode } from "./resolve.js"; -import { symlink } from "./symlink.js"; -import { withDB } from "./with-db.js"; -import { writeFile } from "./writeFile.js"; - -describe("symlink", () => { - it("creates a symlink node with the requested target", async () => { - await withDB((db) => { - symlink(db, "/target", "/link", () => 5000); - expect(readlink(db, "/link")).toBe("/target"); - }); - }); - - it("rejects EEXIST when the path already exists", async () => { - await withDB(async (db) => { - await writeFile(db, "/a", "x", {}, () => 0); - expect(() => symlink(db, "/wherever", "/a", () => 0)).toThrowError( - expect.objectContaining({ code: "EEXIST" }), - ); - }); - }); - - it("rejects ENOENT when the parent directory is missing", async () => { - await withDB((db) => { - expect(() => symlink(db, "/t", "/no/such/parent/link", () => 0)).toThrowError( - expect.objectContaining({ code: "ENOENT" }), - ); - }); - }); - - it("bumps rev and records mtime", async () => { - await withDB((db) => { - const before = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; - symlink(db, "/t", "/link", () => 4242); - const after = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; - expect(after).toBe(before + 1); - const row = db.one<{ mtime: number; rev: number; link_target: string }>( - "SELECT n.mtime, n.rev, n.link_target FROM vfs_nodes n JOIN vfs_dirents d ON d.child_inode = n.inode WHERE d.name = ?", - "link", - ); - expect(row?.mtime).toBe(4242); - expect(row?.rev).toBe(after); - expect(row?.link_target).toBe("/t"); - }); - }); - - it("rejects EROFS when the link path overlaps a read-only mount root", async () => { - // Same guard that writeFile and mkdir consult — a symlink that - // lands inside a read-only mount is a write that the indexer - // must reject before the node table sees it. - await withDB((db) => { - db.run("INSERT INTO _vfs_mounts (root, kind, mode) VALUES ('/mnt', 'r2', 'read-only')"); - invalidateReadOnlyMountCache(db); - expect(() => symlink(db, "/elsewhere", "/mnt/link", () => 0)).toThrowError( - expect.objectContaining({ code: "EROFS" }), - ); - }); - }); - - it("creates a symlink inside a nested directory", async () => { - await withDB((db) => { - mkdir(db, "/a/b", { recursive: true }, () => 0); - symlink(db, "/t", "/a/b/link", () => 0); - expect(readlink(db, "/a/b/link")).toBe("/t"); - }); - }); -}); - -describe("readlink", () => { - it("returns the stored target", async () => { - await withDB((db) => { - symlink(db, "/some/target", "/link", () => 0); - expect(readlink(db, "/link")).toBe("/some/target"); - }); - }); - - it("throws ENOENT when the path does not exist", async () => { - await withDB((db) => { - expect(() => readlink(db, "/missing")).toThrowError( - expect.objectContaining({ code: "ENOENT" }), - ); - }); - }); - - it("throws EINVAL when the path is not a symlink", async () => { - await withDB(async (db) => { - await writeFile(db, "/file", "x", {}, () => 0); - expect(() => readlink(db, "/file")).toThrowError(expect.objectContaining({ code: "EINVAL" })); - }); - }); -}); - -describe("resolveInode + symlinks", () => { - it("resolveInode follows symlinks by default", async () => { - await withDB(async (db) => { - await writeFile(db, "/target", "content", {}, () => 0); - symlink(db, "/target", "/link", () => 0); - const node = resolveInode(db, "/link"); - // The target is a file, so following lands on the file node. - expect(node?.type).toBe("file"); - }); - }); - - it("resolveInode with followSymlinks=false returns the link itself", async () => { - await withDB(async (db) => { - await writeFile(db, "/target", "content", {}, () => 0); - symlink(db, "/target", "/link", () => 0); - const node = resolveInode(db, "/link", { followSymlinks: false }); - expect(node?.type).toBe("symlink"); - }); - }); - - it("follows a chain of symlinks", async () => { - await withDB(async (db) => { - await writeFile(db, "/target", "content", {}, () => 0); - symlink(db, "/target", "/a", () => 0); - symlink(db, "/a", "/b", () => 0); - symlink(db, "/b", "/c", () => 0); - expect(resolveInode(db, "/c")?.type).toBe("file"); - }); - }); - - it("returns null on a dangling symlink when following", async () => { - await withDB((db) => { - symlink(db, "/no/such/target", "/dangling", () => 0); - expect(resolveInode(db, "/dangling")).toBeNull(); - }); - }); - - it("returns the symlink node on a dangling symlink with followSymlinks=false", async () => { - await withDB((db) => { - symlink(db, "/no/such/target", "/dangling", () => 0); - const node = resolveInode(db, "/dangling", { followSymlinks: false }); - expect(node?.type).toBe("symlink"); - }); - }); - - it("throws ELOOP on a cycle", async () => { - await withDB((db) => { - symlink(db, "/b", "/a", () => 0); - symlink(db, "/a", "/b", () => 0); - expect(() => resolveInode(db, "/a")).toThrowError(expect.objectContaining({ code: "ELOOP" })); - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/symlink.ts b/spikes/349-dofs/vendor/dofs/src/fs/symlink.ts deleted file mode 100644 index cbd6f2e9..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/symlink.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { createWorkspaceError } from "../errors.js"; -import { canonicalizePath } from "../path.js"; -import { incrementRev } from "../rev.js"; -import { ROOT_INODE } from "../schema/index.js"; -import type { Database } from "../storage.js"; -import { assertNotReadOnly } from "./mount-guard.js"; -import { invalidateResolveSubtree } from "./resolveCache.js"; - -// Create a symlink node. The target is stored as-is — it can be a -// relative or absolute path, dangling or live. resolveInode follows -// it transparently when callers walk through this entry. -export function symlink(db: Database, target: string, path: string, now: () => number): void { - const { parts, path: canonical } = canonicalizePath(path); - if (parts.length === 0) { - throw createWorkspaceError("EEXIST", "cannot symlink onto root", canonical); - } - assertNotReadOnly(db, canonical); - - db.transactionSync(() => { - // Walk to the parent dirent. Intermediate segments must be real - // directories; we don't auto-create them. - let parentInode = ROOT_INODE; - for (let i = 0; i < parts.length - 1; i++) { - const child = db.one<{ child_inode: number }>( - "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", - parentInode, - parts[i], - ); - if (child === undefined) { - throw createWorkspaceError("ENOENT", `parent directory missing: ${canonical}`, canonical); - } - const next = db.one<{ inode: number; type: "file" | "dir" | "symlink" }>( - "SELECT inode, type FROM vfs_nodes WHERE inode = ?", - child.child_inode, - ); - if (next === undefined || next.type !== "dir") { - throw createWorkspaceError( - "ENOTDIR", - `parent path segment is not a directory: ${canonical}`, - canonical, - ); - } - parentInode = next.inode; - } - - const leafName = parts[parts.length - 1]; - const existing = db.one<{ child_inode: number }>( - "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", - parentInode, - leafName, - ); - if (existing !== undefined) { - throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); - } - - const rev = incrementRev(db); - const mtime = now(); - // RETURNING folds the rowid read into the INSERT. - const row = db.one<{ inode: number }>( - "INSERT INTO vfs_nodes (type, mode, mtime, rev, link_target) VALUES ('symlink', ?, ?, ?, ?) RETURNING inode", - 0o777, - mtime, - rev, - target, - ); - if (row === undefined) { - throw createWorkspaceError("EIO", "failed to allocate inode"); - } - const inode = row.inode; - db.run( - "INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", - parentInode, - leafName, - inode, - ); - // Subtree, not exact: paths *through* the new link (e.g. /s/x when - // /s -> a populated dir) now resolve, so any cached negative - // beneath the link must be dropped. - invalidateResolveSubtree(db, canonical); - }); -} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/unlink.ts b/spikes/349-dofs/vendor/dofs/src/fs/unlink.ts deleted file mode 100644 index 7a7d33c2..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/unlink.ts +++ /dev/null @@ -1,34 +0,0 @@ -import type { Database } from "../storage.js"; - -type NodeType = "file" | "dir" | "symlink"; - -// Remove a single (parent, name) dirent and reap the child inode's -// node and chunk rows only once its last link disappears. A file inode -// can carry several hardlink names, so the node and its chunks survive -// until the final dirent is gone. Returns true when the inode was -// reaped, false when other links keep it alive. -// -// Callers own rev bumps and tombstones; this helper touches only -// vfs_dirents, vfs_chunks, and vfs_nodes. It is the single place the -// refcount-gated reap is implemented — rm, rename, and the sync apply -// path all funnel through here so the invariant lives once. -export function unlinkDirent( - db: Database, - parentInode: number, - name: string, - childInode: number, - type: NodeType, -): boolean { - db.run("DELETE FROM vfs_dirents WHERE parent_inode = ? AND name = ?", parentInode, name); - const remaining = db.scalar( - "SELECT COUNT(*) FROM vfs_dirents WHERE child_inode = ?", - childInode, - ); - if ((remaining ?? 0) > 0) return false; - - if (type === "file") { - db.run("DELETE FROM vfs_chunks WHERE inode = ?", childInode); - } - db.run("DELETE FROM vfs_nodes WHERE inode = ?", childInode); - return true; -} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/watch.ts b/spikes/349-dofs/vendor/dofs/src/fs/watch.ts deleted file mode 100644 index 6c087e6f..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/watch.ts +++ /dev/null @@ -1,163 +0,0 @@ -// Directory watcher backed by polling vfs_meta.rev. -// -// On each tick, coalesceChanges yields every path touched since -// the watcher last looked. We filter by the watched directory and -// recursive flag, then emit one 'change' event per path with -// fs.watch-compatible (eventType, filename) arguments. -// -// The polling cadence is the provider's watchIntervalMs (default -// 100 ms). That's already what node's fs.watch uses internally on -// platforms without inotify, and it's slow enough that the SQL -// scan stays in the noise even with many watchers active. -// -// Coverage: there is no watch.test.ts here because watch is only -// reachable through SQLiteWorkspaceProvider.watch / -// watchAsyncIterable; the test surface is provider.watch.test.ts, -// which exercises both the EventEmitter and the AsyncIterable -// adapters end-to-end. - -import { EventEmitter } from "node:events"; - -import { canonicalizePath } from "../path.js"; -import type { Database } from "../storage.js"; -import { coalesceChanges } from "../sync/coalesce.js"; -import { currentRev } from "../sync/watermarks.js"; - -export interface WatchOptions { - // Recurse into subdirectories. When false (default) the watcher - // only fires for direct children of `path`. - recursive?: boolean; - // AbortSignal that closes the watcher when triggered. - signal?: AbortSignal; - // Override the poll interval. Default comes from the provider. - interval?: number; -} - -export interface WatchEvent { - eventType: "rename" | "change"; - filename: string; -} - -export interface WatchHandle extends EventEmitter { - close(): void; -} - -export function createWatcher( - db: Database, - path: string, - options: WatchOptions, - defaultInterval: number, -): WatchHandle { - const { path: canonical } = canonicalizePath(path); - const prefix = canonical === "/" ? "/" : `${canonical}/`; - const recursive = options.recursive === true; - const interval = options.interval ?? defaultInterval; - - const emitter = new EventEmitter() as WatchHandle; - let cursor = currentRev(db); - let closed = false; - - const tick = async () => { - if (closed) return; - try { - const seen = new Set(); - for await (const entry of coalesceChanges(db, cursor)) { - // Filter to entries inside the watched scope. - if (!isInScope(entry.path, canonical, prefix, recursive)) continue; - if (seen.has(entry.path)) continue; - seen.add(entry.path); - const filename = relativeName(entry.path, canonical); - const eventType: "rename" | "change" = entry.kind === "delete" ? "rename" : "change"; - emitter.emit("change", eventType, filename); - } - cursor = currentRev(db); - } catch (error) { - emitter.emit("error", error); - } - }; - - const handle = setInterval(() => void tick(), interval); - handle.unref?.(); - - emitter.close = () => { - if (closed) return; - closed = true; - clearInterval(handle); - emitter.emit("close"); - }; - - if (options.signal !== undefined) { - if (options.signal.aborted) { - emitter.close(); - } else { - options.signal.addEventListener("abort", () => emitter.close(), { - once: true, - }); - } - } - - return emitter; -} - -function isInScope( - entryPath: string, - watchedPath: string, - prefix: string, - recursive: boolean, -): boolean { - if (entryPath === watchedPath) return true; - if (!entryPath.startsWith(prefix)) return false; - if (recursive) return true; - // Non-recursive: only direct children. No extra '/' in the - // remainder past the prefix. - const remainder = entryPath.slice(prefix.length); - return !remainder.includes("/"); -} - -function relativeName(entryPath: string, watchedPath: string): string { - if (entryPath === watchedPath) return ""; - const prefix = watchedPath === "/" ? "/" : `${watchedPath}/`; - return entryPath.startsWith(prefix) ? entryPath.slice(prefix.length) : entryPath; -} - -// Adapter from EventEmitter-based watcher to AsyncIterable for -// for-await consumers. Mirrors @platformatic/vfs's VFSWatchAsyncIterable. -export function createWatchAsyncIterable(watcher: WatchHandle): AsyncIterable & { - return(): Promise<{ value: undefined; done: true }>; -} { - const pending: WatchEvent[] = []; - const waiters: ((result: IteratorResult) => void)[] = []; - let done = false; - - watcher.on("change", (eventType: "rename" | "change", filename: string) => { - const event: WatchEvent = { eventType, filename }; - const next = waiters.shift(); - if (next) next({ value: event, done: false }); - else pending.push(event); - }); - watcher.on("close", () => { - done = true; - while (waiters.length > 0) { - const next = waiters.shift(); - if (next) next({ value: undefined as never, done: true }); - } - }); - - return { - [Symbol.asyncIterator]() { - return this as unknown as AsyncIterator; - }, - next(): Promise> { - const buffered = pending.shift(); - if (buffered) return Promise.resolve({ value: buffered, done: false }); - if (done) return Promise.resolve({ value: undefined as never, done: true }); - return new Promise((resolve) => waiters.push(resolve)); - }, - async return() { - watcher.close(); - return { value: undefined, done: true as const }; - }, - } as unknown as AsyncIterable & { - return(): Promise<{ value: undefined; done: true }>; - }; -} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/with-db.ts b/spikes/349-dofs/vendor/dofs/src/fs/with-db.ts deleted file mode 100644 index 38f82ff7..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/with-db.ts +++ /dev/null @@ -1,55 +0,0 @@ -// Node-backed implementation. Selected by the default vitest config. -// The workers config aliases this module to ./with-db.workers.ts so the -// same test source runs against a real Durable Object. - -import { initializeSchema } from "../schema/index.js"; -import { Database } from "../storage.js"; -import { SQLiteTestStorage } from "../testing.js"; - -export interface WithDBOptions { - now?: () => number; -} - -export async function withDB( - fn: (db: Database) => T | Promise, - options: WithDBOptions = {}, -): Promise { - const storage = new SQLiteTestStorage(); - const db = new Database(storage); - initializeSchema(db, options.now ?? (() => 1000)); - try { - return await fn(db); - } finally { - storage.close(); - } -} - -// Two independent DBs split across a snapshot/apply pair. The shape -// matches the workerd backend, which can't hold two DOs alive at -// once because of cross-DO I/O isolation — the snapshot pass -// captures everything the apply pass needs as plain serializable -// values. -// -// Under node we run both callbacks in the same process with two -// SQLiteTestStorage instances. Either shape would work here; we -// match the workerd API so test code stays uniform. -export async function withTwoDBs( - snapshot: (a: Database) => S | Promise, - apply: (b: Database, snapshot: S) => T | Promise, - options: WithDBOptions = {}, -): Promise { - const storageA = new SQLiteTestStorage(); - const storageB = new SQLiteTestStorage(); - const a = new Database(storageA); - const b = new Database(storageB); - const now = options.now ?? (() => 1000); - initializeSchema(a, now); - initializeSchema(b, now); - try { - const captured = await snapshot(a); - return await apply(b, captured); - } finally { - storageA.close(); - storageB.close(); - } -} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/with-db.workers.ts b/spikes/349-dofs/vendor/dofs/src/fs/with-db.workers.ts deleted file mode 100644 index addfaf7a..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/with-db.workers.ts +++ /dev/null @@ -1,68 +0,0 @@ -// Workers-backed implementation. Vitest config aliases ./with-db.js to -// this file when running under @cloudflare/vitest-pool-workers, so every -// test that calls withDB(fn) ends up driving a real Durable Object's -// SQLite storage instead of a node:sqlite in-memory DB. - -import { env, runInDurableObject } from "cloudflare:test"; -import type { TestBindings } from "../../tests/worker.js"; -import { initializeSchema } from "../schema/index.js"; -import { Database } from "../storage.js"; -import type { DurableObjectStorageLike } from "../types.js"; - -export interface WithDBOptions { - now?: () => number; -} - -// Each call gets a fresh DO instance so tests don't bleed into each -// other. newUniqueId() gives a name that never collides between runs. -function freshStub() { - const id = (env as unknown as TestBindings).TestStorage.newUniqueId(); - return (env as unknown as TestBindings).TestStorage.get(id); -} - -export async function withDB( - fn: (db: Database) => T | Promise, - options: WithDBOptions = {}, -): Promise { - const stub = freshStub(); - return runInDurableObject(stub, async (_instance: unknown, state: DurableObjectState) => { - const db = new Database(state.storage as unknown as DurableObjectStorageLike); - initializeSchema(db, options.now ?? (() => 1000)); - return await fn(db); - }); -} - -// Two independent DOs. Each newUniqueId() gives a fresh DO with its -// own SqlStorage. We can't nest runInDurableObject calls because -// workerd hard-isolates I/O objects between DOs in the same isolate -// (the Database wrapper captures the SqlStorage as such an object). -// Snapshot A's outputs to plain serializable values inside its DO -// context, then apply them in B's. The caller drives the two passes -// via a snapshot and apply callback pair. -// -// This is the workerd shape; the node-side withTwoDBs runs both -// callbacks in the same process with two real SQLiteTestStorage -// instances. Tests that need the simpler signature should branch on -// the node-only path. -export async function withTwoDBs( - snapshot: (a: Database) => S | Promise, - apply: (b: Database, snapshot: S) => T | Promise, - options: WithDBOptions = {}, -): Promise { - const stubA = freshStub(); - const stubB = freshStub(); - const now = options.now ?? (() => 1000); - const captured = await runInDurableObject( - stubA, - async (_a: unknown, stateA: DurableObjectState) => { - const a = new Database(stateA.storage as unknown as DurableObjectStorageLike); - initializeSchema(a, now); - return await snapshot(a); - }, - ); - return runInDurableObject(stubB, async (_b: unknown, stateB: DurableObjectState) => { - const b = new Database(stateB.storage as unknown as DurableObjectStorageLike); - initializeSchema(b, now); - return await apply(b, captured); - }); -} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/writeBuffer.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/writeBuffer.test.ts deleted file mode 100644 index 891f5e3e..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/writeBuffer.test.ts +++ /dev/null @@ -1,273 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import type { Database } from "../storage.js"; -import { readRangeSync } from "./readFile.js"; -import { resolveInode } from "./resolve.js"; -import { stat } from "./stat.js"; -import { withDB } from "./with-db.js"; -import { - CHUNK_SIZE, - createFileSync, - openWriteBufferForCreateSync, - openWriteBufferSync, - releaseWriteBufferSync, - truncateFileSync, - writeRangeSync, -} from "./writeFile.js"; - -function bytesOf(text: string): Uint8Array { - return new TextEncoder().encode(text); -} - -function blobCount(db: Database): number { - return db.scalar("SELECT COUNT(*) FROM vfs_blobs") ?? 0; -} - -function orphanBlobCount(db: Database): number { - return ( - db.scalar( - "SELECT COUNT(*) FROM vfs_blobs b WHERE NOT EXISTS (SELECT 1 FROM vfs_chunks c WHERE c.hash = b.hash)", - ) ?? 0 - ); -} - -function chunkCount(db: Database, path: string): number { - const node = resolveInode(db, path); - if (node === null) throw new Error(`missing node: ${path}`); - return db.scalar("SELECT COUNT(*) FROM vfs_chunks WHERE inode = ?", node.inode) ?? 0; -} - -describe("buffered write lifecycle", () => { - it("buffers many small writes and stages blobs only on release", async () => { - await withDB(async (db) => { - createFileSync(db, "/buffered.bin", {}, () => 1000); - openWriteBufferSync(db, "/buffered.bin"); - - const big = new Uint8Array(CHUNK_SIZE); - big.fill(7); - // Write the same chunk-sized payload eight times at offset 0. - // Pre-buffer, each write would stage one orphan blob per - // intermediate state. - for (let i = 0; i < 8; i++) { - writeRangeSync(db, "/buffered.bin", big, 0, {}, () => 1001 + i); - } - // While the buffer is open we keep no chunk or blob rows yet. - expect(blobCount(db)).toBe(0); - - releaseWriteBufferSync(db, "/buffered.bin", () => 1100); - - // After release: exactly one blob, no orphans, content matches. - expect(blobCount(db)).toBe(1); - expect(orphanBlobCount(db)).toBe(0); - const final = readRangeSync(db, "/buffered.bin", 0, CHUNK_SIZE); - expect(final.byteLength).toBe(CHUNK_SIZE); - expect(final[0]).toBe(7); - }); - }); - - it("serves buffered reads before release", async () => { - await withDB(async (db) => { - createFileSync(db, "/buffered.txt", {}, () => 1000); - openWriteBufferSync(db, "/buffered.txt"); - - writeRangeSync(db, "/buffered.txt", bytesOf("hello"), 0, {}, () => 1001); - // Reading through the same db sees the buffered bytes even - // though no chunk row has been written. - expect(new TextDecoder().decode(readRangeSync(db, "/buffered.txt", 0, 5))).toBe("hello"); - expect(chunkCount(db, "/buffered.txt")).toBe(0); - - releaseWriteBufferSync(db, "/buffered.txt", () => 1100); - expect(new TextDecoder().decode(readRangeSync(db, "/buffered.txt", 0, 5))).toBe("hello"); - }); - }); - - it("truncate updates the buffer instead of rewriting chunks", async () => { - await withDB(async (db) => { - createFileSync(db, "/trunc.bin", {}, () => 1000); - openWriteBufferSync(db, "/trunc.bin"); - - const payload = new Uint8Array(CHUNK_SIZE * 2); - payload.fill(1); - writeRangeSync(db, "/trunc.bin", payload, 0, {}, () => 1001); - truncateFileSync(db, "/trunc.bin", CHUNK_SIZE - 100, () => 1002); - expect(chunkCount(db, "/trunc.bin")).toBe(0); - - releaseWriteBufferSync(db, "/trunc.bin", () => 1100); - expect(chunkCount(db, "/trunc.bin")).toBe(1); - const final = readRangeSync(db, "/trunc.bin", 0, CHUNK_SIZE); - expect(final.byteLength).toBe(CHUNK_SIZE - 100); - }); - }); - - it("hardlinks share the same buffered bytes by inode", async () => { - await withDB(async (db) => { - createFileSync(db, "/a.txt", {}, () => 1000); - // Both paths point at the same inode. Open under /a.txt, then - // write under /b.txt: the buffer is keyed by inode so the write - // lands in the same cache entry. - const { link } = await import("./link.js"); - link(db, "/a.txt", "/b.txt"); - openWriteBufferSync(db, "/a.txt"); - - // Multiple intermediate writes through both paths. Pre-buffer - // each one would have staged its own blob and orphaned the - // previous state; the buffer keeps them in memory until release. - writeRangeSync(db, "/b.txt", bytesOf("step-1"), 0, {}, () => 1001); - writeRangeSync(db, "/a.txt", bytesOf("step-2"), 0, {}, () => 1002); - writeRangeSync(db, "/b.txt", bytesOf("shared"), 0, {}, () => 1003); - expect(new TextDecoder().decode(readRangeSync(db, "/a.txt", 0, 6))).toBe("shared"); - expect(blobCount(db)).toBe(0); - - releaseWriteBufferSync(db, "/a.txt", () => 1100); - expect(new TextDecoder().decode(readRangeSync(db, "/b.txt", 0, 6))).toBe("shared"); - // Exactly one blob for the final state, regardless of how many - // intermediate writes the open window saw. - expect(blobCount(db)).toBe(1); - }); - }); - - it("commits a chunked file with one blob per chunk on release", async () => { - await withDB(async (db) => { - createFileSync(db, "/big.bin", {}, () => 1000); - openWriteBufferSync(db, "/big.bin"); - - // Three chunks of distinct content. Pre-buffer, each write - // would create the chunk row eagerly and a partial-tail write - // would create an orphan blob for the previous tail size. - const payload = new Uint8Array(CHUNK_SIZE * 3); - payload.fill(1, 0, CHUNK_SIZE); - payload.fill(2, CHUNK_SIZE, CHUNK_SIZE * 2); - payload.fill(3, CHUNK_SIZE * 2); - writeRangeSync(db, "/big.bin", payload, 0, {}, () => 1001); - - releaseWriteBufferSync(db, "/big.bin", () => 1100); - expect(chunkCount(db, "/big.bin")).toBe(3); - expect(orphanBlobCount(db)).toBe(0); - }); - }); -}); - -describe("deferred-create lifecycle", () => { - it("holds the file in memory until release commits one transaction", async () => { - await withDB(async (db) => { - openWriteBufferForCreateSync(db, "/pending.txt", { mode: 0o600 }, () => 1000); - - // No SQL row exists yet but the path is reachable via the - // path-keyed pending cache: stat, read, and write all see it. - expect(stat(db, "/pending.txt").size).toBe(0); - writeRangeSync(db, "/pending.txt", bytesOf("hello"), 0, {}, () => 1001); - expect(stat(db, "/pending.txt").size).toBe(5); - expect(new TextDecoder().decode(readRangeSync(db, "/pending.txt", 0, 5))).toBe("hello"); - expect(resolveInode(db, "/pending.txt")).toBeNull(); - expect(blobCount(db)).toBe(0); - - releaseWriteBufferSync(db, "/pending.txt", () => 1100); - - // Release committed the INSERT, dirent, and chunk rows in one - // transaction; the path now resolves to a real inode and the - // stat sees the persisted size. - const node = resolveInode(db, "/pending.txt"); - expect(node?.type).toBe("file"); - expect(node?.mode).toBe(0o600); - expect(stat(db, "/pending.txt").size).toBe(5); - expect(blobCount(db)).toBe(1); - expect(orphanBlobCount(db)).toBe(0); - }); - }); - - it("rejects a second openWriteBufferForCreateSync against the same path", async () => { - await withDB(async (db) => { - openWriteBufferForCreateSync(db, "/dupe.txt", {}, () => 1000); - expect(() => openWriteBufferForCreateSync(db, "/dupe.txt", {}, () => 1001)).toThrowError( - expect.objectContaining({ code: "EEXIST" }), - ); - }); - }); - - it("surfaces pending files in readdir before release", async () => { - await withDB(async (db) => { - const { readdir } = await import("./readdir.js"); - const { mkdir } = await import("./mkdir.js"); - mkdir(db, "/d", {}, () => 1000); - openWriteBufferForCreateSync(db, "/d/pending.txt", {}, () => 1001); - - const names = readdir(db, "/d").map((entry) => entry.name); - expect(names).toEqual(["pending.txt"]); - - releaseWriteBufferSync(db, "/d/pending.txt", () => 1100); - const after = readdir(db, "/d").map((entry) => entry.name); - expect(after).toEqual(["pending.txt"]); - }); - }); - - it("defers commit until the matching release count is reached", async () => { - await withDB(async (db) => { - createFileSync(db, "/multi.txt", {}, () => 1000); - writeRangeSync(db, "/multi.txt", bytesOf("seed"), 0, {}, () => 1001); - // Establish the on-disk shape we'll observe against. - const seedBlobs = blobCount(db); - const seedChunks = chunkCount(db, "/multi.txt"); - - // Two opens of the same path share a single inode-keyed entry. - openWriteBufferSync(db, "/multi.txt"); - openWriteBufferSync(db, "/multi.txt"); - writeRangeSync(db, "/multi.txt", bytesOf("DIRTY"), 0, {}, () => 1002); - - // First release decrements but does not commit — chunk/blob - // shape unchanged. - releaseWriteBufferSync(db, "/multi.txt", () => 1003); - expect(blobCount(db)).toBe(seedBlobs); - expect(chunkCount(db, "/multi.txt")).toBe(seedChunks); - - // Second release commits. Reading back through the chunk - // store sees the dirty bytes. - releaseWriteBufferSync(db, "/multi.txt", () => 1004); - const final = readRangeSync(db, "/multi.txt", 0, 5); - expect(new TextDecoder().decode(final)).toBe("DIRTY"); - - // A fresh open after release starts a clean buffer; an - // immediate release without writes is a no-op. - openWriteBufferSync(db, "/multi.txt"); - releaseWriteBufferSync(db, "/multi.txt", () => 1005); - // No corruption: file content still matches. - const stable = readRangeSync(db, "/multi.txt", 0, 5); - expect(new TextDecoder().decode(stable)).toBe("DIRTY"); - }); - }); - - it("flushPendingByPath promotes the buffer without consuming the open count", async () => { - const { flushPendingByPath } = await import("./writeFile.js"); - await withDB(async (db) => { - openWriteBufferForCreateSync(db, "/promote.txt", {}, () => 1000); - writeRangeSync(db, "/promote.txt", bytesOf("before"), 0, {}, () => 1001); - expect(resolveInode(db, "/promote.txt")).toBeNull(); - - // Promote the pending entry without releasing it. - const committed = flushPendingByPath(db, "/promote.txt", () => 1002); - expect(committed).toBe(true); - - // The path now resolves; the open buffer survived the - // promotion under the real inode key. - const node = resolveInode(db, "/promote.txt"); - expect(node?.type).toBe("file"); - expect(chunkCount(db, "/promote.txt")).toBeGreaterThan(0); - - // Further writes route through the inode-keyed cache. The - // matching release commits the final bytes over the - // promoted state. - writeRangeSync(db, "/promote.txt", bytesOf("after-x"), 0, {}, () => 1003); - releaseWriteBufferSync(db, "/promote.txt", () => 1004); - const final = readRangeSync(db, "/promote.txt", 0, 7); - expect(new TextDecoder().decode(final)).toBe("after-x"); - }); - }); - - it("flushPendingByPath returns false when no pending buffer is open", async () => { - const { flushPendingByPath } = await import("./writeFile.js"); - await withDB(async (db) => { - createFileSync(db, "/already.txt", {}, () => 1000); - expect(flushPendingByPath(db, "/already.txt", () => 1001)).toBe(false); - expect(flushPendingByPath(db, "/missing.txt", () => 1002)).toBe(false); - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/writeBuffer.ts b/spikes/349-dofs/vendor/dofs/src/fs/writeBuffer.ts deleted file mode 100644 index 72ff047a..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/writeBuffer.ts +++ /dev/null @@ -1,138 +0,0 @@ -// In-process write buffer cache. -// -// Holds per-inode mutable byte buffers between an explicit open and -// release. While a buffer is open, all reads and writes for that -// inode go through the buffer rather than the SQLite blob/chunk -// store. Release commits the bytes to chunks once per file -// and evicts the entry, so per-syscall writes no longer accumulate -// orphan blob rows in the store. -// -// The cache is keyed by Database so a fresh database (a test, a -// rebooted DO incarnation) starts with an empty cache. - -import type { Database } from "../storage.js"; - -export interface WriteBufferEntry { - // Growable backing store. byteLength is capacity; logical length - // lives in `size`. - buf: Uint8Array; - // Logical end-of-file in `buf`. - size: number; - // True once writeRange/truncate mutates the buffer. A non-dirty - // buffer is one that the caller opened but never wrote to; release - // is a no-op in that case so we do not touch the existing chunks. - dirty: boolean; - // Open handle count. Each FUSE open/create increments this; each - // release decrements. The buffer commits and evicts when the count - // reaches zero. - openCount: number; - // Mode the caller wants persisted on release. Defaults to the - // inode's existing mode at open time when the caller has none. - mode: number; - // Pending-create state. When set, no inode row exists yet; release - // will INSERT the node + dirent + chunks in one transaction. The - // synthetic inode id used to key this entry in the cache is stored - // here so release can find and remove the entry without scanning - // the cache. - pending?: { - parentInode: number; - leafName: string; - canonicalPath: string; - pendingInode: number; - mtime: number; - }; -} - -interface DatabaseCache { - byInode: Map; - byPendingPath: Map; - nextPendingInode: number; -} - -const caches = new WeakMap(); - -function cacheFor(db: Database): DatabaseCache { - let cache = caches.get(db); - if (cache === undefined) { - cache = { byInode: new Map(), byPendingPath: new Map(), nextPendingInode: -1 }; - caches.set(db, cache); - } - return cache; -} - -export function getWriteBuffer(db: Database, inode: number): WriteBufferEntry | undefined { - return caches.get(db)?.byInode.get(inode); -} - -export function getPendingWriteBufferByPath( - db: Database, - canonicalPath: string, -): WriteBufferEntry | undefined { - return caches.get(db)?.byPendingPath.get(canonicalPath); -} - -// List pending-create buffers whose parent dirent matches `parentInode`. -// Used by readdir so freshly-created-but-not-yet-released files show -// up in directory listings between open and release. -export function listPendingByParent(db: Database, parentInode: number): WriteBufferEntry[] { - const cache = caches.get(db); - if (cache === undefined) return []; - const out: WriteBufferEntry[] = []; - for (const entry of cache.byPendingPath.values()) { - if (entry.pending?.parentInode === parentInode) out.push(entry); - } - return out; -} - -export function setWriteBuffer(db: Database, inode: number, entry: WriteBufferEntry): void { - const cache = cacheFor(db); - cache.byInode.set(inode, entry); - if (entry.pending !== undefined) { - cache.byPendingPath.set(entry.pending.canonicalPath, entry); - } -} - -export function deleteWriteBuffer(db: Database, inode: number): void { - const cache = caches.get(db); - if (cache === undefined) return; - const entry = cache.byInode.get(inode); - if (entry?.pending !== undefined) { - cache.byPendingPath.delete(entry.pending.canonicalPath); - } - cache.byInode.delete(inode); -} - -// Allocate a synthetic negative inode id for a pending file. The -// real id is assigned by SQLite when release INSERTs the node row; -// the synthetic value just lets the buffer cache key entries -// before that point. -export function allocatePendingInode(db: Database): number { - const cache = cacheFor(db); - const next = cache.nextPendingInode; - cache.nextPendingInode -= 1; - return next; -} - -// Re-key a pending entry to the real inode assigned by SQLite at -// commit time, dropping the pending-path index. -export function promotePendingToInode(db: Database, pendingInode: number, realInode: number): void { - const cache = caches.get(db); - if (cache === undefined) return; - const entry = cache.byInode.get(pendingInode); - if (entry === undefined) return; - if (entry.pending !== undefined) { - cache.byPendingPath.delete(entry.pending.canonicalPath); - entry.pending = undefined; - } - cache.byInode.delete(pendingInode); - cache.byInode.set(realInode, entry); -} - -export function ensureCapacity(entry: WriteBufferEntry, needed: number): void { - if (entry.buf.byteLength >= needed) return; - let cap = Math.max(entry.buf.byteLength * 2, 64 * 1024); - while (cap < needed) cap *= 2; - const next = new Uint8Array(cap); - next.set(entry.buf.subarray(0, entry.size), 0); - entry.buf = next; -} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/writeFile.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/writeFile.test.ts deleted file mode 100644 index d8df6b74..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/writeFile.test.ts +++ /dev/null @@ -1,395 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { ROOT_INODE } from "../schema/index.js"; -import type { Database } from "../storage.js"; -import { mkdir } from "./mkdir.js"; -import { resolveInode } from "./resolve.js"; -import { withDB } from "./with-db.js"; -import { CHUNK_SIZE, writeFile, writeFileRangesSync, writeFileSync } from "./writeFile.js"; - -// Reassemble a file's bytes by stitching its chunk rows together. -// A deliberately minimal helper so writeFile tests can stand alone -// without depending on readFile. -function readBack(db: Database, path: string): Uint8Array { - const node = resolveInode(db, path); - if (node === null) throw new Error(`no such path: ${path}`); - if (node.type !== "file") throw new Error(`not a file: ${path}`); - const chunks = db.all<{ hash: Uint8Array; size: number }>( - "SELECT hash, size FROM vfs_chunks WHERE inode = ? ORDER BY idx", - node.inode, - ); - const parts: Uint8Array[] = []; - let total = 0; - for (const chunk of chunks) { - const row = db.one<{ bytes: Uint8Array }>( - "SELECT bytes FROM vfs_blob_bytes WHERE hash = ?", - chunk.hash, - ); - if (row === undefined) throw new Error("missing blob bytes"); - parts.push(row.bytes); - total += row.bytes.byteLength; - } - const out = new Uint8Array(total); - let offset = 0; - for (const part of parts) { - out.set(part, offset); - offset += part.byteLength; - } - return out; -} - -function chunkRows(db: Database, path: string): Array<{ hash: Uint8Array; size: number }> { - const node = resolveInode(db, path); - if (node === null) throw new Error(`no such path: ${path}`); - return db.all<{ hash: Uint8Array; size: number }>( - "SELECT hash, size FROM vfs_chunks WHERE inode = ? ORDER BY idx", - node.inode, - ); -} - -function countBlobs(db: Database): number { - return db.scalar("SELECT COUNT(*) FROM vfs_blobs") ?? 0; -} - -function streamOf(...chunks: Uint8Array[]): ReadableStream { - let i = 0; - return new ReadableStream({ - pull(controller) { - if (i < chunks.length) { - controller.enqueue(chunks[i++]); - } else { - controller.close(); - } - }, - }); -} - -describe("writeFile", () => { - it("writes a small string and stores one chunk", async () => { - await withDB(async (db) => { - await writeFile(db, "/hello.txt", "hello fuse", {}, () => 1234); - - const bytes = readBack(db, "/hello.txt"); - expect(new TextDecoder().decode(bytes)).toBe("hello fuse"); - - const chunkCount = db.scalar( - "SELECT COUNT(*) FROM vfs_chunks WHERE inode = (SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?)", - ROOT_INODE, - "hello.txt", - ); - expect(chunkCount).toBe(1); - }); - }); - - it("writeFileSync stores small strings as a single chunk row", async () => { - await withDB(async (db) => { - writeFileSync(db, "/hello.txt", new TextEncoder().encode("hello fuse"), {}, () => 1234); - - const bytes = readBack(db, "/hello.txt"); - expect(new TextDecoder().decode(bytes)).toBe("hello fuse"); - - const chunkCount = db.scalar( - "SELECT COUNT(*) FROM vfs_chunks WHERE inode = (SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?)", - ROOT_INODE, - "hello.txt", - ); - expect(chunkCount).toBe(1); - }); - }); - - it("atomically rejects an exclusive write when the target exists", async () => { - await withDB(async (db) => { - await writeFile(db, "/exclusive.txt", "first", {}, () => 1); - await expect( - writeFile(db, "/exclusive.txt", "second", { exclusive: true }, () => 2), - ).rejects.toMatchObject({ code: "EEXIST" }); - expect(new TextDecoder().decode(readBack(db, "/exclusive.txt"))).toBe("first"); - }); - }); - - it("rejects an exclusive streaming write before reading its source", async () => { - await withDB(async (db) => { - await writeFile(db, "/exclusive-stream.txt", "first", {}, () => 1); - let pulls = 0; - const source = new ReadableStream( - { - pull(controller) { - pulls += 1; - controller.enqueue(new TextEncoder().encode("second")); - }, - }, - { highWaterMark: 0 }, - ); - await expect( - writeFile(db, "/exclusive-stream.txt", source, { exclusive: true }, () => 2), - ).rejects.toMatchObject({ code: "EEXIST" }); - expect(pulls).toBe(0); - }); - }); - - it("accepts a Uint8Array", async () => { - await withDB(async (db) => { - const data = new Uint8Array([1, 2, 3, 4, 5]); - await writeFile(db, "/data.bin", data, {}, () => 0); - expect(Array.from(readBack(db, "/data.bin"))).toEqual([1, 2, 3, 4, 5]); - }); - }); - - it("accepts a ReadableStream and joins its chunks", async () => { - await withDB(async (db) => { - await writeFile( - db, - "/streamed.txt", - streamOf(new TextEncoder().encode("hello "), new TextEncoder().encode("stream")), - {}, - () => 0, - ); - expect(new TextDecoder().decode(readBack(db, "/streamed.txt"))).toBe("hello stream"); - }); - }); - - it("writes an empty file (zero chunks, zero size)", async () => { - await withDB(async (db) => { - await writeFile(db, "/empty", "", {}, () => 0); - const node = resolveInode(db, "/empty"); - expect(node?.type).toBe("file"); - const chunks = db.scalar( - "SELECT COUNT(*) FROM vfs_chunks WHERE inode = ?", - node?.inode, - ); - expect(chunks).toBe(0); - }); - }); - - it("splits content larger than CHUNK_SIZE across multiple chunks", async () => { - await withDB(async (db) => { - const oneChunk = new Uint8Array(CHUNK_SIZE); - oneChunk.fill(0x41); - const trailing = new Uint8Array(100); - trailing.fill(0x42); - const combined = new Uint8Array(CHUNK_SIZE + 100); - combined.set(oneChunk, 0); - combined.set(trailing, CHUNK_SIZE); - await writeFile(db, "/big", combined, {}, () => 0); - - const node = resolveInode(db, "/big"); - const chunkCount = db.scalar( - "SELECT COUNT(*) FROM vfs_chunks WHERE inode = ?", - node?.inode, - ); - expect(chunkCount).toBe(2); - - const sizes = db - .all<{ idx: number; size: number }>( - "SELECT idx, size FROM vfs_chunks WHERE inode = ? ORDER BY idx", - node?.inode, - ) - .map((r) => r.size); - expect(sizes).toEqual([CHUNK_SIZE, 100]); - - const round = readBack(db, "/big"); - expect(round.byteLength).toBe(CHUNK_SIZE + 100); - expect(round[0]).toBe(0x41); - expect(round[CHUNK_SIZE]).toBe(0x42); - }); - }); - - it("range writes reuse unchanged chunks", async () => { - await withDB(async (db) => { - const first = new Uint8Array(CHUNK_SIZE); - first.fill(0x41); - const second = new Uint8Array(CHUNK_SIZE); - second.fill(0x42); - const third = new Uint8Array(CHUNK_SIZE); - third.fill(0x43); - const original = new Uint8Array(3 * CHUNK_SIZE); - original.set(first, 0); - original.set(second, CHUNK_SIZE); - original.set(third, 2 * CHUNK_SIZE); - await writeFile(db, "/big", original, {}, () => 100); - const beforeChunks = chunkRows(db, "/big"); - - let stagedBlobs = 0; - const run = db.run.bind(db); - db.run = (query: string, ...bindings: unknown[]) => { - if (query.startsWith("INSERT INTO vfs_blobs")) stagedBlobs += 1; - return run(query, ...bindings); - }; - - const next = new Uint8Array(original); - const changedOffset = CHUNK_SIZE + 123; - next[changedOffset] = 0x99; - writeFileRangesSync( - db, - "/big", - next, - [{ start: changedOffset, end: changedOffset + 1 }], - {}, - () => 200, - ); - - expect(stagedBlobs).toBe(1); - expect(Array.from(readBack(db, "/big"))).toEqual(Array.from(next)); - const afterChunks = chunkRows(db, "/big"); - expect(afterChunks).toHaveLength(3); - expect(afterChunks[0].hash).toEqual(beforeChunks[0].hash); - expect(afterChunks[1].hash).not.toEqual(beforeChunks[1].hash); - expect(afterChunks[2].hash).toEqual(beforeChunks[2].hash); - }); - }); - - it("dedups identical content across two paths into one blob row", async () => { - await withDB(async (db) => { - await writeFile(db, "/a.txt", "shared", {}, () => 0); - await writeFile(db, "/b.txt", "shared", {}, () => 0); - expect(countBlobs(db)).toBe(1); - }); - }); - - it("overwriting reuses the blob when content is unchanged", async () => { - await withDB(async (db) => { - await writeFile(db, "/x.txt", "same", {}, () => 0); - const before = countBlobs(db); - await writeFile(db, "/x.txt", "same", {}, () => 0); - expect(countBlobs(db)).toBe(before); - }); - }); - - it("overwriting replaces chunk rows; old content blob remains for GC", async () => { - await withDB(async (db) => { - await writeFile(db, "/x.txt", "first", {}, () => 0); - await writeFile(db, "/x.txt", "second-version", {}, () => 0); - expect(new TextDecoder().decode(readBack(db, "/x.txt"))).toBe("second-version"); - expect(countBlobs(db)).toBe(2); - }); - }); - - it("rejects ENOENT when the parent directory is missing", async () => { - await withDB(async (db) => { - await expect(writeFile(db, "/no/such/dir/file.txt", "hi", {}, () => 0)).rejects.toMatchObject( - { - code: "ENOENT", - }, - ); - }); - }); - - it("rejects EISDIR when the path resolves to a directory", async () => { - await withDB(async (db) => { - mkdir(db, "/d", {}, () => 0); - await expect(writeFile(db, "/d", "x", {}, () => 0)).rejects.toMatchObject({ - code: "EISDIR", - }); - }); - }); - - it("honors mode and bumps rev on first write", async () => { - await withDB(async (db) => { - const beforeRev = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'"); - await writeFile(db, "/run.sh", "#!/bin/sh\n", { mode: 0o755 }, () => 4242); - const node = resolveInode(db, "/run.sh"); - expect(node?.mode).toBe(0o755); - expect(node?.mtime).toBe(4242); - const afterRev = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'"); - expect(afterRev).toBe((beforeRev ?? 0) + 1); - const nodeRev = db.scalar("SELECT rev FROM vfs_nodes WHERE inode = ?", node?.inode); - expect(nodeRev).toBe(afterRev); - }); - }); - - it("updates mtime and rev on overwrite", async () => { - await withDB(async (db) => { - await writeFile(db, "/x.txt", "v1", {}, () => 100); - const v1 = resolveInode(db, "/x.txt"); - const v1Rev = db.scalar("SELECT rev FROM vfs_nodes WHERE inode = ?", v1?.inode); - - await writeFile(db, "/x.txt", "v2", {}, () => 200); - const v2 = resolveInode(db, "/x.txt"); - expect(v2?.inode).toBe(v1?.inode); - expect(v2?.mtime).toBe(200); - const v2Rev = db.scalar("SELECT rev FROM vfs_nodes WHERE inode = ?", v2?.inode); - expect((v2Rev ?? 0) > (v1Rev ?? 0)).toBe(true); - }); - }); - - it("writes into a nested directory", async () => { - await withDB(async (db) => { - mkdir(db, "/a/b", { recursive: true }, () => 0); - await writeFile(db, "/a/b/c.txt", "nested", {}, () => 0); - expect(new TextDecoder().decode(readBack(db, "/a/b/c.txt"))).toBe("nested"); - }); - }); - - it("stages blobs incrementally as the stream produces them", async () => { - await withDB(async (db) => { - // Stream 3 CHUNK_SIZE-aligned source chunks. After the first - // is pulled, the receiver should have already staged it — - // we don't want to wait for the whole stream to drain. - const filler = new Uint8Array(CHUNK_SIZE); - filler.fill(0x41); - const filler2 = new Uint8Array(CHUNK_SIZE); - filler2.fill(0x42); - const filler3 = new Uint8Array(CHUNK_SIZE); - filler3.fill(0x43); - - let pulled = 0; - let blobsAfterFirstPull: number | undefined; - const stream = new ReadableStream({ - async pull(controller) { - if (pulled === 0) { - controller.enqueue(filler); - } else if (pulled === 1) { - // Snapshot blob count after the writer has consumed - // the first source chunk but before we hand it the - // second. With streaming this is ≥ 1; with buffering - // it stays at 0 until end-of-stream. - blobsAfterFirstPull = countBlobs(db); - controller.enqueue(filler2); - } else if (pulled === 2) { - controller.enqueue(filler3); - } else { - controller.close(); - } - pulled++; - }, - }); - - await writeFile(db, "/big.bin", stream, {}, () => 0); - expect(blobsAfterFirstPull).toBeGreaterThanOrEqual(1); - expect(countBlobs(db)).toBe(3); - const back = readBack(db, "/big.bin"); - expect(back.byteLength).toBe(3 * CHUNK_SIZE); - expect(back[0]).toBe(0x41); - expect(back[CHUNK_SIZE]).toBe(0x42); - expect(back[2 * CHUNK_SIZE]).toBe(0x43); - }); - }); - - it("chunks correctly when source ReadableStream chunks don't align to CHUNK_SIZE", async () => { - await withDB(async (db) => { - // Source emits oddly-sized parts: 100 bytes, then CHUNK_SIZE, - // then 50 bytes. Total = CHUNK_SIZE + 150 → 2 chunks. - const a = new Uint8Array(100); - a.fill(0x31); - const b = new Uint8Array(CHUNK_SIZE); - b.fill(0x32); - const c = new Uint8Array(50); - c.fill(0x33); - await writeFile(db, "/oddly.bin", streamOf(a, b, c), {}, () => 0); - const back = readBack(db, "/oddly.bin"); - expect(back.byteLength).toBe(CHUNK_SIZE + 150); - expect(back[0]).toBe(0x31); - expect(back[99]).toBe(0x31); - expect(back[100]).toBe(0x32); - expect(back[CHUNK_SIZE + 99]).toBe(0x32); - expect(back[CHUNK_SIZE + 100]).toBe(0x33); - // 2 chunks (first 512KiB, then 150-byte trailing). - const node = resolveInode(db, "/oddly.bin"); - const chunkCount = db.scalar( - "SELECT COUNT(*) FROM vfs_chunks WHERE inode = ?", - node?.inode, - ); - expect(chunkCount).toBe(2); - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/writeFile.ts b/spikes/349-dofs/vendor/dofs/src/fs/writeFile.ts deleted file mode 100644 index 48f6fc53..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/writeFile.ts +++ /dev/null @@ -1,1070 +0,0 @@ -import { createHash } from "node:crypto"; -import { createWorkspaceError } from "../errors.js"; -import { canonicalizePath } from "../path.js"; -import { incrementRev } from "../rev.js"; -import { ROOT_INODE } from "../schema/index.js"; -import type { Database } from "../storage.js"; -import { stageBlob } from "../sync/blobs.js"; -import { buildManifest } from "../sync/manifests.js"; -import { getBlobBytes } from "./blobCache.js"; -import { assertNotReadOnly } from "./mount-guard.js"; -import { invalidateResolveExact } from "./resolveCache.js"; -import { - allocatePendingInode, - deleteWriteBuffer, - ensureCapacity as ensureBufferCapacity, - getPendingWriteBufferByPath, - getWriteBuffer, - promotePendingToInode, - setWriteBuffer, - type WriteBufferEntry, -} from "./writeBuffer.js"; - -// Fixed chunk size. Exported so tests can size inputs precisely -// without hard-coding the magic number twice. -export const CHUNK_SIZE = 512 * 1024; - -export type WriteFileContent = string | Uint8Array | ReadableStream; - -export interface WriteFileOptions { - mode?: number; - /** Fail with EEXIST when the target already exists. */ - exclusive?: boolean; -} - -export interface WriteFileRange { - start: number; - end: number; -} - -// Resolve directory-only paths (the parent of the target file). The -// final segment is handled by the caller. Returns the parent inode or -// throws ENOENT/ENOTDIR. -function resolveParent(db: Database, parts: string[], canonical: string): number { - let parentInode = ROOT_INODE; - for (let i = 0; i < parts.length - 1; i++) { - const name = parts[i]; - const child = db.one<{ child_inode: number }>( - "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", - parentInode, - name, - ); - if (child === undefined) { - throw createWorkspaceError("ENOENT", `parent directory missing: ${canonical}`, canonical); - } - const next = db.one<{ inode: number; type: "file" | "dir" }>( - "SELECT inode, type FROM vfs_nodes WHERE inode = ?", - child.child_inode, - ); - if (next === undefined) { - throw createWorkspaceError("ENOENT", `dangling dirent: ${canonical}`, canonical); - } - if (next.type !== "dir") { - throw createWorkspaceError( - "ENOTDIR", - `parent path segment is not a directory: ${canonical}`, - canonical, - ); - } - parentInode = next.inode; - } - return parentInode; -} - -async function materialize(content: string | Uint8Array): Promise { - if (typeof content === "string") { - return new TextEncoder().encode(content); - } - return content; -} - -// sha256 with a synchronous code path so writeFile can be called both -// from async drivers (the FS API) and from sync drivers (the -// VirtualProvider). node:crypto is available natively on Node and -// polyfilled by workerd. -function sha256(bytes: Uint8Array): Uint8Array { - const hash = createHash("sha256"); - hash.update(bytes); - return new Uint8Array(hash.digest()); -} - -interface PreparedChunk { - hash: Uint8Array; - bytes: Uint8Array; - size: number; -} - -interface ChunkRef { - hash: Uint8Array; - size: number; -} - -export function chunksOf(bytes: Uint8Array): PreparedChunk[] { - const chunks: PreparedChunk[] = []; - for (let offset = 0; offset < bytes.byteLength; offset += CHUNK_SIZE) { - const end = Math.min(offset + CHUNK_SIZE, bytes.byteLength); - // subarray (not slice) avoids an extra copy; sha256() takes its own - // copy when needed. - const slice = bytes.subarray(offset, end); - const hash = sha256(slice); - chunks.push({ hash, bytes: slice, size: slice.byteLength }); - } - return chunks; -} - -export async function writeFile( - db: Database, - path: string, - content: WriteFileContent, - options: WriteFileOptions, - now: () => number, -): Promise { - if (content instanceof ReadableStream) { - await writeFileStreaming(db, path, content, options, now); - return; - } - const bytes = await materialize(content); - writeFileSync(db, path, bytes, options, now); -} - -// Streaming write path. Reads the source one source-chunk at a time, -// re-windows into fixed CHUNK_SIZE pieces, hashes each window, and -// stages it into vfs_blobs / vfs_blob_bytes as it goes. The final -// inode / dirent / vfs_chunks / manifest writes happen in a single -// short transaction once the source is drained, against a list of -// {hash, size} entries that's O(file_size / CHUNK_SIZE) bytes — not -// O(file_size). -// -// Failure mid-stream leaves blob rows behind; gc() reaps orphans on -// its next pass since no node references them. -async function writeFileStreaming( - db: Database, - path: string, - source: ReadableStream, - options: WriteFileOptions, - now: () => number, -): Promise { - const { parts, path: canonical } = canonicalizePath(path); - if (parts.length === 0) { - throw createWorkspaceError("EISDIR", "cannot write to the root directory", canonical); - } - // Reject before we stage any blob bytes so known failures do not grow - // orphan blob rows that gc() then has to reap. - assertNotReadOnly(db, canonical); - if (options.exclusive) { - const parentInode = resolveParent(db, parts, canonical); - const existing = db.one( - "SELECT 1 FROM vfs_dirents WHERE parent_inode = ? AND name = ?", - parentInode, - parts[parts.length - 1], - ); - if (existing !== undefined) { - throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); - } - } - const mode = (options.mode ?? 0o644) & 0o7777; - const mtime = now(); - - const chunkRefs: Array<{ hash: Uint8Array; size: number }> = []; - // Carry-over buffer: bytes left over from the previous source chunk - // that didn't fill a CHUNK_SIZE window. - let carry: Uint8Array | undefined; - - const flush = (chunk: Uint8Array): void => { - const hash = sha256(chunk); - stageBlob(db, hash, chunk, mtime); - chunkRefs.push({ hash, size: chunk.byteLength }); - }; - - const reader = source.getReader(); - try { - while (true) { - const { value, done } = await reader.read(); - if (done) break; - if (value === undefined || value.byteLength === 0) continue; - let input = value; - if (carry !== undefined) { - // Splice carry-over onto the front of this source chunk so - // we can re-window cleanly. - const merged = new Uint8Array(carry.byteLength + input.byteLength); - merged.set(carry, 0); - merged.set(input, carry.byteLength); - input = merged; - carry = undefined; - } - let offset = 0; - while (input.byteLength - offset >= CHUNK_SIZE) { - // Copy the window so the staged blob doesn't alias a - // larger backing buffer. - const window = input.slice(offset, offset + CHUNK_SIZE); - flush(window); - offset += CHUNK_SIZE; - } - if (offset < input.byteLength) { - carry = input.slice(offset); - } - } - } finally { - reader.releaseLock(); - } - if (carry !== undefined && carry.byteLength > 0) { - flush(carry); - } - - // Wire up the inode against the staged blobs in one short - // transaction. From this point on the SQL is the same shape as the - // synchronous path — only the chunk-bytes step is skipped because - // stageBlob already landed them above. - db.transactionSync(() => { - const parentInode = resolveParent(db, parts, canonical); - const leafName = parts[parts.length - 1]; - const existing = db.one<{ child_inode: number }>( - "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", - parentInode, - leafName, - ); - let inode: number; - if (existing !== undefined) { - if (options.exclusive) { - throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); - } - const node = db.one<{ type: "file" | "dir" }>( - "SELECT type FROM vfs_nodes WHERE inode = ?", - existing.child_inode, - ); - if (node?.type === "dir") { - throw createWorkspaceError("EISDIR", `path is a directory: ${canonical}`, canonical); - } - inode = existing.child_inode; - db.run("DELETE FROM vfs_chunks WHERE inode = ?", inode); - } else { - inode = insertFileNode(db, mode, mtime); - insertFileDirent(db, parentInode, leafName, inode, canonical); - } - for (let idx = 0; idx < chunkRefs.length; idx++) { - const ref = chunkRefs[idx]; - db.run( - "INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", - inode, - idx, - ref.hash, - ref.size, - ); - } - const manifestHash = buildManifest(db, chunkRefs, mtime); - const rev = incrementRev(db); - let totalSize = 0; - for (const ref of chunkRefs) totalSize += ref.size; - db.run( - "UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ?, size = ?, manifest_hash = ? WHERE inode = ?", - mode, - mtime, - rev, - totalSize, - manifestHash, - inode, - ); - }); -} - -// Allocate a fresh file inode row with the supplied mode and mtime, -// using SQLite's RETURNING so the new rowid comes back in the same -// statement instead of through a follow-up SELECT last_insert_rowid(). -// Link a freshly created file inode into its parent directory and drop -// any cached negative resolution for the new path. The single choke -// point for every new-file dirent, so the resolve cache stays correct -// on create without touching the overwrite path (which reuses the -// existing inode and dirent, so its resolution is unchanged). A new -// file is a leaf with no descendants, so exact invalidation suffices. -function insertFileDirent( - db: Database, - parentInode: number, - leafName: string, - childInode: number, - canonicalPath: string, -): void { - db.run( - "INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", - parentInode, - leafName, - childInode, - ); - invalidateResolveExact(db, canonicalPath); -} - -function insertFileNode(db: Database, mode: number, mtime: number): number { - const row = db.one<{ inode: number }>( - "INSERT INTO vfs_nodes (type, mode, mtime, rev) VALUES ('file', ?, ?, 0) RETURNING inode", - mode, - mtime, - ); - if (row === undefined) { - throw createWorkspaceError("EIO", "failed to allocate inode"); - } - return row.inode; -} - -function upsertChunkBlob(db: Database, chunk: PreparedChunk, lastSeen: number): void { - db.run( - "INSERT INTO vfs_blobs (hash, size, last_seen) VALUES (?, ?, ?) ON CONFLICT(hash) DO UPDATE SET last_seen = excluded.last_seen", - chunk.hash, - chunk.size, - lastSeen, - ); - db.run( - "INSERT INTO vfs_blob_bytes (hash, bytes) VALUES (?, ?) ON CONFLICT(hash) DO NOTHING", - chunk.hash, - chunk.bytes, - ); -} - -function replaceChunkRows( - db: Database, - inode: number, - chunks: ChunkRef[], - manifestTime: number, -): Uint8Array { - db.run("DELETE FROM vfs_chunks WHERE inode = ?", inode); - for (let idx = 0; idx < chunks.length; idx++) { - const chunk = chunks[idx]; - db.run( - "INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", - inode, - idx, - chunk.hash, - chunk.size, - ); - } - return buildManifest(db, chunks, manifestTime); -} - -function rangesOverlap(start: number, end: number, ranges: WriteFileRange[]): boolean { - for (const range of ranges) { - if (range.start < end && start < range.end) return true; - } - return false; -} - -function normalizeRanges(ranges: WriteFileRange[], size: number): WriteFileRange[] { - const normalized = ranges - .map((range) => ({ - start: Math.max(0, Math.min(size, Math.floor(range.start))), - end: Math.max(0, Math.min(size, Math.ceil(range.end))), - })) - .filter((range) => range.start < range.end) - .sort((a, b) => a.start - b.start); - - const merged: WriteFileRange[] = []; - for (const range of normalized) { - const previous = merged.at(-1); - if (previous === undefined || previous.end < range.start) { - merged.push({ ...range }); - } else { - previous.end = Math.max(previous.end, range.end); - } - } - return merged; -} - -function existingChunkRefs(db: Database, inode: number): ChunkRef[] { - return db.all("SELECT hash, size FROM vfs_chunks WHERE inode = ? ORDER BY idx", inode); -} - -function fileSizeForInode(db: Database, inode: number): number { - return db.scalar("SELECT size FROM vfs_nodes WHERE inode = ?", inode) ?? 0; -} - -function readChunkBytes(db: Database, inode: number, idx: number): Uint8Array { - const chunk = db.one<{ hash: Uint8Array }>( - "SELECT hash FROM vfs_chunks WHERE inode = ? AND idx = ?", - inode, - idx, - ); - if (chunk === undefined) return new Uint8Array(); - const bytes = getBlobBytes(db, chunk.hash); - if (bytes === undefined) { - throw createWorkspaceError("EIO", "missing blob bytes"); - } - return bytes; -} - -function resolveFileInode(db: Database, path: string): { inode: number; mode: number } { - const { path: canonical } = canonicalizePath(path); - const node = db.one<{ inode: number; type: "file" | "dir"; mode: number }>( - `SELECT n.inode AS inode, n.type AS type, n.mode AS mode - FROM vfs_nodes n - WHERE n.inode = ( - SELECT child_inode - FROM vfs_dirents - WHERE parent_inode = ? AND name = ? - )`, - ...parentAndNameForResolvedPath(db, path), - ); - if (node === undefined) { - throw createWorkspaceError("ENOENT", `no such file: ${canonical}`, canonical); - } - if (node.type !== "file") { - throw createWorkspaceError("EISDIR", `path is a directory: ${canonical}`, canonical); - } - return { inode: node.inode, mode: node.mode }; -} - -function parentAndNameForResolvedPath(db: Database, path: string): [number, string] { - const { parts, path: canonical } = canonicalizePath(path); - if (parts.length === 0) { - throw createWorkspaceError("EISDIR", "cannot write to the root directory", canonical); - } - return [resolveParent(db, parts, canonical), parts[parts.length - 1]]; -} - -// Update an inode's chunk-backed representation in place. Iterates over -// the full chunk grid but only touches `vfs_chunks` rows whose contents -// or size actually changed, so untouched chunk rows keep their -// rowids and the surrounding rows do not churn. The manifest is -// invalidated rather than recomputed; sync rebuilds it lazily. -function applyChunkedInodeUpdate( - db: Database, - inode: number, - size: number, - mode: number, - mtime: number, - isTouched: (idx: number, start: number, end: number) => boolean, - buildChunkBytes: (idx: number, start: number, end: number, existing: Uint8Array) => Uint8Array, -): void { - const oldChunks = existingChunkRefs(db, inode); - const chunkCount = Math.ceil(size / CHUNK_SIZE); - const oldChunkCount = oldChunks.length; - - for (let idx = 0; idx < chunkCount; idx++) { - const start = idx * CHUNK_SIZE; - const end = Math.min(start + CHUNK_SIZE, size); - const intendedSize = end - start; - const old = oldChunks[idx]; - const touched = isTouched(idx, start, end); - // Stable chunk: existed before with the same logical size and the - // caller did not flag it as touched. Skip without issuing SQL so - // its rowid stays put. - if (old !== undefined && old.size === intendedSize && !touched) continue; - - const existingBytes = old !== undefined ? readChunkBytes(db, inode, idx) : new Uint8Array(); - const chunkBytes = buildChunkBytes(idx, start, end, existingBytes); - if (chunkBytes.byteLength !== intendedSize) { - throw createWorkspaceError("EIO", "chunk builder returned wrong size"); - } - const chunk = { hash: sha256(chunkBytes), bytes: chunkBytes, size: chunkBytes.byteLength }; - upsertChunkBlob(db, chunk, mtime); - db.run( - "INSERT OR REPLACE INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", - inode, - idx, - chunk.hash, - chunk.size, - ); - } - - // Drop any old chunks past the new end of file (shrink case). - if (oldChunkCount > chunkCount) { - db.run("DELETE FROM vfs_chunks WHERE inode = ? AND idx >= ?", inode, chunkCount); - } - - const rev = incrementRev(db); - db.run( - "UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ?, size = ?, manifest_hash = NULL WHERE inode = ?", - mode, - mtime, - rev, - size, - inode, - ); -} - -export function createFileSync( - db: Database, - path: string, - options: WriteFileOptions, - now: () => number, -): void { - const { path: canonical } = canonicalizePath(path); - assertNotReadOnly(db, canonical); - const [parentInode, leafName] = parentAndNameForResolvedPath(db, path); - const mode = (options.mode ?? 0o644) & 0o7777; - const mtime = now(); - - db.transactionSync(() => { - const existing = db.one<{ child_inode: number }>( - "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", - parentInode, - leafName, - ); - if (existing !== undefined) { - throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); - } - const rev = incrementRev(db); - // INSERT with RETURNING folds the last_insert_rowid lookup into - // the same statement, and computing rev up front lets us write - // the node row with its final stamp in one shot. - const row = db.one<{ inode: number }>( - "INSERT INTO vfs_nodes (type, mode, mtime, rev, manifest_hash) VALUES ('file', ?, ?, ?, NULL) RETURNING inode", - mode, - mtime, - rev, - ); - if (row === undefined) throw createWorkspaceError("EIO", "failed to allocate inode"); - insertFileDirent(db, parentInode, leafName, row.inode, canonical); - }); -} - -// Open a write buffer for an existing file. Subsequent writes, -// truncates, and reads against the same Database operate on the -// buffer instead of the SQLite chunk/blob store. Release commits -// the bytes back to chunks. -export function openWriteBufferSync(db: Database, path: string): void { - const { path: canonical } = canonicalizePath(path); - const pending = getPendingWriteBufferByPath(db, canonical); - if (pending !== undefined) { - pending.openCount += 1; - return; - } - const { inode, mode } = resolveFileInode(db, path); - const existing = getWriteBuffer(db, inode); - if (existing !== undefined) { - existing.openCount += 1; - return; - } - setWriteBuffer(db, inode, { - buf: new Uint8Array(0), - size: 0, - dirty: false, - openCount: 1, - mode, - }); -} - -// Create a new file lazily: stash a pending-create write buffer -// keyed by path, without touching SQL until release. createFileSync -// + openWriteBufferSync + writes + releaseWriteBufferSync would -// otherwise spend two transactions per file (one INSERT round and -// one chunk-commit round); this collapses them into a single -// INSERT-and-chunks transaction at release time. -// -// Throws EEXIST if a path already resolves to a live node or to -// another pending buffer. -export function openWriteBufferForCreateSync( - db: Database, - path: string, - options: WriteFileOptions, - now: () => number, -): void { - const { path: canonical } = canonicalizePath(path); - assertNotReadOnly(db, canonical); - if (getPendingWriteBufferByPath(db, canonical) !== undefined) { - throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); - } - const [parentInode, leafName] = parentAndNameForResolvedPath(db, path); - const existing = db.one<{ child_inode: number }>( - "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", - parentInode, - leafName, - ); - if (existing !== undefined) { - throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); - } - const mode = (options.mode ?? 0o644) & 0o7777; - const mtime = now(); - const pendingInode = allocatePendingInode(db); - setWriteBuffer(db, pendingInode, { - buf: new Uint8Array(0), - size: 0, - dirty: true, - openCount: 1, - mode, - pending: { parentInode, leafName, canonicalPath: canonical, pendingInode, mtime }, - }); -} - -// Release one open of an inode's write buffer. When the open count -// reaches zero, commit the buffered bytes to chunk rows and drop -// the entry. The committed mode is the buffer's mode at release -// time so an intermediate chmod survives. Pending-create entries -// emit their INSERT + dirent + chunks in the same transaction. -export function releaseWriteBufferSync(db: Database, path: string, now: () => number): void { - const { path: canonical } = canonicalizePath(path); - const pending = getPendingWriteBufferByPath(db, canonical); - if (pending !== undefined) { - releasePendingBuffer(db, pending, now); - return; - } - const node = resolveFileInode(db, path); - const entry = getWriteBuffer(db, node.inode); - if (entry === undefined) return; - entry.openCount -= 1; - if (entry.openCount > 0) return; - - if (!entry.dirty) { - deleteWriteBuffer(db, node.inode); - return; - } - - const mtime = now(); - const mode = entry.mode & 0o7777; - const buffered = entry.buf.subarray(0, entry.size); - - db.transactionSync(() => { - if (entry.size === 0) { - // An empty file owns no chunk rows; clear any old ones the - // buffer would otherwise have replaced and bump metadata. - db.run("DELETE FROM vfs_chunks WHERE inode = ?", node.inode); - const rev = incrementRev(db); - db.run( - "UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ?, size = 0, manifest_hash = NULL WHERE inode = ?", - mode, - mtime, - rev, - node.inode, - ); - return; - } - applyChunkedInodeUpdate( - db, - node.inode, - entry.size, - mode, - mtime, - (_idx, start, end) => start < entry.size && end > 0, - (_idx, start, end) => buffered.subarray(start, Math.min(end, entry.size)), - ); - }); - - deleteWriteBuffer(db, node.inode); -} - -// Commit a pending-create buffer to SQLite. Returns the real inode -// allocated by the INSERT, or throws. Promotes the cache entry's key -// from the synthetic pending id to the real inode so subsequent -// reads/writes through the inode-keyed cache still see the same -// buffer. Caller owns the lifecycle of the now-promoted entry. -function commitPendingBuffer(db: Database, entry: WriteBufferEntry, now: () => number): number { - if (entry.pending === undefined) { - throw createWorkspaceError("EIO", "commitPendingBuffer called on non-pending entry"); - } - const { parentInode, leafName, canonicalPath, pendingInode } = entry.pending; - const mtime = now(); - const mode = entry.mode & 0o7777; - const buffered = entry.buf.subarray(0, entry.size); - - let realInode = 0; - try { - db.transactionSync(() => { - // Re-check at commit time: a non-buffered writeFile or another - // out-of-band path could have landed between open and release. - const collision = db.one<{ child_inode: number }>( - "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", - parentInode, - leafName, - ); - if (collision !== undefined) { - throw createWorkspaceError( - "EEXIST", - `path exists at commit time: ${canonicalPath}`, - canonicalPath, - ); - } - const rev = incrementRev(db); - const row = db.one<{ inode: number }>( - "INSERT INTO vfs_nodes (type, mode, mtime, rev, size, manifest_hash) VALUES ('file', ?, ?, ?, ?, NULL) RETURNING inode", - mode, - mtime, - rev, - entry.size, - ); - if (row === undefined) { - throw createWorkspaceError("EIO", "failed to allocate inode"); - } - insertFileDirent(db, parentInode, leafName, row.inode, canonicalPath); - if (entry.size > 0) { - const inode = row.inode; - const chunkCount = Math.ceil(entry.size / CHUNK_SIZE); - for (let idx = 0; idx < chunkCount; idx++) { - const start = idx * CHUNK_SIZE; - const end = Math.min(start + CHUNK_SIZE, entry.size); - const chunkBytes = buffered.subarray(start, end); - const chunk = { - hash: sha256(chunkBytes), - bytes: chunkBytes, - size: chunkBytes.byteLength, - }; - upsertChunkBlob(db, chunk, mtime); - db.run( - "INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", - inode, - idx, - chunk.hash, - chunk.size, - ); - } - } - realInode = row.inode; - }); - } catch (error) { - // Transaction rolled back; drop the buffer so the next caller - // starts clean. - deleteWriteBuffer(db, pendingInode); - throw error; - } - promotePendingToInode(db, pendingInode, realInode); - return realInode; -} - -/** - * @internal - * Bridges a pending-create write buffer into the SQL world ahead of a - * dirent-mutating provider operation (link, rename, unlink). Leaves - * the open count untouched so a still-open handle keeps writing into - * the now-promoted buffer. Returns true when a pending buffer was - * committed. External callers should never invoke this directly. - */ -export function flushPendingByPath(db: Database, path: string, now: () => number): boolean { - const { path: canonical } = canonicalizePath(path); - const entry = getPendingWriteBufferByPath(db, canonical); - if (entry === undefined || entry.pending === undefined) return false; - commitPendingBuffer(db, entry, now); - return true; -} - -function releasePendingBuffer(db: Database, entry: WriteBufferEntry, now: () => number): void { - if (entry.pending === undefined) return; - entry.openCount -= 1; - if (entry.openCount > 0) return; - - const inode = commitPendingBuffer(db, entry, now); - // File is closed; drop the now-promoted entry. A subsequent open - // hits the SQL path and gets a fresh buffer if needed. - deleteWriteBuffer(db, inode); -} - -// Hydrate a freshly-opened buffer with the inode's current bytes -// the first time we mutate it. Avoids paying the read cost when the -// caller opens a file just to truncate or overwrite it. -function hydrateBufferIfNeeded(db: Database, inode: number, entry: WriteBufferEntry): void { - if (entry.dirty) return; - const existingSize = fileSizeForInode(db, inode); - if (existingSize === 0) { - entry.dirty = true; - return; - } - ensureBufferCapacity(entry, existingSize); - let copied = 0; - for (let idx = 0; copied < existingSize; idx++) { - const chunk = readChunkBytes(db, inode, idx); - if (chunk.byteLength === 0) break; - entry.buf.set(chunk, copied); - copied += chunk.byteLength; - } - entry.size = existingSize; - entry.dirty = true; -} - -export function writeRangeSync( - db: Database, - path: string, - bytes: Uint8Array, - offset: number, - options: WriteFileOptions, - now: () => number, -): number { - const { path: canonical } = canonicalizePath(path); - assertNotReadOnly(db, canonical); - if (!Number.isInteger(offset) || offset < 0) { - throw createWorkspaceError("EINVAL", `invalid write offset: ${offset}`, canonical); - } - if (bytes.byteLength === 0) return 0; - const mtime = now(); - - // Pending-create files don't have an inode yet; route the write - // straight into the path-keyed buffer. - const pending = getPendingWriteBufferByPath(db, canonical); - if (pending !== undefined) { - const writeEnd = offset + bytes.byteLength; - ensureBufferCapacity(pending, writeEnd); - if (offset > pending.size) { - pending.buf.fill(0, pending.size, offset); - } - pending.buf.set(bytes, offset); - if (writeEnd > pending.size) pending.size = writeEnd; - pending.mode = (options.mode ?? pending.mode) & 0o7777; - pending.dirty = true; - return bytes.byteLength; - } - - const { inode, mode: existingMode } = resolveFileInode(db, path); - const mode = (options.mode ?? existingMode) & 0o7777; - const buffered = getWriteBuffer(db, inode); - - // Buffered path: mutate the in-memory bytes and defer storage - // writes until release. Reads through the same Database see the - // buffer's current bytes via readRangeSync's buffer check. - if (buffered !== undefined) { - hydrateBufferIfNeeded(db, inode, buffered); - const writeEnd = offset + bytes.byteLength; - ensureBufferCapacity(buffered, writeEnd); - if (offset > buffered.size) { - buffered.buf.fill(0, buffered.size, offset); - } - buffered.buf.set(bytes, offset); - if (writeEnd > buffered.size) buffered.size = writeEnd; - buffered.mode = mode; - buffered.dirty = true; - return bytes.byteLength; - } - - db.transactionSync(() => { - const oldSize = fileSizeForInode(db, inode); - const writeEnd = offset + bytes.byteLength; - const nextSize = Math.max(oldSize, writeEnd); - - applyChunkedInodeUpdate( - db, - inode, - nextSize, - mode, - mtime, - (_idx, start, end) => offset < end && start < writeEnd, - (_idx, start, end, existing) => { - const chunkBytes = new Uint8Array(end - start); - chunkBytes.set(existing.subarray(0, Math.min(existing.byteLength, chunkBytes.byteLength))); - if (offset < end && start < writeEnd) { - const copyStart = Math.max(start, offset); - const copyEnd = Math.min(end, writeEnd); - chunkBytes.set(bytes.subarray(copyStart - offset, copyEnd - offset), copyStart - start); - } - return chunkBytes; - }, - ); - }); - - return bytes.byteLength; -} - -export function truncateFileSync( - db: Database, - path: string, - size: number, - now: () => number, -): void { - const { path: canonical } = canonicalizePath(path); - assertNotReadOnly(db, canonical); - if (!Number.isInteger(size) || size < 0) { - throw createWorkspaceError("EINVAL", `invalid truncate size: ${size}`, canonical); - } - const mtime = now(); - - // Pending-create files truncate in-place on the path-keyed buffer. - const pending = getPendingWriteBufferByPath(db, canonical); - if (pending !== undefined) { - if (size > pending.size) { - ensureBufferCapacity(pending, size); - pending.buf.fill(0, pending.size, size); - } - pending.size = size; - pending.dirty = true; - return; - } - - const { inode, mode } = resolveFileInode(db, path); - const buffered = getWriteBuffer(db, inode); - - if (buffered !== undefined) { - hydrateBufferIfNeeded(db, inode, buffered); - if (size > buffered.size) { - ensureBufferCapacity(buffered, size); - buffered.buf.fill(0, buffered.size, size); - } - buffered.size = size; - buffered.dirty = true; - return; - } - - db.transactionSync(() => { - const oldSize = fileSizeForInode(db, inode); - if (oldSize === size) return; - - if (size === 0) { - db.run("DELETE FROM vfs_chunks WHERE inode = ?", inode); - const rev = incrementRev(db); - db.run( - "UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ?, size = 0, manifest_hash = NULL WHERE inode = ?", - mode, - mtime, - rev, - inode, - ); - return; - } - - applyChunkedInodeUpdate( - db, - inode, - size, - mode, - mtime, - () => false, - (_idx, start, end, existing) => { - const chunkBytes = new Uint8Array(end - start); - chunkBytes.set(existing.subarray(0, Math.min(existing.byteLength, chunkBytes.byteLength))); - return chunkBytes; - }, - ); - }); -} - -// Synchronous entry point used by the VirtualProvider. Identical SQL -// to the async path; differs only in that the bytes have already been -// materialized. -export function writeFileSync( - db: Database, - path: string, - bytes: Uint8Array, - options: WriteFileOptions, - now: () => number, -): void { - const { parts, path: canonical } = canonicalizePath(path); - if (parts.length === 0) { - throw createWorkspaceError("EISDIR", "cannot write to the root directory", canonical); - } - assertNotReadOnly(db, canonical); - const mode = (options.mode ?? 0o644) & 0o7777; - const mtime = now(); - - db.transactionSync(() => { - const parentInode = resolveParent(db, parts, canonical); - const leafName = parts[parts.length - 1]; - const existing = db.one<{ child_inode: number }>( - "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", - parentInode, - leafName, - ); - - let inode: number; - if (existing !== undefined) { - if (options.exclusive) { - throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); - } - const node = db.one<{ type: "file" | "dir" }>( - "SELECT type FROM vfs_nodes WHERE inode = ?", - existing.child_inode, - ); - if (node?.type === "dir") { - throw createWorkspaceError("EISDIR", `path is a directory: ${canonical}`, canonical); - } - inode = existing.child_inode; - // Replace the existing representation. Orphaned blobs (if any) - // are cleaned up by a later gc() pass. - db.run("DELETE FROM vfs_chunks WHERE inode = ?", inode); - } else { - inode = insertFileNode(db, mode, mtime); - insertFileDirent(db, parentInode, leafName, inode, canonical); - } - - const rev = incrementRev(db); - const chunks = chunksOf(bytes); - // Upsert blobs and write the new chunk list. - for (let idx = 0; idx < chunks.length; idx++) { - const chunk = chunks[idx]; - upsertChunkBlob(db, chunk, mtime); - db.run( - "INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", - inode, - idx, - chunk.hash, - chunk.size, - ); - } - - const manifestHash = buildManifest(db, chunks, mtime); - db.run( - "UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ?, size = ?, manifest_hash = ? WHERE inode = ?", - mode, - mtime, - rev, - bytes.byteLength, - manifestHash, - inode, - ); - }); -} - -export function writeFileRangesSync( - db: Database, - path: string, - bytes: Uint8Array, - dirtyRanges: WriteFileRange[], - options: WriteFileOptions, - now: () => number, -): void { - const { parts, path: canonical } = canonicalizePath(path); - if (parts.length === 0) { - throw createWorkspaceError("EISDIR", "cannot write to the root directory", canonical); - } - assertNotReadOnly(db, canonical); - const mode = (options.mode ?? 0o644) & 0o7777; - const ranges = normalizeRanges(dirtyRanges, bytes.byteLength); - const mtime = now(); - db.transactionSync(() => { - const parentInode = resolveParent(db, parts, canonical); - const leafName = parts[parts.length - 1]; - const existing = db.one<{ child_inode: number }>( - "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", - parentInode, - leafName, - ); - - let inode: number; - let oldChunks: ChunkRef[] = []; - if (existing !== undefined) { - const node = db.one<{ type: "file" | "dir" }>( - "SELECT type FROM vfs_nodes WHERE inode = ?", - existing.child_inode, - ); - if (node?.type === "dir") { - throw createWorkspaceError("EISDIR", `path is a directory: ${canonical}`, canonical); - } - inode = existing.child_inode; - oldChunks = existingChunkRefs(db, inode); - } else { - inode = insertFileNode(db, mode, mtime); - insertFileDirent(db, parentInode, leafName, inode, canonical); - } - - const rev = incrementRev(db); - const nextChunks: ChunkRef[] = []; - const chunkCount = Math.ceil(bytes.byteLength / CHUNK_SIZE); - for (let idx = 0; idx < chunkCount; idx++) { - const start = idx * CHUNK_SIZE; - const end = Math.min(start + CHUNK_SIZE, bytes.byteLength); - const size = end - start; - const oldChunk = oldChunks[idx]; - if (oldChunk !== undefined && oldChunk.size === size && !rangesOverlap(start, end, ranges)) { - nextChunks.push(oldChunk); - continue; - } - const chunk = { - hash: sha256(bytes.subarray(start, end)), - bytes: bytes.subarray(start, end), - size, - }; - upsertChunkBlob(db, chunk, mtime); - nextChunks.push({ hash: chunk.hash, size: chunk.size }); - } - - const manifestHash = replaceChunkRows(db, inode, nextChunks, mtime); - db.run( - "UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ?, size = ?, manifest_hash = ? WHERE inode = ?", - mode, - mtime, - rev, - bytes.byteLength, - manifestHash, - inode, - ); - }); -} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/writeRange.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/writeRange.test.ts deleted file mode 100644 index 46d08548..00000000 --- a/spikes/349-dofs/vendor/dofs/src/fs/writeRange.test.ts +++ /dev/null @@ -1,193 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import type { Database } from "../storage.js"; -import { link } from "./link.js"; -import { readFile } from "./readFile.js"; -import { resolveInode } from "./resolve.js"; -import { withDB } from "./with-db.js"; -import { - CHUNK_SIZE, - createFileSync, - truncateFileSync, - writeFileSync, - writeRangeSync, -} from "./writeFile.js"; - -function bytesOf(text: string): Uint8Array { - return new TextEncoder().encode(text); -} - -async function readBytes(db: Database, path: string): Promise { - const stream = await readFile(db, path); - const reader = stream.getReader(); - const chunks: Uint8Array[] = []; - let total = 0; - while (true) { - const { value, done } = await reader.read(); - if (done) break; - if (value === undefined) continue; - chunks.push(value); - total += value.byteLength; - } - const out = new Uint8Array(total); - let offset = 0; - for (const chunk of chunks) { - out.set(chunk, offset); - offset += chunk.byteLength; - } - return out; -} - -function manifestHash(db: Database, path: string): Uint8Array | null { - const node = resolveInode(db, path); - if (node === null) throw new Error(`missing node: ${path}`); - return ( - db.one<{ manifest_hash: Uint8Array | null }>( - "SELECT manifest_hash FROM vfs_nodes WHERE inode = ?", - node.inode, - )?.manifest_hash ?? null - ); -} - -function chunkRows( - db: Database, - path: string, -): Array<{ idx: number; hash: Uint8Array; size: number }> { - const node = resolveInode(db, path); - if (node === null) throw new Error(`missing node: ${path}`); - return db.all<{ idx: number; hash: Uint8Array; size: number }>( - "SELECT idx, hash, size FROM vfs_chunks WHERE inode = ? ORDER BY idx", - node.inode, - ); -} - -describe("direct range writes", () => { - it("creates an empty file with no chunk rows", async () => { - await withDB(async (db) => { - createFileSync(db, "/empty.txt", { mode: 0o600 }, () => 1000); - - const node = resolveInode(db, "/empty.txt"); - expect(node?.type).toBe("file"); - expect(node?.mode).toBe(0o600); - expect(chunkRows(db, "/empty.txt")).toEqual([]); - }); - }); - - it("writes small ranges and stores them as a single chunk", async () => { - await withDB(async (db) => { - createFileSync(db, "/small.txt", {}, () => 1000); - - expect(writeRangeSync(db, "/small.txt", bytesOf("hello"), 0, {}, () => 1001)).toBe(5); - expect(writeRangeSync(db, "/small.txt", bytesOf("y"), 4, {}, () => 1002)).toBe(1); - - expect(new TextDecoder().decode(await readBytes(db, "/small.txt"))).toBe("helly"); - expect(chunkRows(db, "/small.txt")).toHaveLength(1); - }); - }); - - it("zero-fills sparse writes", async () => { - await withDB(async (db) => { - createFileSync(db, "/sparse.txt", {}, () => 1000); - - writeRangeSync(db, "/sparse.txt", bytesOf("x"), 3, {}, () => 1001); - - expect(Array.from(await readBytes(db, "/sparse.txt"))).toEqual([0, 0, 0, 120]); - }); - }); - - it("updates only affected chunk hashes for chunk-backed files", async () => { - await withDB(async (db) => { - const original = new Uint8Array(CHUNK_SIZE * 3); - original.fill(1, 0, CHUNK_SIZE); - original.fill(2, CHUNK_SIZE, CHUNK_SIZE * 2); - original.fill(3, CHUNK_SIZE * 2, CHUNK_SIZE * 3); - writeFileSync(db, "/large.bin", original, {}, () => 1000); - const before = chunkRows(db, "/large.bin"); - - writeRangeSync(db, "/large.bin", new Uint8Array([9, 9, 9]), CHUNK_SIZE + 10, {}, () => 1001); - const after = chunkRows(db, "/large.bin"); - - expect(after).toHaveLength(3); - expect(Buffer.from(after[0].hash).equals(Buffer.from(before[0].hash))).toBe(true); - expect(Buffer.from(after[1].hash).equals(Buffer.from(before[1].hash))).toBe(false); - expect(Buffer.from(after[2].hash).equals(Buffer.from(before[2].hash))).toBe(true); - const bytes = await readBytes(db, "/large.bin"); - expect(bytes[CHUNK_SIZE + 9]).toBe(2); - expect(Array.from(bytes.subarray(CHUNK_SIZE + 10, CHUNK_SIZE + 13))).toEqual([9, 9, 9]); - expect(bytes[CHUNK_SIZE + 13]).toBe(2); - }); - }); - - it("writes through hardlinks by shared inode", async () => { - await withDB(async (db) => { - createFileSync(db, "/a.txt", {}, () => 1000); - link(db, "/a.txt", "/b.txt"); - - writeRangeSync(db, "/b.txt", bytesOf("shared"), 0, {}, () => 1001); - - expect(new TextDecoder().decode(await readBytes(db, "/a.txt"))).toBe("shared"); - expect(resolveInode(db, "/a.txt")?.inode).toBe(resolveInode(db, "/b.txt")?.inode); - }); - }); - - it("skips rewriting untouched chunks on a small range write", async () => { - await withDB(async (db) => { - const original = new Uint8Array(CHUNK_SIZE * 3); - original.fill(1, 0, CHUNK_SIZE); - original.fill(2, CHUNK_SIZE, CHUNK_SIZE * 2); - original.fill(3, CHUNK_SIZE * 2, CHUNK_SIZE * 3); - writeFileSync(db, "/large.bin", original, {}, () => 1000); - - // Touch only the middle chunk, at a later mtime. - writeRangeSync(db, "/large.bin", new Uint8Array([7]), CHUNK_SIZE + 10, {}, () => 1001); - - // applyChunkedInodeUpdate skips SQL entirely for unchanged chunks, - // so their blobs are never re-upserted and keep the original - // last_seen; only the rewritten middle chunk's blob is stamped with - // the new mtime. (vfs_chunks is WITHOUT ROWID, so there is no rowid - // to watch. last_seen is bumped only by upsertChunkBlob, so an - // unchanged last_seen proves the chunk row was skipped.) - const rows = chunkRows(db, "/large.bin"); - expect(rows).toHaveLength(3); - const lastSeen = (hash: Uint8Array): number | undefined => - db.one<{ last_seen: number }>("SELECT last_seen FROM vfs_blobs WHERE hash = ?", hash) - ?.last_seen; - expect(lastSeen(rows[0].hash)).toBe(1000); - expect(lastSeen(rows[1].hash)).toBe(1001); - expect(lastSeen(rows[2].hash)).toBe(1000); - }); - }); - - it("invalidates the manifest hash after a direct range write", async () => { - await withDB(async (db) => { - const original = new Uint8Array(CHUNK_SIZE * 2); - original.fill(1, 0, CHUNK_SIZE); - original.fill(2, CHUNK_SIZE); - writeFileSync(db, "/large.bin", original, {}, () => 1000); - expect(manifestHash(db, "/large.bin")).not.toBe(null); - - writeRangeSync(db, "/large.bin", new Uint8Array([5]), 10, {}, () => 1001); - expect(manifestHash(db, "/large.bin")).toBe(null); - }); - }); - - it("truncates chunk-backed files without rewriting untouched chunks", async () => { - await withDB(async (db) => { - const original = new Uint8Array(CHUNK_SIZE * 2 + 100); - original.fill(1, 0, CHUNK_SIZE); - original.fill(2, CHUNK_SIZE, CHUNK_SIZE * 2); - original.fill(3, CHUNK_SIZE * 2); - writeFileSync(db, "/truncate.bin", original, {}, () => 1000); - const before = chunkRows(db, "/truncate.bin"); - - truncateFileSync(db, "/truncate.bin", CHUNK_SIZE + 50, () => 1001); - const after = chunkRows(db, "/truncate.bin"); - - expect(after).toHaveLength(2); - expect(after[1].size).toBe(50); - expect(Buffer.from(after[0].hash).equals(Buffer.from(before[0].hash))).toBe(true); - expect(Buffer.from(after[1].hash).equals(Buffer.from(before[1].hash))).toBe(false); - expect((await readBytes(db, "/truncate.bin")).byteLength).toBe(CHUNK_SIZE + 50); - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/index.ts b/spikes/349-dofs/vendor/dofs/src/index.ts deleted file mode 100644 index 76ef05a5..00000000 --- a/spikes/349-dofs/vendor/dofs/src/index.ts +++ /dev/null @@ -1,68 +0,0 @@ -export type { WorkspaceErrorCode, WorkspaceFsError } from "./errors.js"; -export { createWorkspaceError } from "./errors.js"; -export { chmod } from "./fs/chmod.js"; -export { - WorkspaceFilesystem, - type WorkspaceFilesystemOptions, -} from "./fs/filesystem.js"; -export type { WorkspaceFoundEntry } from "./fs/find.js"; -export type { GrepOptions, WorkspaceGrepMatch } from "./fs/grep.js"; -export { link } from "./fs/link.js"; -export type { MkdirOptions } from "./fs/mkdir.js"; -// Read-only mount enforcement. The workspace-side indexer writes -// _vfs_mounts; the helpers here let it invalidate the in-Database -// cache after a write, and let dofs callers (and tests) inspect or -// assert against the registered roots without re-implementing the -// overlap check. -export { - assertNotReadOnly, - getReadOnlyMountRoots, - invalidateReadOnlyMountCache, - readOnlyRootFor, -} from "./fs/mount-guard.js"; -export type { ReaddirOptions, WorkspaceDirentResult } from "./fs/readdir.js"; -export type { ReadFileOptions } from "./fs/readFile.js"; -export { readlink } from "./fs/readlink.js"; -export type { RmOptions } from "./fs/rm.js"; -export { lstat, stat, type WorkspaceStatResult } from "./fs/stat.js"; -export { symlink } from "./fs/symlink.js"; -export type { WriteFileContent, WriteFileOptions } from "./fs/writeFile.js"; -export type { SQLiteWorkspaceProviderOptions } from "./provider.js"; -export { SQLiteWorkspaceProvider } from "./provider.js"; -export { initializeSchema, ROOT_INODE, SCHEMA_VERSION } from "./schema/index.js"; -export { Database } from "./storage.js"; -export type { ApplyOptions, ApplyResult, SkippedEntry } from "./sync/apply.js"; -// Sync protocol building blocks. The wire wiring lives in -// @cloudflare/computer-rpc; these are the helpers that wiring binds -// to a Database. -export { applyChanges, applyChangesSync } from "./sync/apply.js"; -export { stageBlob } from "./sync/blobs.js"; -export type { ChangeEntry } from "./sync/changes.js"; -export { materialiseChange } from "./sync/changes.js"; -export type { CoalesceOptions } from "./sync/coalesce.js"; -export { coalesceChanges } from "./sync/coalesce.js"; -export { fetchChanges, fetchObjects, hasObjects } from "./sync/fetch.js"; -export { DEFAULT_IGNORE, isIgnored } from "./sync/ignore.js"; -export { assertAppliedPushCursor } from "./sync/invariant.js"; -export type { ManifestChunk } from "./sync/manifests.js"; -export { buildManifest, MANIFEST_VERSION } from "./sync/manifests.js"; -export { pushObjects } from "./sync/push.js"; -export type { ChangeCursor, WatermarkKey } from "./sync/watermarks.js"; -export { - compareChangeCursors, - currentRev, - readFetchCursor, - readWatermark, - writeFetchCursor, - writeWatermark, -} from "./sync/watermarks.js"; -export type { ExecutedStatement } from "./testing-recording.js"; -// RecordingStorage is workerd-safe (pure JS). SQLiteTestStorage -// wraps node:sqlite and must be imported from -// '@cloudflare/dofs/testing' under node-only call sites. -export { RecordingStorage } from "./testing-recording.js"; -export type { - DurableObjectStorageLike, - SQLCursorLike, - SQLStorageLike, -} from "./types.js"; diff --git a/spikes/349-dofs/vendor/dofs/src/path.test.ts b/spikes/349-dofs/vendor/dofs/src/path.test.ts deleted file mode 100644 index 6d513baa..00000000 --- a/spikes/349-dofs/vendor/dofs/src/path.test.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { canonicalizePath } from "./path.js"; - -describe("canonicalizePath", () => { - it("requires an absolute path", () => { - expect(() => canonicalizePath("workspace/file.txt")).toThrowError(/must be absolute/); - }); - - it("normalizes duplicate slashes and dot segments", () => { - expect(canonicalizePath("/workspace//src/./index.ts")).toEqual({ - path: "/workspace/src/index.ts", - parts: ["workspace", "src", "index.ts"], - name: "index.ts", - parentPath: "/workspace/src", - }); - }); - - it("canonicalizes parent segments without escaping root", () => { - expect(canonicalizePath("/workspace/src/../README.md").path).toBe("/workspace/README.md"); - expect(() => canonicalizePath("/..")).toThrowError(/escapes root/); - }); - - it("represents root explicitly", () => { - expect(canonicalizePath("/")).toEqual({ - path: "/", - parts: [], - name: "", - parentPath: undefined, - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/path.ts b/spikes/349-dofs/vendor/dofs/src/path.ts deleted file mode 100644 index 777a39a0..00000000 --- a/spikes/349-dofs/vendor/dofs/src/path.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { invalidPath } from "./errors.js"; - -export interface CanonicalPath { - path: string; - parts: string[]; - name: string; - parentPath: string | undefined; -} - -export function canonicalizePath(path: string): CanonicalPath { - if (path.length === 0) { - throw invalidPath(path, "empty"); - } - - if (!path.startsWith("/")) { - throw invalidPath(path, "must be absolute"); - } - - if (path.includes("\0")) { - throw invalidPath(path, "contains NUL byte"); - } - - const parts: string[] = []; - for (const part of path.split("/")) { - if (part === "" || part === ".") { - continue; - } - - if (part === "..") { - if (parts.length === 0) { - throw invalidPath(path, "escapes root"); - } - parts.pop(); - continue; - } - - parts.push(part); - } - - const canonical = parts.length === 0 ? "/" : `/${parts.join("/")}`; - const name = parts.length === 0 ? "" : parts[parts.length - 1]; - const parentParts = parts.slice(0, -1); - const parentPath = - parts.length === 0 ? undefined : parentParts.length === 0 ? "/" : `/${parentParts.join("/")}`; - - return { - path: canonical, - parts, - name, - parentPath, - }; -} diff --git a/spikes/349-dofs/vendor/dofs/src/provider.fd.test.ts b/spikes/349-dofs/vendor/dofs/src/provider.fd.test.ts deleted file mode 100644 index e7b0f5b8..00000000 --- a/spikes/349-dofs/vendor/dofs/src/provider.fd.test.ts +++ /dev/null @@ -1,376 +0,0 @@ -// File descriptor / positional I/O tests for SQLiteWorkspaceProvider. -// Separate file so the read/write coverage can grow without bloating -// the scaffold test file. - -import { describe, expect, it } from "vitest"; - -import { resolveInode } from "./fs/resolve.js"; -import { withDB } from "./fs/with-db.js"; -import { SQLiteWorkspaceProvider } from "./provider.js"; - -async function withProvider(fn: (p: SQLiteWorkspaceProvider) => T | Promise): Promise { - return withDB((db) => fn(new SQLiteWorkspaceProvider(db, { now: () => 1000 }))); -} - -// 512 KiB to match writeFile's CHUNK_SIZE so tests can deliberately -// straddle chunk boundaries. -const CHUNK_SIZE = 512 * 1024; - -function chunkHashes(p: SQLiteWorkspaceProvider, path: string): Buffer[] { - const node = resolveInode(p.db, path); - if (node === null) throw new Error(`missing node: ${path}`); - return p.db - .all<{ hash: Uint8Array }>( - "SELECT hash FROM vfs_chunks WHERE inode = ? ORDER BY idx", - node.inode, - ) - .map((row) => Buffer.from(row.hash)); -} - -describe("SQLiteWorkspaceProvider — file descriptors", () => { - it("openSync allocates a positive integer", async () => { - await withProvider((p) => { - p.writeFileSync("/a", "hello"); - const fd = p.openSync("/a", "r"); - expect(typeof fd).toBe("number"); - expect((fd as number) > 0).toBe(true); - p.closeSync(fd as number); - }); - }); - - it("openSync('r') on a missing file throws ENOENT", async () => { - await withProvider((p) => { - expect(() => p.openSync("/missing", "r")).toThrowError( - expect.objectContaining({ code: "ENOENT" }), - ); - }); - }); - - it("openSync('w') creates a missing file as empty", async () => { - await withProvider((p) => { - const fd = p.openSync("/new", "w"); - expect(p.existsSync("/new")).toBe(true); - expect(p.statSync("/new").size).toBe(0); - p.closeSync(fd as number); - }); - }); - - it("openSync('w') truncates an existing file", async () => { - await withProvider((p) => { - p.writeFileSync("/a", "before"); - const fd = p.openSync("/a", "w"); - expect(p.statSync("/a").size).toBe(0); - p.closeSync(fd as number); - }); - }); - - it("openSync('a') opens for append without truncating", async () => { - await withProvider((p) => { - p.writeFileSync("/a", "hello"); - const fd = p.openSync("/a", "a"); - expect(p.statSync("/a").size).toBe(5); - p.closeSync(fd as number); - }); - }); - - it("closeSync on an unknown fd throws EBADF", async () => { - await withProvider((p) => { - expect(() => p.closeSync(9999)).toThrowError(expect.objectContaining({ code: "EBADF" })); - }); - }); - - it("fstatSync mirrors statSync for the fd's path", async () => { - await withProvider((p) => { - p.writeFileSync("/a", "hello", { mode: 0o644 }); - const fd = p.openSync("/a", "r") as number; - const s = p.fstatSync(fd); - expect(s.size).toBe(5); - expect(s.isFile()).toBe(true); - p.closeSync(fd); - }); - }); -}); - -describe("SQLiteWorkspaceProvider — readSync", () => { - it("reads from the fd's position when position is null", async () => { - await withProvider((p) => { - p.writeFileSync("/a", "hello workspace"); - const fd = p.openSync("/a", "r") as number; - const buf = Buffer.alloc(5); - const n = p.readSync(fd, buf, 0, 5, null); - expect(n).toBe(5); - expect(buf.toString()).toBe("hello"); - // Position advanced; next read continues. - const n2 = p.readSync(fd, buf, 0, 5, null); - expect(n2).toBe(5); - expect(buf.toString()).toBe(" work"); - p.closeSync(fd); - }); - }); - - it("reads at an explicit position without moving the fd", async () => { - await withProvider((p) => { - p.writeFileSync("/a", "hello workspace"); - const fd = p.openSync("/a", "r") as number; - const buf = Buffer.alloc(5); - const n = p.readSync(fd, buf, 0, 5, 6); - expect(n).toBe(5); - expect(buf.toString()).toBe("works"); - // Fd position unchanged: still at 0. - p.readSync(fd, buf, 0, 5, null); - expect(buf.toString()).toBe("hello"); - p.closeSync(fd); - }); - }); - - it("returns 0 when reading past EOF", async () => { - await withProvider((p) => { - p.writeFileSync("/a", "tiny"); - const fd = p.openSync("/a", "r") as number; - const buf = Buffer.alloc(10); - expect(p.readSync(fd, buf, 0, 10, 100)).toBe(0); - p.closeSync(fd); - }); - }); - - it("respects the buffer offset", async () => { - await withProvider((p) => { - p.writeFileSync("/a", "abcde"); - const fd = p.openSync("/a", "r") as number; - const buf = Buffer.alloc(10).fill(0x2e); // '.' - const n = p.readSync(fd, buf, 3, 5, 0); - expect(n).toBe(5); - expect(buf.toString()).toBe("...abcde.."); - p.closeSync(fd); - }); - }); - - it("reads across a chunk boundary", async () => { - await withProvider((p) => { - // Build a file that crosses one 512KiB boundary. - const bytes = new Uint8Array(CHUNK_SIZE + 100); - bytes.fill(0x41); - for (let i = CHUNK_SIZE; i < bytes.byteLength; i++) bytes[i] = 0x42; - p.writeFileSync("/big", Buffer.from(bytes)); - const fd = p.openSync("/big", "r") as number; - const buf = Buffer.alloc(200); - // Straddle the boundary: read 200 bytes starting 100 bytes before it. - const n = p.readSync(fd, buf, 0, 200, CHUNK_SIZE - 100); - expect(n).toBe(200); - // First 100 bytes are 'A' (pre-boundary), remaining 100 are 'B'. - for (let i = 0; i < 100; i++) expect(buf[i]).toBe(0x41); - for (let i = 100; i < 200; i++) expect(buf[i]).toBe(0x42); - p.closeSync(fd); - }); - }); -}); - -describe("SQLiteWorkspaceProvider — direct range methods", () => { - it("exposes direct create, write range, and truncate methods", async () => { - await withProvider((p) => { - p.createFileSync("/direct.txt", { mode: 0o600 }); - expect(p.statSync("/direct.txt").mode & 0o777).toBe(0o600); - - expect(p.writeRangeSync("/direct.txt", Buffer.from("abcdef"), 0)).toBe(6); - expect(p.writeRangeSync("/direct.txt", Buffer.from("Z"), 3)).toBe(1); - expect(p.readFileSync("/direct.txt", "utf8")).toBe("abcZef"); - - p.truncateFileSync("/direct.txt", 4); - expect(p.readFileSync("/direct.txt", "utf8")).toBe("abcZ"); - }); - }); -}); - -describe("SQLiteWorkspaceProvider — writeSync", () => { - it("writes at position 0 and updates content", async () => { - await withProvider((p) => { - p.writeFileSync("/a", "hello"); - const fd = p.openSync("/a", "r+") as number; - const n = p.writeSync(fd, Buffer.from("HELLO"), 0, 5, 0); - expect(n).toBe(5); - p.closeSync(fd); - expect(p.readFileSync("/a", "utf8")).toBe("HELLO"); - }); - }); - - it("writes at a non-zero offset", async () => { - await withProvider((p) => { - p.writeFileSync("/a", "hello world"); - const fd = p.openSync("/a", "r+") as number; - p.writeSync(fd, Buffer.from("WORLD"), 0, 5, 6); - p.closeSync(fd); - expect(p.readFileSync("/a", "utf8")).toBe("hello WORLD"); - }); - }); - - it("extends the file when writing past EOF", async () => { - await withProvider((p) => { - p.writeFileSync("/a", "hi"); - const fd = p.openSync("/a", "r+") as number; - p.writeSync(fd, Buffer.from("bye"), 0, 3, 2); - p.closeSync(fd); - expect(p.readFileSync("/a", "utf8")).toBe("hibye"); - }); - }); - - it("zero-fills the gap when writing past current size", async () => { - await withProvider((p) => { - p.writeFileSync("/a", "ab"); - const fd = p.openSync("/a", "r+") as number; - p.writeSync(fd, Buffer.from("z"), 0, 1, 5); - p.closeSync(fd); - const stat = p.statSync("/a"); - expect(stat.size).toBe(6); - const out = p.readFileSync("/a") as Buffer; - expect(out[0]).toBe(0x61); // 'a' - expect(out[1]).toBe(0x62); // 'b' - expect(out[2]).toBe(0); - expect(out[3]).toBe(0); - expect(out[4]).toBe(0); - expect(out[5]).toBe(0x7a); // 'z' - }); - }); - - it("advances the fd position when position is null", async () => { - await withProvider((p) => { - p.writeFileSync("/a", ""); - const fd = p.openSync("/a", "w") as number; - p.writeSync(fd, Buffer.from("abc"), 0, 3, null); - p.writeSync(fd, Buffer.from("def"), 0, 3, null); - p.closeSync(fd); - expect(p.readFileSync("/a", "utf8")).toBe("abcdef"); - }); - }); - - it("writes across a chunk boundary, splicing in the affected chunks only", async () => { - await withProvider((p) => { - const before = new Uint8Array(CHUNK_SIZE + 100); - before.fill(0x41); - for (let i = CHUNK_SIZE; i < before.byteLength; i++) before[i] = 0x42; - p.writeFileSync("/big", Buffer.from(before)); - - const fd = p.openSync("/big", "r+") as number; - // Overwrite 200 bytes that straddle the boundary with 'Z'. - const stamp = Buffer.alloc(200, 0x5a); - p.writeSync(fd, stamp, 0, 200, CHUNK_SIZE - 100); - p.closeSync(fd); - - const out = p.readFileSync("/big") as Buffer; - expect(out.byteLength).toBe(CHUNK_SIZE + 100); - expect(out[0]).toBe(0x41); - expect(out[CHUNK_SIZE - 101]).toBe(0x41); - expect(out[CHUNK_SIZE - 100]).toBe(0x5a); - expect(out[CHUNK_SIZE + 99]).toBe(0x5a); - // Anything past the overwrite is whatever remained of 'B'. - // (The original had only 100 B-bytes total, all of which we overwrote.) - }); - }); - - it("reuses untouched chunk rows for positional writes", async () => { - await withProvider((p) => { - const before = new Uint8Array(CHUNK_SIZE * 3); - before.fill(1, 0, CHUNK_SIZE); - before.fill(2, CHUNK_SIZE, CHUNK_SIZE * 2); - before.fill(3, CHUNK_SIZE * 2); - p.writeFileSync("/big", Buffer.from(before)); - const oldHashes = chunkHashes(p, "/big"); - - const fd = p.openSync("/big", "r+") as number; - p.writeSync(fd, Buffer.from([9, 9, 9]), 0, 3, CHUNK_SIZE + 10); - p.closeSync(fd); - const newHashes = chunkHashes(p, "/big"); - - expect(newHashes[0].equals(oldHashes[0])).toBe(true); - expect(newHashes[1].equals(oldHashes[1])).toBe(false); - expect(newHashes[2].equals(oldHashes[2])).toBe(true); - }); - }); - - it("openSync('a') starts the fd at EOF", async () => { - await withProvider((p) => { - p.writeFileSync("/a", "hello"); - const fd = p.openSync("/a", "a") as number; - p.writeSync(fd, Buffer.from(" world"), 0, 6, null); - p.closeSync(fd); - expect(p.readFileSync("/a", "utf8")).toBe("hello world"); - }); - }); -}); - -describe("SQLiteWorkspaceProvider — truncateSync / ftruncateSync", () => { - it("truncateSync shrinks a file", async () => { - await withProvider((p) => { - p.writeFileSync("/a", "hello world"); - p.truncateSync("/a", 5); - expect(p.readFileSync("/a", "utf8")).toBe("hello"); - }); - }); - - it("truncateSync grows a file with zero fill", async () => { - await withProvider((p) => { - p.writeFileSync("/a", "abc"); - p.truncateSync("/a", 6); - const out = p.readFileSync("/a") as Buffer; - expect(out.byteLength).toBe(6); - expect(out[0]).toBe(0x61); - expect(out[3]).toBe(0); - }); - }); - - it("truncateSync to 0 leaves an empty file", async () => { - await withProvider((p) => { - p.writeFileSync("/a", "hello"); - p.truncateSync("/a", 0); - expect(p.statSync("/a").size).toBe(0); - }); - }); - - it("truncateSync at the same size is a no-op", async () => { - await withProvider((p) => { - p.writeFileSync("/a", "hello"); - p.truncateSync("/a", 5); - expect(p.readFileSync("/a", "utf8")).toBe("hello"); - }); - }); - - it("truncateSync shrinks across a chunk boundary", async () => { - await withProvider((p) => { - const bytes = new Uint8Array(CHUNK_SIZE + 100); - bytes.fill(0x41); - p.writeFileSync("/big", Buffer.from(bytes)); - p.truncateSync("/big", CHUNK_SIZE - 10); - expect(p.statSync("/big").size).toBe(CHUNK_SIZE - 10); - }); - }); - - it("truncateSync grows across a chunk boundary", async () => { - await withProvider((p) => { - p.writeFileSync("/a", "ab"); - p.truncateSync("/a", CHUNK_SIZE + 100); - expect(p.statSync("/a").size).toBe(CHUNK_SIZE + 100); - const out = p.readFileSync("/a") as Buffer; - expect(out[0]).toBe(0x61); - expect(out[1]).toBe(0x62); - expect(out[2]).toBe(0); - expect(out[CHUNK_SIZE + 99]).toBe(0); - }); - }); - - it("truncateSync on a missing file throws ENOENT", async () => { - await withProvider((p) => { - expect(() => p.truncateSync("/missing", 0)).toThrowError( - expect.objectContaining({ code: "ENOENT" }), - ); - }); - }); - - it("ftruncateSync mirrors truncateSync through an fd", async () => { - await withProvider((p) => { - p.writeFileSync("/a", "hello world"); - const fd = p.openSync("/a", "r+") as number; - p.ftruncateSync(fd, 5); - p.closeSync(fd); - expect(p.readFileSync("/a", "utf8")).toBe("hello"); - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/provider.test.ts b/spikes/349-dofs/vendor/dofs/src/provider.test.ts deleted file mode 100644 index 769086d7..00000000 --- a/spikes/349-dofs/vendor/dofs/src/provider.test.ts +++ /dev/null @@ -1,854 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { withDB } from "./fs/with-db.js"; -import { SQLiteWorkspaceProvider } from "./provider.js"; -import type { Database } from "./storage.js"; -import type { ChangeEntry } from "./sync/changes.js"; -import { coalesceChanges } from "./sync/coalesce.js"; - -// Each provider test gets a fresh DB via withDB, which the workers -// runner aliases to a DO-backed implementation. The provider holds -// no I/O resources of its own, so it's safe to construct inside the -// withDB callback and let the storage handle teardown. -async function withProvider(fn: (p: SQLiteWorkspaceProvider) => T | Promise): Promise { - return withDB((db) => fn(new SQLiteWorkspaceProvider(db, { now: () => 1000 }))); -} - -async function withProviderAndDB( - fn: (p: SQLiteWorkspaceProvider, db: Database) => T | Promise, -): Promise { - return withDB((db) => fn(new SQLiteWorkspaceProvider(db, { now: () => 1000 }), db)); -} - -async function drainChanges(db: Database, afterRev: number): Promise { - const out: ChangeEntry[] = []; - for await (const entry of coalesceChanges(db, afterRev)) out.push(entry); - return out; -} - -function kindPath(entries: ChangeEntry[]): Array<[ChangeEntry["kind"], string]> { - return entries.map((entry) => [entry.kind, entry.path]).sort((a, b) => a[1].localeCompare(b[1])); -} - -describe("SQLiteWorkspaceProvider — capability flags", () => { - it("reports the supported feature set", async () => { - await withProvider((p) => { - expect(p.readonly).toBe(false); - expect(p.supportsSymlinks).toBe(true); - expect(p.supportsWatch).toBe(true); - }); - }); -}); - -describe("SQLiteWorkspaceProvider — implemented methods", () => { - it("mkdirSync creates a directory", async () => { - await withProvider((p) => { - p.mkdirSync("/a", { mode: 0o755 }); - expect(p.existsSync("/a")).toBe(true); - }); - }); - - it("statSync returns a VirtualStats-shaped object", async () => { - await withProvider((p) => { - p.mkdirSync("/a", {}); - const s = p.statSync("/a"); - expect(s.isDirectory()).toBe(true); - expect(s.isFile()).toBe(false); - expect(s.isSymbolicLink()).toBe(false); - // 0o40755 — S_IFDIR or permissions. Linux FUSE rejects a - // stat without the file-type bits, so we always set them. - expect(s.mode).toBe(0o40755); - expect(typeof s.ino).toBe("number"); - expect(typeof s.mtimeMs).toBe("number"); - expect(s.mtime).toBeInstanceOf(Date); - }); - }); - - it("lstatSync returns the same shape as statSync today (no symlinks yet)", async () => { - await withProvider((p) => { - p.mkdirSync("/a", {}); - expect(p.lstatSync("/a").isDirectory()).toBe(true); - }); - }); - - it("readdirSync returns names by default and dirent objects with withFileTypes", async () => { - await withProvider((p) => { - p.mkdirSync("/a", {}); - p.mkdirSync("/b", {}); - expect(p.readdirSync("/")).toEqual(["a", "b"]); - const dirents = p.readdirSync("/", { withFileTypes: true }); - expect(Array.isArray(dirents)).toBe(true); - expect((dirents as Array<{ name: string; isDirectory(): boolean }>)[0].isDirectory()).toBe( - true, - ); - }); - }); - - it("unlinkSync removes a file", async () => { - await withProvider((p) => { - p.writeFileSync("/a.txt", "hi"); - p.unlinkSync("/a.txt"); - expect(p.existsSync("/a.txt")).toBe(false); - }); - }); - - it("linkSync creates a second path to the same file inode", async () => { - await withProvider((p) => { - p.writeFileSync("/a.txt", "hi"); - p.linkSync("/a.txt", "/b.txt"); - - const a = p.statSync("/a.txt"); - const b = p.statSync("/b.txt"); - expect(a.ino).toBe(b.ino); - expect(a.nlink).toBe(2); - expect(b.nlink).toBe(2); - expect(p.readFileSync("/b.txt", "utf8")).toBe("hi"); - }); - }); - - it("writes through one hardlink are visible through the other", async () => { - await withProvider((p) => { - p.writeFileSync("/a.txt", "hi"); - p.linkSync("/a.txt", "/b.txt"); - p.writeFileSync("/b.txt", "bye"); - - expect(p.readFileSync("/a.txt", "utf8")).toBe("bye"); - expect(p.statSync("/a.txt").nlink).toBe(2); - expect(p.statSync("/b.txt").nlink).toBe(2); - }); - }); - - it("unlinkSync removes one hardlink without deleting the inode", async () => { - await withProvider((p) => { - p.writeFileSync("/a.txt", "hi"); - p.linkSync("/a.txt", "/b.txt"); - p.unlinkSync("/a.txt"); - - expect(p.existsSync("/a.txt")).toBe(false); - expect(p.readFileSync("/b.txt", "utf8")).toBe("hi"); - expect(p.statSync("/b.txt").nlink).toBe(1); - }); - }); - - it("renameSync from one hardlink onto another removes only the source name", async () => { - await withProvider((p) => { - p.writeFileSync("/a.txt", "hi"); - p.linkSync("/a.txt", "/b.txt"); - p.renameSync("/a.txt", "/b.txt"); - - expect(p.existsSync("/a.txt")).toBe(false); - expect(p.readFileSync("/b.txt", "utf8")).toBe("hi"); - expect(p.statSync("/b.txt").nlink).toBe(1); - }); - }); - - it("linkSync rejects invalid links", async () => { - await withProvider((p) => { - p.writeFileSync("/a.txt", "hi"); - p.mkdirSync("/dir", {}); - - expect(() => p.linkSync("/missing", "/missing-link")).toThrowError( - expect.objectContaining({ code: "ENOENT" }), - ); - expect(() => p.linkSync("/a.txt", "/a.txt")).toThrowError( - expect.objectContaining({ code: "EEXIST" }), - ); - expect(() => p.linkSync("/dir", "/dir-link")).toThrowError( - expect.objectContaining({ code: "EPERM" }), - ); - expect(() => p.linkSync("/a.txt", "/missing-parent/b.txt")).toThrowError( - expect.objectContaining({ code: "ENOENT" }), - ); - }); - }); - - it("unlinkSync removes a symlink without deleting its target", async () => { - await withProvider((p) => { - p.writeFileSync("/target", "content"); - p.symlinkSync("/target", "/link"); - - p.unlinkSync("/link"); - - expect(p.existsSync("/link")).toBe(false); - expect(p.readFileSync("/target", "utf8")).toBe("content"); - }); - }); - - it("rmdirSync removes an empty directory", async () => { - await withProvider((p) => { - p.mkdirSync("/a", {}); - p.rmdirSync("/a"); - expect(p.existsSync("/a")).toBe(false); - }); - }); - - it("renameSync moves an entry", async () => { - await withProvider((p) => { - p.writeFileSync("/a", "x"); - p.renameSync("/a", "/b"); - expect(p.existsSync("/a")).toBe(false); - expect(p.existsSync("/b")).toBe(true); - }); - }); - - it("writeFileSync + readFileSync round-trip a string", async () => { - await withProvider((p) => { - p.writeFileSync("/a.txt", "hello workspace"); - expect(p.readFileSync("/a.txt", "utf8")).toBe("hello workspace"); - }); - }); - - it("writeFileSync + readFileSync round-trip bytes", async () => { - await withProvider((p) => { - p.writeFileSync("/a.bin", Buffer.from([1, 2, 3])); - const back = p.readFileSync("/a.bin"); - expect(back).toBeInstanceOf(Buffer); - expect(Array.from(back as Buffer)).toEqual([1, 2, 3]); - }); - }); - - it("existsSync returns true / false correctly", async () => { - await withProvider((p) => { - expect(p.existsSync("/missing")).toBe(false); - p.mkdirSync("/d", {}); - expect(p.existsSync("/d")).toBe(true); - }); - }); - - it("realpathSync returns the canonical path", async () => { - await withProvider((p) => { - p.mkdirSync("/a", {}); - expect(p.realpathSync("/a/./../a")).toBe("/a"); - }); - }); - - it("accessSync resolves for existing paths and throws ENOENT for missing", async () => { - await withProvider((p) => { - p.mkdirSync("/a", {}); - expect(() => p.accessSync("/a")).not.toThrow(); - expect(() => p.accessSync("/missing")).toThrowError( - expect.objectContaining({ code: "ENOENT" }), - ); - }); - }); -}); - -describe("SQLiteWorkspaceProvider — renameSync overwrite matrix", () => { - it("records rename as an old-path delete and new-path live entry", async () => { - await withProviderAndDB(async (p, db) => { - p.writeFileSync("/src", "new"); - const cursor = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; - - p.renameSync("/src", "/dst"); - - const entries = await drainChanges(db, cursor); - expect(kindPath(entries)).toEqual([ - ["file", "/dst"], - ["delete", "/src"], - ]); - }); - }); - - it("records directory rename for the whole moved subtree", async () => { - await withProviderAndDB(async (p, db) => { - p.mkdirSync("/src", {}); - p.writeFileSync("/src/a.txt", "a"); - p.mkdirSync("/src/sub", {}); - p.writeFileSync("/src/sub/b.txt", "b"); - const cursor = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; - - p.renameSync("/src", "/dst"); - - const entries = await drainChanges(db, cursor); - expect(kindPath(entries)).toEqual([ - ["dir", "/dst"], - ["file", "/dst/a.txt"], - ["dir", "/dst/sub"], - ["file", "/dst/sub/b.txt"], - ["delete", "/src"], - ["delete", "/src/a.txt"], - ["delete", "/src/sub"], - ["delete", "/src/sub/b.txt"], - ]); - }); - }); - - it("records rename tombstones at the resolved old file path", async () => { - await withProviderAndDB(async (p, db) => { - p.mkdirSync("/real", {}); - p.writeFileSync("/real/file.txt", "x"); - p.symlinkSync("/real", "/link"); - const cursor = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; - - p.renameSync("/link/file.txt", "/dst.txt"); - - const entries = await drainChanges(db, cursor); - expect(kindPath(entries)).toEqual([ - ["file", "/dst.txt"], - ["delete", "/real/file.txt"], - ]); - }); - }); - - it("records directory rename tombstones at the resolved old subtree paths", async () => { - await withProviderAndDB(async (p, db) => { - p.mkdirSync("/real", {}); - p.mkdirSync("/real/dir", {}); - p.writeFileSync("/real/dir/a.txt", "a"); - p.mkdirSync("/real/dir/sub", {}); - p.writeFileSync("/real/dir/sub/b.txt", "b"); - p.symlinkSync("/real", "/link"); - const cursor = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; - - p.renameSync("/link/dir", "/dst"); - - const entries = await drainChanges(db, cursor); - expect(kindPath(entries)).toEqual([ - ["dir", "/dst"], - ["file", "/dst/a.txt"], - ["dir", "/dst/sub"], - ["file", "/dst/sub/b.txt"], - ["delete", "/real/dir"], - ["delete", "/real/dir/a.txt"], - ["delete", "/real/dir/sub"], - ["delete", "/real/dir/sub/b.txt"], - ]); - }); - }); - - it("subtree rename writes one tombstone per edge and one rev stamp per inode", async () => { - await withProviderAndDB(async (p, db) => { - // A nested subtree with a hardlink inside it: the file inode is - // reachable by two names, so both edges must be tombstoned while - // the single inode is stamped once. - p.mkdirSync("/src", {}); - p.writeFileSync("/src/a.txt", "a"); - p.mkdirSync("/src/sub", {}); - p.writeFileSync("/src/sub/b.txt", "b"); - p.linkSync("/src/a.txt", "/src/sub/a2.txt"); - const cursor = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; - - p.renameSync("/src", "/dst"); - - // Every edge of the moved subtree, tombstoned at its old path and - // sharing the rename's rev. - const tombstones = db.all<{ rev: number; path: string; op: string }>( - "SELECT rev, path, op FROM vfs_changes WHERE rev > ? ORDER BY path", - cursor, - ); - const rev = tombstones[0]?.rev; - expect(tombstones).toEqual([ - { rev, path: "/src", op: "delete" }, - { rev, path: "/src/a.txt", op: "delete" }, - { rev, path: "/src/sub", op: "delete" }, - { rev, path: "/src/sub/a2.txt", op: "delete" }, - { rev, path: "/src/sub/b.txt", op: "delete" }, - ]); - - // The shared rev lands on exactly the four subtree inodes (the - // hardlinked file counted once) and on nothing else. - const stamped = db - .all<{ inode: number }>("SELECT inode FROM vfs_nodes WHERE rev = ? ORDER BY inode", rev) - .map((r) => r.inode); - const expected = [ - p.statSync("/dst").ino, - p.statSync("/dst/a.txt").ino, - p.statSync("/dst/sub").ino, - p.statSync("/dst/sub/b.txt").ino, - ].sort((a, b) => a - b); - expect(stamped).toEqual(expected); - }); - }); - - it("same-inode rename through a symlinked file path is a no-op", async () => { - await withProviderAndDB(async (p, db) => { - p.mkdirSync("/real", {}); - p.writeFileSync("/real/file.txt", "x"); - p.symlinkSync("/real", "/link"); - const cursor = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; - - p.renameSync("/real/file.txt", "/link/file.txt"); - - expect(p.readFileSync("/real/file.txt", "utf8")).toBe("x"); - expect(p.readFileSync("/link/file.txt", "utf8")).toBe("x"); - expect(await drainChanges(db, cursor)).toEqual([]); - }); - }); - - it("same-inode rename through a symlinked directory path is a no-op", async () => { - await withProviderAndDB(async (p, db) => { - p.mkdirSync("/real", {}); - p.mkdirSync("/real/dir", {}); - p.writeFileSync("/real/dir/a.txt", "a"); - p.symlinkSync("/real", "/link"); - const cursor = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; - - p.renameSync("/real/dir", "/link/dir"); - - expect(p.readFileSync("/real/dir/a.txt", "utf8")).toBe("a"); - expect(p.readFileSync("/link/dir/a.txt", "utf8")).toBe("a"); - expect(await drainChanges(db, cursor)).toEqual([]); - }); - }); - - it("file → existing file overwrites atomically", async () => { - await withProvider((p) => { - p.writeFileSync("/src", "new"); - p.writeFileSync("/dst", "old"); - p.renameSync("/src", "/dst"); - expect(p.existsSync("/src")).toBe(false); - expect(p.readFileSync("/dst", "utf8")).toBe("new"); - }); - }); - - it("dir → existing non-empty dir throws ENOTEMPTY", async () => { - await withProvider((p) => { - p.mkdirSync("/src", {}); - p.mkdirSync("/dst", {}); - p.writeFileSync("/dst/inside", "x"); - expect(() => p.renameSync("/src", "/dst")).toThrowError( - expect.objectContaining({ code: "ENOTEMPTY" }), - ); - // Both directories survive the failed rename. - expect(p.existsSync("/src")).toBe(true); - expect(p.existsSync("/dst/inside")).toBe(true); - }); - }); - - it("dir → existing empty dir succeeds", async () => { - await withProvider((p) => { - p.mkdirSync("/src", {}); - p.writeFileSync("/src/inside", "x"); - p.mkdirSync("/dst", {}); - p.renameSync("/src", "/dst"); - expect(p.existsSync("/src")).toBe(false); - expect(p.readFileSync("/dst/inside", "utf8")).toBe("x"); - }); - }); - - it.each([ - { - name: "file → existing symlink", - setup(p: SQLiteWorkspaceProvider) { - p.writeFileSync("/target", "x"); - p.writeFileSync("/src", "new"); - p.symlinkSync("/target", "/dst"); - }, - assertRenamed(p: SQLiteWorkspaceProvider) { - expect(p.existsSync("/src")).toBe(false); - expect(p.lstatSync("/dst").isFile()).toBe(true); - expect(p.readFileSync("/dst", "utf8")).toBe("new"); - expect(p.readFileSync("/target", "utf8")).toBe("x"); - }, - }, - { - name: "symlink → existing file", - setup(p: SQLiteWorkspaceProvider) { - p.writeFileSync("/target", "x"); - p.symlinkSync("/target", "/src"); - p.writeFileSync("/dst", "old"); - }, - assertRenamed(p: SQLiteWorkspaceProvider) { - expect(p.existsSync("/src")).toBe(false); - expect(p.lstatSync("/dst").isSymbolicLink()).toBe(true); - expect(p.readlinkSync("/dst")).toBe("/target"); - }, - }, - { - name: "symlink → existing symlink", - setup(p: SQLiteWorkspaceProvider) { - p.writeFileSync("/target", "x"); - p.writeFileSync("/other", "y"); - p.symlinkSync("/target", "/src"); - p.symlinkSync("/other", "/dst"); - }, - assertRenamed(p: SQLiteWorkspaceProvider) { - expect(p.existsSync("/src")).toBe(false); - expect(p.lstatSync("/dst").isSymbolicLink()).toBe(true); - expect(p.readlinkSync("/dst")).toBe("/target"); - }, - }, - ])("$name overwrites atomically", async ({ setup, assertRenamed }) => { - await withProvider((p) => { - setup(p); - p.renameSync("/src", "/dst"); - assertRenamed(p); - }); - }); - - it.each([ - { - name: "file → existing empty dir", - code: "EISDIR", - setup(p: SQLiteWorkspaceProvider) { - p.writeFileSync("/src", "new"); - p.mkdirSync("/dst", {}); - }, - assertUnchanged(p: SQLiteWorkspaceProvider) { - expect(p.readFileSync("/src", "utf8")).toBe("new"); - expect(p.statSync("/dst").isDirectory()).toBe(true); - }, - }, - { - name: "symlink → existing empty dir", - code: "EISDIR", - setup(p: SQLiteWorkspaceProvider) { - p.writeFileSync("/target", "x"); - p.symlinkSync("/target", "/src"); - p.mkdirSync("/dst", {}); - }, - assertUnchanged(p: SQLiteWorkspaceProvider) { - expect(p.readlinkSync("/src")).toBe("/target"); - expect(p.statSync("/dst").isDirectory()).toBe(true); - }, - }, - { - name: "dir → existing file", - code: "ENOTDIR", - setup(p: SQLiteWorkspaceProvider) { - p.mkdirSync("/src", {}); - p.writeFileSync("/dst", "old"); - }, - assertUnchanged(p: SQLiteWorkspaceProvider) { - expect(p.statSync("/src").isDirectory()).toBe(true); - expect(p.readFileSync("/dst", "utf8")).toBe("old"); - }, - }, - { - name: "dir → existing symlink", - code: "ENOTDIR", - setup(p: SQLiteWorkspaceProvider) { - p.writeFileSync("/target", "x"); - p.mkdirSync("/src", {}); - p.symlinkSync("/target", "/dst"); - }, - assertUnchanged(p: SQLiteWorkspaceProvider) { - expect(p.statSync("/src").isDirectory()).toBe(true); - expect(p.readlinkSync("/dst")).toBe("/target"); - }, - }, - ])("$name rejects without recording sync changes", async ({ code, setup, assertUnchanged }) => { - await withProviderAndDB(async (p, db) => { - setup(p); - const cursor = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; - - expect(() => p.renameSync("/src", "/dst")).toThrowError(expect.objectContaining({ code })); - - assertUnchanged(p); - expect(await drainChanges(db, cursor)).toEqual([]); - }); - }); - - it("source missing throws ENOENT", async () => { - await withProvider((p) => { - expect(() => p.renameSync("/missing", "/dst")).toThrowError( - expect.objectContaining({ code: "ENOENT" }), - ); - }); - }); - - it("same-path rename validates the source before no-op", async () => { - await withProvider((p) => { - expect(() => p.renameSync("/missing", "/missing")).toThrowError( - expect.objectContaining({ code: "ENOENT" }), - ); - }); - }); - - it("same-path rename of an existing file is a no-op", async () => { - await withProviderAndDB(async (p, db) => { - p.writeFileSync("/src", "x"); - const cursor = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; - - p.renameSync("/src", "/src"); - - expect(p.readFileSync("/src", "utf8")).toBe("x"); - expect(await drainChanges(db, cursor)).toEqual([]); - }); - }); - - it("rename of a file into itself as a parent does not report directory self-move", async () => { - await withProvider((p) => { - p.writeFileSync("/file", "x"); - expect(() => p.renameSync("/file", "/file/child")).toThrowError( - expect.objectContaining({ code: "ENOENT" }), - ); - }); - }); - - it("rename onto root throws EINVAL", async () => { - await withProvider((p) => { - p.writeFileSync("/src", "x"); - expect(() => p.renameSync("/src", "/")).toThrowError( - expect.objectContaining({ code: "EINVAL" }), - ); - }); - }); - - it("rename into a missing parent throws ENOENT", async () => { - await withProvider((p) => { - p.writeFileSync("/src", "x"); - expect(() => p.renameSync("/src", "/no-such-dir/dst")).toThrowError( - expect.objectContaining({ code: "ENOENT" }), - ); - }); - }); - - it("rename of a symlink moves the link itself", async () => { - await withProvider((p) => { - p.writeFileSync("/target", "x"); - p.symlinkSync("/target", "/link"); - p.renameSync("/link", "/moved"); - expect(p.existsSync("/target")).toBe(true); - expect(p.readlinkSync("/moved")).toBe("/target"); - expect(p.existsSync("/link")).toBe(false); - }); - }); - - it("rename of a directory into its own subtree throws EINVAL", async () => { - await withProvider((p) => { - p.mkdirSync("/src", {}); - p.mkdirSync("/src/sub", {}); - expect(() => p.renameSync("/src", "/src/sub/dst")).toThrowError( - expect.objectContaining({ code: "EINVAL" }), - ); - }); - }); - - it("rename of a directory through a symlink into its own subtree throws EINVAL", async () => { - await withProviderAndDB(async (p, db) => { - p.mkdirSync("/src", {}); - p.mkdirSync("/src/sub", {}); - p.symlinkSync("/src/sub", "/link"); - const cursor = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; - - expect(() => p.renameSync("/src", "/link/dst")).toThrowError( - expect.objectContaining({ code: "EINVAL" }), - ); - - expect(p.statSync("/src/sub").isDirectory()).toBe(true); - expect(p.readlinkSync("/link")).toBe("/src/sub"); - expect(await drainChanges(db, cursor)).toEqual([]); - }); - }); - - it("rename of a directory through a symlink out of its subtree succeeds", async () => { - await withProviderAndDB(async (p, db) => { - p.mkdirSync("/src", {}); - p.mkdirSync("/other", {}); - // /src/link points outside the source subtree, so the destination - // /src/link/dst resolves to /other/dst even though the literal path - // is lexically under /src. The self-move guard is inode-based and - // must allow this move rather than reject it on a textual prefix. - p.symlinkSync("/other", "/src/link"); - const cursor = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; - - p.renameSync("/src", "/src/link/dst"); - - expect(p.statSync("/other/dst").isDirectory()).toBe(true); - expect(p.existsSync("/src")).toBe(false); - expect(await drainChanges(db, cursor)).not.toEqual([]); - }); - }); -}); - -describe("SQLiteWorkspaceProvider — unimplemented surface (stubs)", () => { - it.each([ - ["appendFileSync", (p: SQLiteWorkspaceProvider) => p.appendFileSync("/x", "y")], - ["copyFileSync", (p: SQLiteWorkspaceProvider) => p.copyFileSync("/x", "/y")], - ["internalModuleStat", (p: SQLiteWorkspaceProvider) => p.internalModuleStat("/x")], - - ["watchFile", (p: SQLiteWorkspaceProvider) => p.watchFile("/x")], - ])("%s throws ENOSYS", async (_name, call) => { - await withProvider((p) => { - expect(() => call(p)).toThrowError(expect.objectContaining({ code: "ENOSYS" })); - }); - }); -}); - -describe("SQLiteWorkspaceProvider — pending-create flush on rename/link/unlink", () => { - it("linkSync commits a pending-create source before adding the second dirent", async () => { - await withProvider((p) => { - p.openWriteBufferForCreateSync("/src.txt", { mode: 0o644 }); - p.writeRangeSync("/src.txt", Buffer.from("linked"), 0); - p.linkSync("/src.txt", "/dst.txt"); - - expect((p.readFileSync("/dst.txt") as Buffer).toString()).toBe("linked"); - expect(p.statSync("/src.txt").ino).toBe(p.statSync("/dst.txt").ino); - expect(p.statSync("/src.txt").nlink).toBe(2); - - p.releaseWriteBufferSync("/src.txt"); - expect((p.readFileSync("/src.txt") as Buffer).toString()).toBe("linked"); - }); - }); - - it("renameSync commits a pending-create source before moving the dirent", async () => { - await withProvider((p) => { - p.openWriteBufferForCreateSync("/from.txt", { mode: 0o644 }); - p.writeRangeSync("/from.txt", Buffer.from("moved"), 0); - p.renameSync("/from.txt", "/to.txt"); - - expect((p.readFileSync("/to.txt") as Buffer).toString()).toBe("moved"); - expect(p.existsSync("/from.txt")).toBe(false); - }); - }); - - it("unlinkSync commits then removes a pending-create file", async () => { - await withProvider((p) => { - p.openWriteBufferForCreateSync("/gone.txt", { mode: 0o644 }); - p.writeRangeSync("/gone.txt", Buffer.from("bye"), 0); - p.unlinkSync("/gone.txt"); - expect(p.existsSync("/gone.txt")).toBe(false); - }); - }); - - it("linkSync flushes a pending-create destination before colliding", async () => { - // Pending /dst is committed before link's existence check - // runs, so the user sees a normal EEXIST against a real inode - // rather than silently losing the pending bytes when the later - // release would have tripped its own EEXIST against the link's - // dirent. - await withProvider((p) => { - p.writeFileSync("/src.txt", "src bytes"); - p.openWriteBufferForCreateSync("/dst.txt", { mode: 0o644 }); - p.writeRangeSync("/dst.txt", Buffer.from("pending dst"), 0); - - expect(() => p.linkSync("/src.txt", "/dst.txt")).toThrowError( - expect.objectContaining({ code: "EEXIST" }), - ); - - // /dst.txt now exists with the previously-pending bytes; the - // release-after-collision finds the inode it expects and is a - // no-op rather than a data loss. - expect((p.readFileSync("/dst.txt") as Buffer).toString()).toBe("pending dst"); - expect(() => p.releaseWriteBufferSync("/dst.txt")).not.toThrow(); - expect((p.readFileSync("/dst.txt") as Buffer).toString()).toBe("pending dst"); - }); - }); - - it("renameSync overwrite evicts the displaced destination's buffer", async () => { - // Open a buffer over an existing /dst, mutate it, then overwrite - // /dst via rename. The buffer's inode is gone from SQL after - // rename; release must not commit chunks against the dead row - // and must not leave a dangling cache entry. - await withProvider((p) => { - p.writeFileSync("/src.txt", "src bytes"); - p.writeFileSync("/dst.txt", "dst bytes"); - const dstInodeBefore = p.statSync("/dst.txt").ino; - p.openWriteBufferSync("/dst.txt"); - p.writeRangeSync("/dst.txt", Buffer.from("dirty"), 0); - - p.renameSync("/src.txt", "/dst.txt"); - - // The path now resolves to the renamed source's inode, not - // the displaced one. Release is a no-op on the now-gone - // displaced inode; the renamed file's bytes are unchanged. - expect(p.statSync("/dst.txt").ino).not.toBe(dstInodeBefore); - expect((p.readFileSync("/dst.txt") as Buffer).toString()).toBe("src bytes"); - expect(() => p.releaseWriteBufferSync("/dst.txt")).not.toThrow(); - expect((p.readFileSync("/dst.txt") as Buffer).toString()).toBe("src bytes"); - }); - }); - - it("unlinkSync drops the inode-keyed buffer when the last link disappears", async () => { - const { getWriteBuffer } = await import("./fs/writeBuffer.js"); - await withProvider((p) => { - p.writeFileSync("/a.txt", "hello"); - const inode = p.statSync("/a.txt").ino; - p.openWriteBufferSync("/a.txt"); - p.writeRangeSync("/a.txt", Buffer.from("WORLD"), 0); - // Buffer is staged in the inode-keyed cache. - expect(getWriteBuffer(p.db, inode)).toBeDefined(); - p.unlinkSync("/a.txt"); - // unlink removed the last link, so the inode row is gone and - // the buffer must not be cached against the dead inode. - expect(p.existsSync("/a.txt")).toBe(false); - expect(getWriteBuffer(p.db, inode)).toBeUndefined(); - }); - }); - - it("unlinkSync keeps the buffer alive when a hardlink remains", async () => { - const { getWriteBuffer } = await import("./fs/writeBuffer.js"); - await withProvider((p) => { - p.writeFileSync("/a.txt", "shared"); - p.linkSync("/a.txt", "/b.txt"); - const inode = p.statSync("/a.txt").ino; - p.openWriteBufferSync("/a.txt"); - p.writeRangeSync("/a.txt", Buffer.from("UPDATED"), 0); - p.unlinkSync("/a.txt"); - // /b.txt still references the inode; the buffer survives in - // the inode-keyed cache and a release through the remaining - // name commits the staged bytes. - expect(p.existsSync("/b.txt")).toBe(true); - expect(getWriteBuffer(p.db, inode)).toBeDefined(); - p.releaseWriteBufferSync("/b.txt"); - expect((p.readFileSync("/b.txt") as Buffer).toString()).toBe("UPDATED"); - }); - }); -}); - -describe("SQLiteWorkspaceProvider — cached vfs_nodes.size", () => { - function readSize(p: SQLiteWorkspaceProvider, name: string): number | undefined { - return p.db.one<{ size: number }>( - "SELECT size FROM vfs_nodes WHERE inode = (SELECT child_inode FROM vfs_dirents WHERE name = ?)", - name, - )?.size; - } - - it("writeFileSync stamps size on first write and on overwrite", async () => { - await withProvider((p) => { - p.writeFileSync("/sized.bin", Buffer.alloc(123, 0x41)); - expect(p.statSync("/sized.bin").size).toBe(123); - expect(readSize(p, "sized.bin")).toBe(123); - - p.writeFileSync("/sized.bin", Buffer.alloc(7, 0x42)); - expect(p.statSync("/sized.bin").size).toBe(7); - expect(readSize(p, "sized.bin")).toBe(7); - }); - }); - - it("writeRangeSync extends the cached size on growth", async () => { - await withProvider((p) => { - p.createFileSync("/range.bin", { mode: 0o644 }); - p.writeRangeSync("/range.bin", Buffer.from("hello"), 0); - expect(readSize(p, "range.bin")).toBe(5); - p.writeRangeSync("/range.bin", Buffer.from("!!"), 10); - expect(p.statSync("/range.bin").size).toBe(12); - expect(readSize(p, "range.bin")).toBe(12); - }); - }); - - it("truncateFileSync updates the cached size on grow and shrink", async () => { - await withProvider((p) => { - p.writeFileSync("/trunc.bin", Buffer.alloc(100, 0x55)); - expect(readSize(p, "trunc.bin")).toBe(100); - - p.truncateFileSync("/trunc.bin", 250); - expect(p.statSync("/trunc.bin").size).toBe(250); - expect(readSize(p, "trunc.bin")).toBe(250); - - p.truncateFileSync("/trunc.bin", 0); - expect(p.statSync("/trunc.bin").size).toBe(0); - expect(readSize(p, "trunc.bin")).toBe(0); - }); - }); - - it("buffered release stamps the cached size of the committed bytes", async () => { - await withProvider((p) => { - p.openWriteBufferForCreateSync("/buffered.bin", { mode: 0o644 }); - p.writeRangeSync("/buffered.bin", Buffer.from("buffered-write"), 0); - expect(readSize(p, "buffered.bin")).toBeUndefined(); - p.releaseWriteBufferSync("/buffered.bin"); - expect(p.statSync("/buffered.bin").size).toBe(14); - expect(readSize(p, "buffered.bin")).toBe(14); - }); - }); - - it("async writeFile stamps size from the buffered bytes", async () => { - await withProvider(async (p) => { - const payload = Buffer.from("async payload"); - await p.writeFile("/streamed.bin", payload); - expect(p.statSync("/streamed.bin").size).toBe(payload.byteLength); - expect(readSize(p, "streamed.bin")).toBe(payload.byteLength); - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/provider.ts b/spikes/349-dofs/vendor/dofs/src/provider.ts deleted file mode 100644 index 5adcafc8..00000000 --- a/spikes/349-dofs/vendor/dofs/src/provider.ts +++ /dev/null @@ -1,936 +0,0 @@ -// SQLiteWorkspaceProvider — a @platformatic/vfs VirtualProvider backed -// by the dofs SQLite store. -// -// Every method on VirtualProvider is declared. Methods we already have -// synchronous building blocks for delegate to the existing fs/ helpers; -// the rest throw ENOSYS so the gaps are visible at the call site. -// Subsequent commits fill in the stubs (file descriptors, positional -// I/O, truncate, symlinks, watch). - -import { createWorkspaceError } from "./errors.js"; -import { getBlobBytes } from "./fs/blobCache.js"; -import { link as linkImpl } from "./fs/link.js"; -import type { MkdirOptions } from "./fs/mkdir.js"; -import { mkdir as mkdirImpl } from "./fs/mkdir.js"; -import { readdir as readdirImpl } from "./fs/readdir.js"; -import { readRangeSync as readRangeSyncImpl } from "./fs/readFile.js"; -import { readlink as readlinkImpl } from "./fs/readlink.js"; -import { rename as renameImpl } from "./fs/rename.js"; -import { resolveInode } from "./fs/resolve.js"; -import { rm as rmImpl } from "./fs/rm.js"; -import { stat as statImpl } from "./fs/stat.js"; -import { symlink as symlinkImpl } from "./fs/symlink.js"; -import { - createWatchAsyncIterable, - createWatcher, - type WatchEvent, - type WatchHandle, - type WatchOptions, -} from "./fs/watch.js"; -import { - deleteWriteBuffer, - getPendingWriteBufferByPath, - getWriteBuffer, -} from "./fs/writeBuffer.js"; -import { - createFileSync as createFileSyncImpl, - flushPendingByPath, - openWriteBufferForCreateSync as openWriteBufferForCreateSyncImpl, - openWriteBufferSync as openWriteBufferSyncImpl, - releaseWriteBufferSync as releaseWriteBufferSyncImpl, - truncateFileSync as truncateFileSyncImpl, - type WriteFileRange, - writeFileRangesSync as writeFileRangesSyncImpl, - writeFileSync as writeFileSyncImpl, - writeRangeSync as writeRangeSyncImpl, -} from "./fs/writeFile.js"; -import { canonicalizePath } from "./path.js"; -import { incrementRev } from "./rev.js"; -import type { Database } from "./storage.js"; - -export interface SQLiteWorkspaceProviderOptions { - // Wall-clock source. Defaults to Date.now so production callers - // don't need to thread one through; tests pin it. - now?: () => number; - // Poll interval for watch() in milliseconds. Defaults to 100 ms - // to match node's fs.watch on most filesystems; tests can lower - // it to keep durations short. - watchIntervalMs?: number; -} - -interface VirtualStatsLike { - dev: number; - mode: number; - nlink: number; - uid: number; - gid: number; - rdev: number; - blksize: number; - ino: number; - size: number; - blocks: number; - atimeMs: number; - mtimeMs: number; - ctimeMs: number; - birthtimeMs: number; - atime: Date; - mtime: Date; - ctime: Date; - birthtime: Date; - isFile(): boolean; - isDirectory(): boolean; - isSymbolicLink(): boolean; - isBlockDevice(): boolean; - isCharacterDevice(): boolean; - isFIFO(): boolean; - isSocket(): boolean; -} - -interface VirtualDirentLike { - name: string; - parentPath: string; - path: string; - isFile(): boolean; - isDirectory(): boolean; - isSymbolicLink(): boolean; - isBlockDevice(): boolean; - isCharacterDevice(): boolean; - isFIFO(): boolean; - isSocket(): boolean; -} - -interface FdState { - path: string; - position: number; - readable: boolean; - writable: boolean; - // append mode pins every writeSync to current EOF rather than - // honouring an explicit position argument. - append: boolean; -} - -export class SQLiteWorkspaceProvider { - readonly db: Database; - readonly now: () => number; - - // Capability flags consulted by @platformatic/vfs callers. - readonly readonly = false; - readonly supportsSymlinks = true; - readonly supportsWatch = true; - - // Fd table. Start at 3 — 0/1/2 are reserved by convention even - // though we don't expose them — so consumers that pass them around - // can't accidentally collide with stdio mental models. - #fds = new Map(); - #nextFd = 3; - - readonly watchIntervalMs: number; - - constructor(db: Database, options: SQLiteWorkspaceProviderOptions = {}) { - this.db = db; - this.now = options.now ?? Date.now; - this.watchIntervalMs = options.watchIntervalMs ?? 100; - } - - // -- Essential primitives ------------------------------------------ - - open(path: string, flags?: string, mode?: number): Promise { - return Promise.resolve(this.openSync(path, flags, mode)); - } - - openSync(path: string, flags: string = "r", _mode?: number): number { - const { read, write, truncate, append, create, exclusive } = parseFlags(flags); - const existing = resolveInode(this.db, path); - - if (existing === null) { - if (!create) { - throw createWorkspaceError("ENOENT", `no such file: ${path}`, path); - } - writeFileSyncImpl(this.db, path, new Uint8Array(), {}, this.now); - } else { - if (existing.type !== "file") { - throw createWorkspaceError("EISDIR", `path is a directory: ${path}`, path); - } - if (exclusive) { - throw createWorkspaceError("EEXIST", `path exists: ${path}`, path); - } - if (truncate) { - writeFileSyncImpl(this.db, path, new Uint8Array(), {}, this.now); - } - } - - const stat = statImpl(this.db, path); - const fd = this.#nextFd++; - this.#fds.set(fd, { - path, - position: append ? stat.size : 0, - readable: read, - writable: write, - append, - }); - return fd; - } - - stat(path: string, options?: { bigint?: boolean }): Promise { - return Promise.resolve(this.statSync(path, options)); - } - - statSync(path: string, _options?: { bigint?: boolean }): VirtualStatsLike { - // statImpl resolves the path once (following symlinks) and returns - // the inode, so nlink comes from the same walk. A pending-create - // file reports inode 0, which yields nlink 1. - const s = statImpl(this.db, path); - return wrapStats({ - mode: s.mode, - size: s.size, - mtimeMs: s.mtime, - ino: s.inode, - isFile: s.isFile, - isDirectory: s.isDirectory, - isSymbolicLink: false, - nlink: linkCount(this.db, s.inode), - }); - } - - lstat(path: string, options?: { bigint?: boolean }): Promise { - return Promise.resolve(this.lstatSync(path, options)); - } - - lstatSync(path: string, _options?: { bigint?: boolean }): VirtualStatsLike { - const { path: canonical } = canonicalizePath(path); - const pending = getPendingWriteBufferByPath(this.db, canonical); - if (pending !== undefined && pending.pending !== undefined) { - return wrapStats({ - mode: pending.mode & 0o7777, - size: pending.size, - mtimeMs: pending.pending.mtime, - ino: 0, - isFile: true, - isDirectory: false, - isSymbolicLink: false, - nlink: 1, - }); - } - const node = resolveInode(this.db, path, { followSymlinks: false }); - if (node === null) { - throw createWorkspaceError("ENOENT", `no such path: ${path}`, path); - } - const isSymlink = node.type === "symlink"; - const size = isSymlink - ? (node.linkTarget ?? "").length - : node.type === "file" - ? fileSize(this.db, node.inode) - : 0; - return wrapStats({ - mode: node.mode, - size, - mtimeMs: node.mtime, - ino: node.inode, - isFile: node.type === "file", - isDirectory: node.type === "dir", - isSymbolicLink: isSymlink, - nlink: linkCount(this.db, node.inode), - }); - } - - readdir( - path: string, - options?: { withFileTypes?: boolean }, - ): Promise { - return Promise.resolve(this.readdirSync(path, options)); - } - - readdirSync(path: string, options?: { withFileTypes?: boolean }): string[] | VirtualDirentLike[] { - const entries = readdirImpl(this.db, path); - if (options?.withFileTypes === true) { - return entries.map((entry) => wrapDirent(entry)); - } - return entries.map((entry) => entry.name); - } - - mkdir(path: string, options?: MkdirOptions): Promise { - return Promise.resolve(this.mkdirSync(path, options)); - } - - mkdirSync(path: string, options?: MkdirOptions): string | undefined { - mkdirImpl(this.db, path, options ?? {}, this.now); - return undefined; - } - - rmdir(path: string): Promise { - this.rmdirSync(path); - return Promise.resolve(); - } - - rmdirSync(path: string): void { - rmImpl(this.db, path, {}); - } - - unlink(path: string): Promise { - this.unlinkSync(path); - return Promise.resolve(); - } - - unlinkSync(path: string): void { - // If a buffered create is still pending for this path, commit - // it first so rm sees a real inode to unlink (and so the - // resulting GC sees the orphaned blob, matching the non-buffered - // shape). The buffer's open handles continue to address bytes - // through the inode-keyed cache. - flushPendingByPath(this.db, path, this.now); - // Capture the target inode before rm runs so we can evict its - // write-buffer cache entry if rm removed the last link. Without - // this, a release-after-unlink leaves the buffer dangling on a - // dead inode and the eventual commit silently affects no rows. - const target = resolveInode(this.db, path, { followSymlinks: false }); - rmImpl(this.db, path, {}); - if (target !== null) { - const stillAlive = this.db.scalar( - "SELECT inode FROM vfs_nodes WHERE inode = ?", - target.inode, - ); - if (stillAlive === undefined) { - deleteWriteBuffer(this.db, target.inode); - } - } - } - - link(existingPath: string, newPath: string): Promise { - this.linkSync(existingPath, newPath); - return Promise.resolve(); - } - - linkSync(existingPath: string, newPath: string): void { - // Commit a still-pending source before adding the second dirent, - // otherwise link has nothing real to point at. Also commit a - // still-pending destination: link's existence check looks at - // dirents, so a pending buffer at newPath wouldn't trip it, and - // the eventual release on that pending buffer would re-check the - // dirent in commitPendingBuffer, throw EEXIST, drop the entry, - // and silently lose the user's bytes. - flushPendingByPath(this.db, existingPath, this.now); - flushPendingByPath(this.db, newPath, this.now); - linkImpl(this.db, existingPath, newPath); - } - - rename(oldPath: string, newPath: string): Promise { - this.renameSync(oldPath, newPath); - return Promise.resolve(); - } - - renameSync(oldPath: string, newPath: string): void { - // Commit any still-pending creates at either end before the rename - // touches dirents: the source needs a real inode to move, and a - // pending buffer at the destination would otherwise slip past - // rename's dirent-based existence check and lose bytes on release. - flushPendingByPath(this.db, oldPath, this.now); - flushPendingByPath(this.db, newPath, this.now); - // Capture the destination inode before the rename so we can evict - // its write-buffer cache entry if the rename displaced and reaped - // it. Without this, a release on an open destination would commit - // chunks against a dead inode (0-row UPDATE, silent data loss). - const displaced = resolveInode(this.db, newPath, { followSymlinks: false }); - renameImpl(this.db, oldPath, newPath); - if (displaced !== null) { - const stillAlive = this.db.scalar( - "SELECT inode FROM vfs_nodes WHERE inode = ?", - displaced.inode, - ); - if (stillAlive === undefined) { - deleteWriteBuffer(this.db, displaced.inode); - } - } - } - - // -- Default implementations --------------------------------------- - - readFile( - path: string, - options?: BufferEncoding | { encoding?: BufferEncoding | null } | null, - ): Promise { - return Promise.resolve(this.readFileSync(path, options)); - } - - readFileSync( - path: string, - options?: BufferEncoding | { encoding?: BufferEncoding | null } | null, - ): Buffer | string { - const encoding = typeof options === "string" ? options : options?.encoding; - const { path: canonical } = canonicalizePath(path); - const pending = getPendingWriteBufferByPath(this.db, canonical); - if (pending !== undefined) { - const snapshot = Buffer.alloc(pending.size); - snapshot.set(pending.buf.subarray(0, pending.size)); - return encoding ? snapshot.toString(encoding) : snapshot; - } - const node = resolveInode(this.db, path); - if (node === null) { - throw createWorkspaceError("ENOENT", `no such file: ${path}`, path); - } - if (node.type !== "file") { - throw createWorkspaceError("EISDIR", `path is a directory: ${path}`, path); - } - // While a buffer is open for this inode it owns the latest - // bytes; serve from it instead of the chunk store. - const buffered = getWriteBuffer(this.db, node.inode); - if (buffered?.dirty) { - const snapshot = Buffer.alloc(buffered.size); - snapshot.set(buffered.buf.subarray(0, buffered.size)); - return encoding ? snapshot.toString(encoding) : snapshot; - } - const chunks = this.db.all<{ hash: Uint8Array; size: number }>( - "SELECT hash, size FROM vfs_chunks WHERE inode = ? ORDER BY idx", - node.inode, - ); - let total = 0; - for (const c of chunks) total += c.size; - const out = Buffer.alloc(total); - let offset = 0; - for (const chunk of chunks) { - const bytes = getBlobBytes(this.db, chunk.hash); - if (bytes === undefined) { - throw createWorkspaceError("EIO", `missing blob bytes for ${path}`, path); - } - out.set(bytes, offset); - offset += bytes.byteLength; - } - return encoding ? out.toString(encoding) : out; - } - - writeFile( - path: string, - data: string | Buffer, - options?: { encoding?: BufferEncoding; mode?: number } | BufferEncoding, - ): Promise { - this.writeFileSync(path, data, options); - return Promise.resolve(); - } - - writeFileSync( - path: string, - data: string | Buffer, - options?: { encoding?: BufferEncoding; mode?: number } | BufferEncoding, - ): void { - const mode = typeof options === "string" ? undefined : options?.mode; - const bytes = - typeof data === "string" - ? new TextEncoder().encode(data) - : new Uint8Array(data.buffer, data.byteOffset, data.byteLength); - writeFileSyncImpl(this.db, path, bytes, { mode }, this.now); - } - - writeFileRangesSync( - path: string, - data: string | Buffer, - ranges: WriteFileRange[], - options?: { encoding?: BufferEncoding; mode?: number } | BufferEncoding, - ): void { - const mode = typeof options === "string" ? undefined : options?.mode; - const bytes = - typeof data === "string" - ? new TextEncoder().encode(data) - : new Uint8Array(data.buffer, data.byteOffset, data.byteLength); - writeFileRangesSyncImpl(this.db, path, bytes, ranges, { mode }, this.now); - } - - createFileSync(path: string, options?: { mode?: number }): void { - createFileSyncImpl(this.db, path, { mode: options?.mode }, this.now); - } - - writeRangeSync( - path: string, - data: string | Buffer | Uint8Array, - offset: number, - options?: { encoding?: BufferEncoding; mode?: number } | BufferEncoding, - ): number { - const mode = typeof options === "string" ? undefined : options?.mode; - const bytes = - typeof data === "string" - ? new TextEncoder().encode(data) - : new Uint8Array(data.buffer, data.byteOffset, data.byteLength); - return writeRangeSyncImpl(this.db, path, bytes, offset, { mode }, this.now); - } - - truncateFileSync(path: string, len: number): void { - truncateFileSyncImpl(this.db, path, len, this.now); - } - - openWriteBufferSync(path: string): void { - openWriteBufferSyncImpl(this.db, path); - } - - openWriteBufferForCreateSync(path: string, options?: { mode?: number }): void { - openWriteBufferForCreateSyncImpl(this.db, path, { mode: options?.mode }, this.now); - } - - releaseWriteBufferSync(path: string): void { - releaseWriteBufferSyncImpl(this.db, path, this.now); - } - - chmodSync(path: string, mode: number): void { - const { path: canonical } = canonicalizePath(path); - const pending = getPendingWriteBufferByPath(this.db, canonical); - if (pending !== undefined) { - // Pending-create files don't have a row yet; stash the mode on - // the buffer so the eventual INSERT picks it up. - pending.mode = mode & 0o7777; - return; - } - const node = resolveInode(this.db, path, { followSymlinks: false }); - if (node === null) { - throw createWorkspaceError("ENOENT", `no such path: ${path}`, path); - } - const rev = incrementRev(this.db); - this.db.run( - "UPDATE vfs_nodes SET mode = ?, rev = ? WHERE inode = ?", - mode & 0o7777, - rev, - node.inode, - ); - } - - appendFile( - _path: string, - _data: string | Buffer, - _options?: { encoding?: BufferEncoding; mode?: number } | BufferEncoding, - ): Promise { - return Promise.reject(notImplemented("appendFile")); - } - - appendFileSync( - _path: string, - _data: string | Buffer, - _options?: { encoding?: BufferEncoding; mode?: number } | BufferEncoding, - ): void { - throw notImplemented("appendFileSync"); - } - - exists(path: string): Promise { - return Promise.resolve(this.existsSync(path)); - } - - existsSync(path: string): boolean { - try { - const { path: canonical } = canonicalizePath(path); - if (getPendingWriteBufferByPath(this.db, canonical) !== undefined) return true; - return resolveInode(this.db, path) !== null; - } catch { - return false; - } - } - - copyFile(_src: string, _dest: string, _mode?: number): Promise { - return Promise.reject(notImplemented("copyFile")); - } - - copyFileSync(_src: string, _dest: string, _mode?: number): void { - throw notImplemented("copyFileSync"); - } - - internalModuleStat(_path: string): number { - // Used by node:vfs module-resolution hooks. The computerd driver doesn't - // need it; if this provider is ever mounted via `vfs.mount()` we'll - // need to return 0 for files, 1 for dirs, -1 for not-found. - throw notImplemented("internalModuleStat"); - } - - realpath(path: string, _options?: { encoding?: BufferEncoding }): Promise { - return Promise.resolve(this.realpathSync(path)); - } - - realpathSync(path: string, _options?: { encoding?: BufferEncoding }): string { - const { path: canonical } = canonicalizePath(path); - if (resolveInode(this.db, canonical) === null) { - throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); - } - return canonical; - } - - access(path: string, _mode?: number): Promise { - this.accessSync(path); - return Promise.resolve(); - } - - accessSync(path: string, _mode?: number): void { - if (resolveInode(this.db, path) === null) { - throw createWorkspaceError("ENOENT", `no such path: ${path}`, path); - } - } - - // -- File descriptors ---------------------------------------------- - - closeSync(fd: number): void { - if (!this.#fds.delete(fd)) { - throw createWorkspaceError("EBADF", `unknown fd ${fd}`); - } - } - - readSync( - fd: number, - buffer: Buffer | Uint8Array, - offset: number, - length: number, - position: number | null, - ): number { - const state = this.#fdOrThrow(fd); - if (!state.readable) { - throw createWorkspaceError("EBADF", `fd ${fd} is not readable`); - } - const startAt = position ?? state.position; - const slice = readRangeSyncImpl(this.db, state.path, startAt, length); - const view = - buffer instanceof Buffer - ? buffer - : Buffer.from(buffer.buffer, buffer.byteOffset, buffer.byteLength); - view.set(slice, offset); - if (position === null || position === undefined) { - state.position = startAt + slice.byteLength; - } - return slice.byteLength; - } - - readRangeSync(path: string, offset: number, length: number): Buffer { - const slice = readRangeSyncImpl(this.db, path, offset, length); - return Buffer.from(slice.buffer, slice.byteOffset, slice.byteLength); - } - - writeSync( - fd: number, - buffer: Buffer | Uint8Array, - offset: number = 0, - length: number = buffer.byteLength - offset, - position: number | null = null, - ): number { - const state = this.#fdOrThrow(fd); - if (!state.writable) { - throw createWorkspaceError("EBADF", `fd ${fd} is not writable`); - } - // Append needs the current EOF, so stat only then. A non-append - // write of >0 bytes doesn't need it: writeRangeSyncImpl resolves the - // path and raises ENOENT/EISDIR. A zero-length write short-circuits - // before that resolve, so keep an explicit existence check for it. - let startAt: number; - if (state.append) { - startAt = this.statSync(state.path).size; - } else { - if (length === 0) { - this.statSync(state.path); - } - startAt = position ?? state.position; - } - const view = - buffer instanceof Buffer - ? new Uint8Array(buffer.buffer, buffer.byteOffset + offset, length) - : new Uint8Array(buffer.buffer, buffer.byteOffset + offset, length); - writeRangeSyncImpl(this.db, state.path, view, startAt, {}, this.now); - if (position === null || position === undefined) { - state.position = startAt + length; - } - return length; - } - - fstatSync(fd: number, _options?: { bigint?: boolean }): VirtualStatsLike { - const state = this.#fdOrThrow(fd); - return this.statSync(state.path); - } - - truncateSync(path: string, len: number): void { - const node = resolveInode(this.db, path); - if (node === null) { - throw createWorkspaceError("ENOENT", `no such path: ${path}`, path); - } - if (node.type !== "file") { - throw createWorkspaceError("EISDIR", `path is a directory: ${path}`, path); - } - truncateFileSyncImpl(this.db, path, len, this.now); - } - - ftruncateSync(fd: number, len: number): void { - const state = this.#fdOrThrow(fd); - this.truncateSync(state.path, len); - } - - #fdOrThrow(fd: number): FdState { - const state = this.#fds.get(fd); - if (state === undefined) { - throw createWorkspaceError("EBADF", `unknown fd ${fd}`); - } - return state; - } - - // -- Symlinks ------------------------------------------------------ - - readlink(path: string, _options?: { encoding?: BufferEncoding }): Promise { - return Promise.resolve(this.readlinkSync(path)); - } - - readlinkSync(path: string, _options?: { encoding?: BufferEncoding }): string { - return readlinkImpl(this.db, path); - } - - symlink(target: string, path: string, _type?: string): Promise { - this.symlinkSync(target, path); - return Promise.resolve(); - } - - symlinkSync(target: string, path: string, _type?: string): void { - symlinkImpl(this.db, target, path, this.now); - } - - // -- Watch ---------------------------------------------------------- - // - // The watcher polls vfs_meta.rev on a timer. Each tick - // coalesceChanges yields every path touched since the last - // observed rev; we filter by the watched directory (and - // recursive flag) and emit one 'change' event per path. Cheap - // because coalesceChanges is one indexed range scan on - // vfs_nodes.rev plus a path walk per touched inode. - // - // Event types follow node's fs.watch convention: - // - 'rename' for deletes (path went away) - // - 'change' for everything else (file/dir/symlink mutation) - // We don't distinguish first-time creation from in-place edit - // — the cost is a per-watcher state map that's bigger than - // the signal is worth. Callers that need rename-vs-change - // semantics can stat the path themselves. - - watch(path: string, options: WatchOptions = {}): WatchHandle { - return createWatcher(this.db, path, options, this.watchIntervalMs); - } - - watchAsync(path: string, options: WatchOptions = {}): AsyncIterable { - return createWatchAsyncIterable(this.watch(path, options)); - } - - // watchFile / unwatchFile fire on stat changes at a single path - // (not the directory under it). Different semantics from watch(); - // editors typically use watch() instead. Leave as ENOSYS until a - // real call site shows up. - watchFile( - _path: string, - _options?: unknown, - _listener?: (curr: VirtualStatsLike, prev: VirtualStatsLike) => void, - ): unknown { - throw notImplemented("watchFile"); - } - - unwatchFile( - _path: string, - _listener?: (curr: VirtualStatsLike, prev: VirtualStatsLike) => void, - ): void { - throw notImplemented("unwatchFile"); - } -} - -function notImplemented(method: string) { - return createWorkspaceError("ENOSYS", `SQLiteWorkspaceProvider.${method} is not implemented yet`); -} - -// -- VirtualStats / VirtualDirent shim ------------------------------ -// -// @platformatic/vfs callers (and FUSE drivers built on top) consult -// the full Node-style stat shape. Most fields don't map onto our -// content-addressed store, so they get sensible constants. The fields -// that do map — mode, size, mtime, ino — are populated for real. - -interface StatsInputs { - mode: number; - size: number; - mtimeMs: number; - ino: number; - isFile: boolean; - isDirectory: boolean; - isSymbolicLink: boolean; - nlink: number; -} - -// POSIX mode-bit constants. Linux FUSE rejects a stat whose mode -// has no S_IF* bits set with EIO — it can't decide whether -// the inode is a regular file, a directory, or a symlink. -const S_IFREG = 0o100000; -const S_IFDIR = 0o040000; -const S_IFLNK = 0o120000; - -function fileTypeBits(input: StatsInputs): number { - if (input.isDirectory) return S_IFDIR; - if (input.isSymbolicLink) return S_IFLNK; - if (input.isFile) return S_IFREG; - return 0; -} - -function linkCount(db: Database, inode: number): number { - const count = db.scalar("SELECT COUNT(*) FROM vfs_dirents WHERE child_inode = ?", inode); - return Math.max(1, count ?? 0); -} - -function fileSize(db: Database, inode: number): number { - const buffered = getWriteBuffer(db, inode); - if (buffered?.dirty) { - return buffered.size; - } - return db.scalar("SELECT size FROM vfs_nodes WHERE inode = ?", inode) ?? 0; -} - -function wrapStats(input: StatsInputs): VirtualStatsLike { - const mtime = new Date(input.mtimeMs); - return { - dev: 0, - mode: (input.mode & 0o7777) | fileTypeBits(input), - nlink: input.nlink, - uid: 0, - gid: 0, - rdev: 0, - blksize: 4096, - ino: input.ino, - size: input.size, - blocks: Math.ceil(input.size / 512), - atimeMs: input.mtimeMs, - mtimeMs: input.mtimeMs, - ctimeMs: input.mtimeMs, - birthtimeMs: input.mtimeMs, - atime: mtime, - mtime, - ctime: mtime, - birthtime: mtime, - isFile: () => input.isFile, - isDirectory: () => input.isDirectory, - isSymbolicLink: () => input.isSymbolicLink, - isBlockDevice: () => false, - isCharacterDevice: () => false, - isFIFO: () => false, - isSocket: () => false, - }; -} - -interface DirentInput { - name: string; - parentPath: string; - isFile: boolean; - isDirectory: boolean; -} - -function wrapDirent(input: DirentInput): VirtualDirentLike { - const fullPath = - input.parentPath === "/" ? `/${input.name}` : `${input.parentPath}/${input.name}`; - return { - name: input.name, - parentPath: input.parentPath, - path: fullPath, - isFile: () => input.isFile, - isDirectory: () => input.isDirectory, - isSymbolicLink: () => false, - isBlockDevice: () => false, - isCharacterDevice: () => false, - isFIFO: () => false, - isSocket: () => false, - }; -} - -interface ParsedFlags { - read: boolean; - write: boolean; - create: boolean; - truncate: boolean; - append: boolean; - exclusive: boolean; -} - -// Translate Node's fs flag strings into the boolean flag set the fd -// table uses. Mirrors the documented behaviour of fs.open(flags) at -// https://nodejs.org/api/fs.html#file-system-flags. -function parseFlags(flags: string): ParsedFlags { - switch (flags) { - case "r": - return { - read: true, - write: false, - create: false, - truncate: false, - append: false, - exclusive: false, - }; - case "r+": - return { - read: true, - write: true, - create: false, - truncate: false, - append: false, - exclusive: false, - }; - case "w": - return { - read: false, - write: true, - create: true, - truncate: true, - append: false, - exclusive: false, - }; - case "w+": - return { - read: true, - write: true, - create: true, - truncate: true, - append: false, - exclusive: false, - }; - case "wx": - return { - read: false, - write: true, - create: true, - truncate: false, - append: false, - exclusive: true, - }; - case "wx+": - return { - read: true, - write: true, - create: true, - truncate: false, - append: false, - exclusive: true, - }; - case "a": - return { - read: false, - write: true, - create: true, - truncate: false, - append: true, - exclusive: false, - }; - case "a+": - return { - read: true, - write: true, - create: true, - truncate: false, - append: true, - exclusive: false, - }; - case "ax": - return { - read: false, - write: true, - create: true, - truncate: false, - append: true, - exclusive: true, - }; - case "ax+": - return { - read: true, - write: true, - create: true, - truncate: false, - append: true, - exclusive: true, - }; - default: - throw createWorkspaceError("EINVAL", `unsupported fs flag: ${flags}`); - } -} diff --git a/spikes/349-dofs/vendor/dofs/src/provider.watch.test.ts b/spikes/349-dofs/vendor/dofs/src/provider.watch.test.ts deleted file mode 100644 index 6f75e484..00000000 --- a/spikes/349-dofs/vendor/dofs/src/provider.watch.test.ts +++ /dev/null @@ -1,158 +0,0 @@ -import type { EventEmitter } from "node:events"; -import { describe, expect, it } from "vitest"; -import { withDB } from "./fs/with-db.js"; -import { SQLiteWorkspaceProvider } from "./provider.js"; - -interface WatchEmitter extends EventEmitter { - close(): void; -} - -interface WatchEvent { - eventType: "rename" | "change"; - filename: string; -} - -async function withProvider(fn: (p: SQLiteWorkspaceProvider) => T | Promise): Promise { - return withDB((db) => - fn(new SQLiteWorkspaceProvider(db, { now: () => 1000, watchIntervalMs: 25 })), - ); -} - -async function nextEvents( - watcher: WatchEmitter, - count: number, - timeoutMs = 1500, -): Promise { - const events: WatchEvent[] = []; - const done = new Promise((resolve, reject) => { - const onChange = (eventType: "rename" | "change", filename: string) => { - events.push({ eventType, filename }); - if (events.length >= count) { - watcher.off("change", onChange); - resolve(); - } - }; - watcher.on("change", onChange); - setTimeout(() => { - watcher.off("change", onChange); - reject(new Error(`timed out waiting for ${count} events; got ${events.length}`)); - }, timeoutMs); - }); - await done; - return events; -} - -describe("SQLiteWorkspaceProvider — watch", () => { - it("supportsWatch is true", async () => { - await withProvider((p) => { - expect(p.supportsWatch).toBe(true); - }); - }); - - it("watch(dir) fires for a write to a direct child", async () => { - await withProvider(async (p) => { - p.mkdirSync("/d", {}); - const w = p.watch("/d", {}) as WatchEmitter; - try { - // Write happens after the watcher captures the baseline rev. - // Give the poll loop one tick to record the initial position. - await new Promise((r) => setTimeout(r, 30)); - p.writeFileSync("/d/a.txt", "hello"); - const events = await nextEvents(w, 1); - expect(events[0]).toMatchObject({ filename: "a.txt" }); - } finally { - w.close(); - } - }); - }); - - it("watch(dir, { recursive: true }) fires for nested writes", async () => { - await withProvider(async (p) => { - p.mkdirSync("/d", {}); - p.mkdirSync("/d/sub", {}); - const w = p.watch("/d", { recursive: true }) as WatchEmitter; - try { - await new Promise((r) => setTimeout(r, 30)); - p.writeFileSync("/d/sub/deep.txt", "yo"); - const events = await nextEvents(w, 1); - expect(events[0].filename).toBe("sub/deep.txt"); - } finally { - w.close(); - } - }); - }); - - it("watch(dir) does not fire for unrelated writes", async () => { - await withProvider(async (p) => { - p.mkdirSync("/watched", {}); - p.mkdirSync("/other", {}); - const w = p.watch("/watched", {}) as WatchEmitter; - const seen: string[] = []; - const onChange = (_t: string, name: string) => seen.push(name); - w.on("change", onChange); - try { - await new Promise((r) => setTimeout(r, 30)); - p.writeFileSync("/other/x.txt", "no"); - await new Promise((r) => setTimeout(r, 100)); - expect(seen).toEqual([]); - } finally { - w.off("change", onChange); - w.close(); - } - }); - }); - - it("watch fires rename for a delete and change for a write", async () => { - await withProvider(async (p) => { - p.mkdirSync("/d", {}); - p.writeFileSync("/d/a.txt", "first"); - const w = p.watch("/d", {}) as WatchEmitter; - try { - await new Promise((r) => setTimeout(r, 30)); - p.writeFileSync("/d/a.txt", "second"); - const first = await nextEvents(w, 1); - expect(first[0].eventType).toBe("change"); - // Separate tick so coalescing doesn't collapse the - // write and the delete into a single delete entry. - await new Promise((r) => setTimeout(r, 50)); - p.unlinkSync("/d/a.txt"); - const second = await nextEvents(w, 1); - expect(second[0].eventType).toBe("rename"); - } finally { - w.close(); - } - }); - }); - - it("watch.close() stops the poll loop", async () => { - await withProvider(async (p) => { - p.mkdirSync("/d", {}); - const w = p.watch("/d", {}) as WatchEmitter; - w.close(); - const seen: string[] = []; - w.on("change", (_t, name) => seen.push(name)); - await new Promise((r) => setTimeout(r, 50)); - p.writeFileSync("/d/a.txt", "noop"); - await new Promise((r) => setTimeout(r, 100)); - expect(seen).toEqual([]); - }); - }); - - it("watchAsync yields events via for-await", async () => { - await withProvider(async (p) => { - p.mkdirSync("/d", {}); - const it = p.watchAsync("/d", {}) as AsyncIterable & { - return(): Promise; - }; - const iter = it[Symbol.asyncIterator]() as AsyncIterator; - try { - await new Promise((r) => setTimeout(r, 30)); - p.writeFileSync("/d/a.txt", "x"); - const { value } = await iter.next(); - expect(value).toMatchObject({ filename: "a.txt" }); - } finally { - await iter.return?.(undefined); - } - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/rev.test.ts b/spikes/349-dofs/vendor/dofs/src/rev.test.ts deleted file mode 100644 index d24bdfe6..00000000 --- a/spikes/349-dofs/vendor/dofs/src/rev.test.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { withDB } from "./fs/with-db.js"; -import { incrementRev } from "./rev.js"; - -describe("incrementRev", () => { - it("returns the new rev value and persists it to vfs_meta", async () => { - await withDB( - (db) => { - // initializeSchema seeds rev = 1. - const next = incrementRev(db); - expect(next).toBe(2); - - const stored = db.scalar("SELECT v FROM vfs_meta WHERE k = ?", "rev"); - expect(stored).toBe(2); - }, - { now: () => 0 }, - ); - }); - - it("issues monotonically increasing revs inside a single transaction", async () => { - await withDB( - (db) => { - let a: number | undefined; - let b: number | undefined; - db.transactionSync(() => { - a = incrementRev(db); - b = incrementRev(db); - }); - expect(a).toBe(2); - expect(b).toBe(3); - const stored = db.scalar("SELECT v FROM vfs_meta WHERE k = ?", "rev"); - expect(stored).toBe(3); - }, - { now: () => 0 }, - ); - }); - - it("rolls back if the surrounding transaction aborts", async () => { - await withDB( - (db) => { - expect(() => { - db.transactionSync(() => { - incrementRev(db); - throw new Error("abort"); - }); - }).toThrow("abort"); - const stored = db.scalar("SELECT v FROM vfs_meta WHERE k = ?", "rev"); - expect(stored).toBe(1); - }, - { now: () => 0 }, - ); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/rev.ts b/spikes/349-dofs/vendor/dofs/src/rev.ts deleted file mode 100644 index df7f6d3c..00000000 --- a/spikes/349-dofs/vendor/dofs/src/rev.ts +++ /dev/null @@ -1,21 +0,0 @@ -import type { Database } from "./storage.js"; - -// Atomic monotonic rev counter. Every FS mutation (mkdir, writeFile, -// rm, ...) calls incrementRev once per transaction and stamps the returned -// value into vfs_nodes.rev. The sync layer reads vfs_meta.rev as -// currentRev and consumes vfs_changes.rev for tombstones. -// -// Must be called inside a transactionSync — the UPDATE and SELECT -// otherwise race with concurrent mutations. The DO single-writer model -// makes that unlikely in practice, but the contract is "wrap me". -export function incrementRev(db: Database): number { - // RETURNING folds the read into the same statement so each mutation - // pays one round-trip instead of two. SQLite has supported it since - // 3.35; both node:sqlite and Cloudflare DO SqlStorage are on newer - // versions. - const row = db.one<{ v: number }>("UPDATE vfs_meta SET v = v + 1 WHERE k = 'rev' RETURNING v"); - if (row === undefined) { - throw new Error("vfs_meta.rev row missing; was initializeSchema run?"); - } - return row.v; -} diff --git a/spikes/349-dofs/vendor/dofs/src/schema/core.ts b/spikes/349-dofs/vendor/dofs/src/schema/core.ts deleted file mode 100644 index bc77dd62..00000000 --- a/spikes/349-dofs/vendor/dofs/src/schema/core.ts +++ /dev/null @@ -1,81 +0,0 @@ -// Filesystem-side tables. These hold the inode graph and the -// content-addressed blob store. See docs/03_filesystem_schema.md. - -// Bumped to 2 when `_vfs_mounts.mode` landed (read-only mount -// enforcement at the data layer). Bumped to 3 when `vfs_nodes` -// gained a cached `size` column so stat() doesn't have to SUM -// chunks on every call. Bumped to 4 when `_vfs_watermark` gained -// a `backend` column so a single workspace can host more than -// one backend with independent sync cursors. Bumped to 5 when -// `vfs_dirents` and `vfs_chunks` became WITHOUT ROWID: their -// composite-PK lookups now read straight from the PK b-tree leaf -// with no rowid indirection, and `child_inode` lives in the -// dirents leaf so the (parent, name) resolve read is covering -// (no separate index needed). See `schema/migrations.ts` for the -// migration list; `sync.ts` carries the fresh-install DDL. -export const SCHEMA_VERSION = 5; -export const ROOT_INODE = 1; - -export const CORE_STATEMENTS = [ - `CREATE TABLE IF NOT EXISTS vfs_meta ( - k TEXT PRIMARY KEY, - v INTEGER NOT NULL - )`, - `CREATE TABLE IF NOT EXISTS vfs_nodes ( - inode INTEGER PRIMARY KEY AUTOINCREMENT, - type TEXT NOT NULL CHECK(type IN ('file','dir','symlink')), - mode INTEGER NOT NULL DEFAULT 493, - mtime INTEGER NOT NULL, - rev INTEGER NOT NULL DEFAULT 0, - mount_root TEXT, - stub_size INTEGER, - manifest_hash BLOB, - link_target TEXT, - size INTEGER NOT NULL DEFAULT 0 - )`, - // WITHOUT ROWID: the row lives in the (parent_inode, name) PK - // b-tree leaf, so resolving a path segment reads child_inode - // directly from the leaf — no autoindex -> rowid hop, and no - // separate covering index. Legal here because the PK is composite - // and the table has no AUTOINCREMENT. Existing databases are - // rebuilt by the v4 -> v5 migration in schema/migrations.ts; keep - // this DDL and that migrator's CREATE in lockstep. - `CREATE TABLE IF NOT EXISTS vfs_dirents ( - parent_inode INTEGER NOT NULL, - name TEXT NOT NULL, - child_inode INTEGER NOT NULL, - PRIMARY KEY (parent_inode, name) - ) WITHOUT ROWID`, - `CREATE INDEX IF NOT EXISTS vfs_dirents_by_child ON vfs_dirents(child_inode)`, - `CREATE INDEX IF NOT EXISTS vfs_nodes_by_rev ON vfs_nodes(rev)`, - // gc/manifests checks every manifest row against vfs_nodes via a - // correlated NOT EXISTS (manifest_hash = ?). Without this index - // gc full-scans vfs_nodes per candidate manifest — O(N×M). - // Partial because the column is null on every dir and symlink - // node, and on files until they get their first content write. - `CREATE INDEX IF NOT EXISTS vfs_nodes_by_manifest_hash - ON vfs_nodes(manifest_hash) WHERE manifest_hash IS NOT NULL`, - `CREATE TABLE IF NOT EXISTS vfs_blobs ( - hash BLOB PRIMARY KEY, - size INTEGER NOT NULL, - last_seen INTEGER NOT NULL - )`, - `CREATE TABLE IF NOT EXISTS vfs_blob_bytes ( - hash BLOB PRIMARY KEY REFERENCES vfs_blobs(hash) ON DELETE CASCADE, - bytes BLOB NOT NULL - )`, - // WITHOUT ROWID: clustered on (inode, idx) so a file's chunks are - // stored and scanned in index order straight from the PK leaf. - // Legal here — composite PK, no AUTOINCREMENT. The bytes live in - // vfs_blob_bytes (content-addressed), so these rows stay small, - // which is what WITHOUT ROWID wants. Rebuilt for existing DBs by - // the v4 -> v5 migration; keep in lockstep with that migrator. - `CREATE TABLE IF NOT EXISTS vfs_chunks ( - inode INTEGER NOT NULL, - idx INTEGER NOT NULL, - hash BLOB NOT NULL, - size INTEGER NOT NULL, - PRIMARY KEY (inode, idx) - ) WITHOUT ROWID`, - `CREATE INDEX IF NOT EXISTS vfs_chunks_by_hash ON vfs_chunks(hash)`, -] as const; diff --git a/spikes/349-dofs/vendor/dofs/src/schema/index.test.ts b/spikes/349-dofs/vendor/dofs/src/schema/index.test.ts deleted file mode 100644 index f07b597e..00000000 --- a/spikes/349-dofs/vendor/dofs/src/schema/index.test.ts +++ /dev/null @@ -1,457 +0,0 @@ -import { SQLiteTestStorage } from "@cloudflare/dofs/testing"; -import { describe, expect, it } from "vitest"; - -import { Database } from "../storage.js"; -import { RecordingStorage } from "../testing-recording.js"; -import { SCHEMA_VERSION } from "./core.js"; -import { initializeSchema } from "./index.js"; - -describe("initializeSchema", () => { - it("lazily initializes the documented schema on first use", () => { - const storage = new RecordingStorage(); - const db = new Database(storage); - - initializeSchema(db, () => 1234); - - const executed = storage.statements.map((statement) => statement.query); - expect(executed).toEqual( - expect.arrayContaining([ - expect.stringContaining("CREATE TABLE IF NOT EXISTS vfs_meta"), - expect.stringContaining("CREATE TABLE IF NOT EXISTS vfs_nodes"), - expect.stringContaining("CREATE TABLE IF NOT EXISTS vfs_dirents"), - expect.stringContaining("CREATE TABLE IF NOT EXISTS vfs_blobs"), - expect.stringContaining("CREATE TABLE IF NOT EXISTS vfs_blob_bytes"), - expect.stringContaining("CREATE TABLE IF NOT EXISTS vfs_chunks"), - expect.stringContaining("CREATE TABLE IF NOT EXISTS vfs_manifests"), - expect.stringContaining("CREATE TABLE IF NOT EXISTS vfs_changes"), - expect.stringContaining("CREATE TABLE IF NOT EXISTS _vfs_watermark"), - expect.stringContaining("CREATE TABLE IF NOT EXISTS _vfs_mounts"), - ]), - ); - expect(storage.statements).toContainEqual( - expect.objectContaining({ - query: expect.stringContaining("INSERT OR IGNORE INTO vfs_nodes"), - bindings: [1, 493, 1234], - }), - ); - }); - - it("rejects a newer on-disk schema version", () => { - const storage = new RecordingStorage({ schemaVersion: 999 }); - const db = new Database(storage); - - expect(() => initializeSchema(db, () => 0)).toThrow( - /Unsupported workspace filesystem schema version 999/, - ); - }); - - it("stamps the current SCHEMA_VERSION in vfs_meta on a fresh DB", () => { - const storage = new SQLiteTestStorage(); - const db = new Database(storage); - - initializeSchema(db, () => 0); - - const row = db.one<{ v: number }>("SELECT v FROM vfs_meta WHERE k = ?", "schema_version"); - expect(row?.v).toBe(SCHEMA_VERSION); - }); - - it("creates _vfs_mounts.mode on a fresh DB with the default and CHECK", () => { - const storage = new SQLiteTestStorage(); - const db = new Database(storage); - - initializeSchema(db, () => 0); - - // Defaults to read-only when the column is omitted. - db.run("INSERT INTO _vfs_mounts (root, kind) VALUES (?, ?)", "/m1", "r2"); - const row = db.one<{ mode: string }>("SELECT mode FROM _vfs_mounts WHERE root = ?", "/m1"); - expect(row?.mode).toBe("read-only"); - - // Explicit read-write is accepted. - db.run( - "INSERT INTO _vfs_mounts (root, kind, mode) VALUES (?, ?, ?)", - "/m2", - "r2", - "read-write", - ); - const row2 = db.one<{ mode: string }>("SELECT mode FROM _vfs_mounts WHERE root = ?", "/m2"); - expect(row2?.mode).toBe("read-write"); - - // The CHECK constraint rejects anything else. - expect(() => - db.run("INSERT INTO _vfs_mounts (root, kind, mode) VALUES (?, ?, ?)", "/m3", "r2", "bogus"), - ).toThrow(/CHECK constraint/); - }); - - it("upgrades a v1 database to the current SCHEMA_VERSION", () => { - // Stage a database at the old shape: _vfs_mounts without the - // mode column, vfs_meta.schema_version = 1. The baseline DDL - // run by initializeSchema is "IF NOT EXISTS" so it won't touch - // the existing _vfs_mounts; the migration must. - const storage = new SQLiteTestStorage(); - const db = new Database(storage); - - db.transactionSync(() => { - db.run( - `CREATE TABLE vfs_meta ( - k TEXT PRIMARY KEY, - v INTEGER NOT NULL - )`, - ); - db.run( - `CREATE TABLE _vfs_mounts ( - root TEXT PRIMARY KEY, - kind TEXT NOT NULL, - indexed INTEGER NOT NULL DEFAULT 0 - )`, - ); - db.run("INSERT INTO _vfs_mounts (root, kind, indexed) VALUES (?, ?, ?)", "/m1", "r2", 1); - db.run("INSERT INTO vfs_meta (k, v) VALUES (?, ?)", "schema_version", 1); - }); - - initializeSchema(db, () => 0); - - // Version bumped. - const versionRow = db.one<{ v: number }>( - "SELECT v FROM vfs_meta WHERE k = ?", - "schema_version", - ); - expect(versionRow?.v).toBe(SCHEMA_VERSION); - - // Existing row preserved and stamped with the conservative - // default so a re-index pass has to opt back into read-write. - const row = db.one<{ root: string; mode: string; indexed: number }>( - "SELECT root, mode, indexed FROM _vfs_mounts WHERE root = ?", - "/m1", - ); - expect(row).toEqual({ root: "/m1", mode: "read-only", indexed: 1 }); - - // Post-migration the CHECK constraint is live. - expect(() => - db.run("INSERT INTO _vfs_mounts (root, kind, mode) VALUES (?, ?, ?)", "/m2", "r2", "bogus"), - ).toThrow(/CHECK constraint/); - }); - - it("backfills vfs_nodes.size from chunk sums on the v2 -> v3 upgrade", () => { - // Stage a database at the v2 shape: vfs_nodes without the - // `size` column, schema_version = 2. The migration adds the - // column with a default of 0 and then UPDATEs it from the - // SUM of vfs_chunks.size for each file inode. - const storage = new SQLiteTestStorage(); - const db = new Database(storage); - - db.transactionSync(() => { - db.run( - `CREATE TABLE vfs_meta ( - k TEXT PRIMARY KEY, - v INTEGER NOT NULL - )`, - ); - db.run( - `CREATE TABLE vfs_nodes ( - inode INTEGER PRIMARY KEY AUTOINCREMENT, - type TEXT NOT NULL CHECK(type IN ('file','dir','symlink')), - mode INTEGER NOT NULL DEFAULT 493, - mtime INTEGER NOT NULL, - rev INTEGER NOT NULL DEFAULT 0, - mount_root TEXT, - stub_size INTEGER, - manifest_hash BLOB, - link_target TEXT - )`, - ); - db.run( - `CREATE TABLE vfs_chunks ( - inode INTEGER NOT NULL, - idx INTEGER NOT NULL, - hash BLOB NOT NULL, - size INTEGER NOT NULL, - PRIMARY KEY (inode, idx) - )`, - ); - // A live file with two chunks summing to 7 bytes, a live dir, - // and a live file with no chunks (empty file). - db.run( - `INSERT INTO vfs_nodes (inode, type, mode, mtime, rev) VALUES - (1, 'dir', 493, 0, 0), - (2, 'file', 420, 0, 0), - (3, 'file', 420, 0, 0)`, - ); - db.run( - "INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", - 2, - 0, - new Uint8Array(32), - 3, - ); - db.run( - "INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", - 2, - 1, - new Uint8Array(32), - 4, - ); - db.run("INSERT INTO vfs_meta (k, v) VALUES (?, ?)", "schema_version", 2); - }); - - initializeSchema(db, () => 0); - - const sizes = db.all<{ inode: number; size: number }>( - "SELECT inode, size FROM vfs_nodes ORDER BY inode", - ); - expect(sizes).toEqual([ - { inode: 1, size: 0 }, - { inode: 2, size: 7 }, - { inode: 3, size: 0 }, - ]); - }); - - it("upgrades a v3 database, backfilling _vfs_watermark with the default backend", () => { - // Stage a database at the v3 shape: _vfs_watermark with the - // old single-column primary key, two existing rows; vfs_nodes - // already has the size column from the v2 -> v3 migration. - const storage = new SQLiteTestStorage(); - const db = new Database(storage); - - db.transactionSync(() => { - db.run( - `CREATE TABLE vfs_meta ( - k TEXT PRIMARY KEY, - v INTEGER NOT NULL - )`, - ); - db.run( - `CREATE TABLE _vfs_watermark ( - k TEXT PRIMARY KEY, - v INTEGER NOT NULL - )`, - ); - db.run("INSERT INTO _vfs_watermark (k, v) VALUES (?, ?)", "pushRev", 42); - db.run("INSERT INTO _vfs_watermark (k, v) VALUES (?, ?)", "fetchRev", 17); - db.run("INSERT INTO vfs_meta (k, v) VALUES (?, ?)", "schema_version", 3); - }); - - initializeSchema(db, () => 0); - - // Version bumped. - const versionRow = db.one<{ v: number }>( - "SELECT v FROM vfs_meta WHERE k = ?", - "schema_version", - ); - expect(versionRow?.v).toBe(SCHEMA_VERSION); - - // Existing rows preserved under the default backend slot. - const push = db.one<{ k: string; backend: string; v: number }>( - "SELECT k, backend, v FROM _vfs_watermark WHERE k = ?", - "pushRev", - ); - expect(push).toEqual({ k: "pushRev", backend: "default", v: 42 }); - const fetch = db.one<{ k: string; backend: string; v: number }>( - "SELECT k, backend, v FROM _vfs_watermark WHERE k = ?", - "fetchRev", - ); - expect(fetch).toEqual({ k: "fetchRev", backend: "default", v: 17 }); - - // The composite PK is live: same key under a different backend - // is allowed and doesn't collide with the migrated row. - db.run("INSERT INTO _vfs_watermark (k, backend, v) VALUES (?, ?, ?)", "pushRev", "worker", 99); - const worker = db.one<{ v: number }>( - "SELECT v FROM _vfs_watermark WHERE k = ? AND backend = ?", - "pushRev", - "worker", - ); - expect(worker?.v).toBe(99); - }); - - it("rebuilds vfs_dirents and vfs_chunks as WITHOUT ROWID on the v4 -> v5 upgrade, preserving all data", () => { - // Stage a v4-shape database: vfs_dirents and vfs_chunks are plain - // rowid tables carrying their secondary indexes; vfs_nodes already - // has the size column. Populate a representative graph — nested - // dirs, a hardlink (one inode, two names), multi-chunk files, and - // content dedup (distinct files sharing blob hashes) — so the - // rebuild is proven lossless, not merely structurally correct. - const storage = new SQLiteTestStorage(); - const db = new Database(storage); - - const hashA = new Uint8Array(32).fill(0xaa); - const hashB = new Uint8Array(32).fill(0xbb); - - db.transactionSync(() => { - db.run(`CREATE TABLE vfs_meta (k TEXT PRIMARY KEY, v INTEGER NOT NULL)`); - db.run( - `CREATE TABLE vfs_nodes ( - inode INTEGER PRIMARY KEY AUTOINCREMENT, - type TEXT NOT NULL CHECK(type IN ('file','dir','symlink')), - mode INTEGER NOT NULL DEFAULT 493, - mtime INTEGER NOT NULL, - rev INTEGER NOT NULL DEFAULT 0, - mount_root TEXT, - stub_size INTEGER, - manifest_hash BLOB, - link_target TEXT, - size INTEGER NOT NULL DEFAULT 0 - )`, - ); - // Pre-migration shape: rowid tables plus their secondary indexes. - db.run( - `CREATE TABLE vfs_dirents ( - parent_inode INTEGER NOT NULL, - name TEXT NOT NULL, - child_inode INTEGER NOT NULL, - PRIMARY KEY (parent_inode, name) - )`, - ); - db.run(`CREATE INDEX vfs_dirents_by_child ON vfs_dirents(child_inode)`); - db.run( - `CREATE TABLE vfs_chunks ( - inode INTEGER NOT NULL, - idx INTEGER NOT NULL, - hash BLOB NOT NULL, - size INTEGER NOT NULL, - PRIMARY KEY (inode, idx) - )`, - ); - db.run(`CREATE INDEX vfs_chunks_by_hash ON vfs_chunks(hash)`); - db.run( - `CREATE TABLE vfs_blobs ( - hash BLOB PRIMARY KEY, - size INTEGER NOT NULL, - last_seen INTEGER NOT NULL - )`, - ); - db.run( - `CREATE TABLE vfs_blob_bytes ( - hash BLOB PRIMARY KEY REFERENCES vfs_blobs(hash) ON DELETE CASCADE, - bytes BLOB NOT NULL - )`, - ); - - // Graph: /(1) -> a(2) -> { f1(3), f2(4), b(5) }, b(5) -> deep(6). - // Hardlink: /a/hard is a second name for inode 3. - db.run( - `INSERT INTO vfs_nodes (inode, type, mode, mtime, rev, size) VALUES - (1, 'dir', 493, 0, 0, 0), - (2, 'dir', 493, 0, 1, 0), - (3, 'file', 420, 0, 2, 10), - (4, 'file', 420, 0, 3, 5), - (5, 'dir', 493, 0, 4, 0), - (6, 'file', 420, 0, 5, 5)`, - ); - db.run( - `INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES - (1, 'a', 2), - (2, 'f1', 3), - (2, 'f2', 4), - (2, 'hard', 3), - (2, 'b', 5), - (5, 'deep', 6)`, - ); - // f1(3): hashA + hashB. f2(4): hashA (dedup). deep(6): hashB (dedup). - // -> 4 chunk rows referencing 2 distinct blobs. - db.run("INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", 3, 0, hashA, 5); - db.run("INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", 3, 1, hashB, 5); - db.run("INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", 4, 0, hashA, 5); - db.run("INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", 6, 0, hashB, 5); - db.run("INSERT INTO vfs_blobs (hash, size, last_seen) VALUES (?, ?, ?)", hashA, 5, 0); - db.run("INSERT INTO vfs_blobs (hash, size, last_seen) VALUES (?, ?, ?)", hashB, 5, 0); - db.run( - "INSERT INTO vfs_blob_bytes (hash, bytes) VALUES (?, ?)", - hashA, - new Uint8Array(5).fill(1), - ); - db.run( - "INSERT INTO vfs_blob_bytes (hash, bytes) VALUES (?, ?)", - hashB, - new Uint8Array(5).fill(2), - ); - - db.run("INSERT INTO vfs_meta (k, v) VALUES (?, ?)", "schema_version", 4); - }); - - // Snapshot the two rebuilt tables before migrating. - const direntsBefore = db.all<{ parent_inode: number; name: string; child_inode: number }>( - "SELECT parent_inode, name, child_inode FROM vfs_dirents ORDER BY parent_inode, name", - ); - const chunksBefore = db.all<{ inode: number; idx: number; hash: Uint8Array; size: number }>( - "SELECT inode, idx, hash, size FROM vfs_chunks ORDER BY inode, idx", - ); - - initializeSchema(db, () => 0); - - // (a) Version bumped. - expect(db.one<{ v: number }>("SELECT v FROM vfs_meta WHERE k = ?", "schema_version")?.v).toBe( - SCHEMA_VERSION, - ); - - const tableSql = (name: string): string => - db.one<{ sql: string }>( - "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?", - name, - )?.sql ?? ""; - - // (b) Both targets are now WITHOUT ROWID; vfs_blob_bytes is untouched. - expect(tableSql("vfs_dirents").toUpperCase()).toContain("WITHOUT ROWID"); - expect(tableSql("vfs_chunks").toUpperCase()).toContain("WITHOUT ROWID"); - expect(tableSql("vfs_blob_bytes").toUpperCase()).not.toContain("WITHOUT ROWID"); - - // (c) Both secondary indexes survived the rebuild. - const indexNames = db - .all<{ name: string }>("SELECT name FROM sqlite_master WHERE type = 'index'") - .map((r) => r.name); - expect(indexNames).toContain("vfs_dirents_by_child"); - expect(indexNames).toContain("vfs_chunks_by_hash"); - - // The rebuild's temp tables are dropped — no leftovers. - const tableNames = db - .all<{ name: string }>("SELECT name FROM sqlite_master WHERE type = 'table'") - .map((r) => r.name); - expect(tableNames).not.toContain("vfs_dirents_v4"); - expect(tableNames).not.toContain("vfs_chunks_v4"); - - // (d) Data survived byte-for-byte. - expect( - db.all("SELECT parent_inode, name, child_inode FROM vfs_dirents ORDER BY parent_inode, name"), - ).toEqual(direntsBefore); - expect(db.all("SELECT inode, idx, hash, size FROM vfs_chunks ORDER BY inode, idx")).toEqual( - chunksBefore, - ); - // Hardlink preserved: inode 3 still reached by both names via the - // recreated child index. - expect( - db.all<{ parent_inode: number; name: string }>( - "SELECT parent_inode, name FROM vfs_dirents WHERE child_inode = ? ORDER BY name", - 3, - ), - ).toEqual([ - { parent_inode: 2, name: "f1" }, - { parent_inode: 2, name: "hard" }, - ]); - // Dedup intact: 4 chunk rows, 2 distinct blobs. - expect(db.one<{ c: number }>("SELECT COUNT(*) AS c FROM vfs_chunks")?.c).toBe(4); - expect(db.one<{ c: number }>("SELECT COUNT(*) AS c FROM vfs_blobs")?.c).toBe(2); - expect(db.one<{ c: number }>("SELECT COUNT(DISTINCT hash) AS c FROM vfs_chunks")?.c).toBe(2); - - // (e) A fresh install lands the identical table shape (modulo - // whitespace) as the migrated database. - const fresh = new Database(new SQLiteTestStorage()); - initializeSchema(fresh, () => 0); - const freshSql = (name: string): string => - fresh.one<{ sql: string }>( - "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?", - name, - )?.sql ?? ""; - const norm = (sql: string): string => sql.replace(/\s+/g, " ").trim().toUpperCase(); - expect(norm(tableSql("vfs_dirents"))).toBe(norm(freshSql("vfs_dirents"))); - expect(norm(tableSql("vfs_chunks"))).toBe(norm(freshSql("vfs_chunks"))); - }); - - it("is idempotent across repeat calls", () => { - const storage = new SQLiteTestStorage(); - const db = new Database(storage); - - initializeSchema(db, () => 0); - expect(() => initializeSchema(db, () => 0)).not.toThrow(); - - const row = db.one<{ v: number }>("SELECT v FROM vfs_meta WHERE k = ?", "schema_version"); - expect(row?.v).toBe(SCHEMA_VERSION); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/schema/index.ts b/spikes/349-dofs/vendor/dofs/src/schema/index.ts deleted file mode 100644 index e0147e96..00000000 --- a/spikes/349-dofs/vendor/dofs/src/schema/index.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { createWorkspaceError } from "../errors.js"; -import type { Database } from "../storage.js"; -import { CORE_STATEMENTS, ROOT_INODE, SCHEMA_VERSION } from "./core.js"; -import { runMigrations } from "./migrations.js"; -import { SYNC_STATEMENTS } from "./sync.js"; - -export { ROOT_INODE, SCHEMA_VERSION } from "./core.js"; - -interface MetaRow { - v: number; -} - -export function initializeSchema(db: Database, now: () => number): void { - db.transactionSync(() => { - // 1. Baseline DDL. Every statement is "CREATE TABLE IF NOT - // EXISTS" / "CREATE INDEX IF NOT EXISTS" so this is a no-op - // on already-initialized databases. Fresh databases come out - // of this step at the latest column shape (SCHEMA_VERSION). - for (const statement of CORE_STATEMENTS) { - db.run(statement); - } - for (const statement of SYNC_STATEMENTS) { - db.run(statement); - } - - // 2. Read the on-disk schema version. Absent → 0 (very first - // boot of this database). The baseline above just created - // every table at the latest shape, so a 0 → SCHEMA_VERSION - // jump has nothing to migrate. - const storedVersion = db.one( - "SELECT v FROM vfs_meta WHERE k = ?", - "schema_version", - )?.v; - const onDiskVersion = storedVersion ?? 0; - - if (onDiskVersion > SCHEMA_VERSION) { - throw createWorkspaceError( - "EIO", - `Unsupported workspace filesystem schema version ${onDiskVersion}`, - ); - } - - // 3. Migrate. Skip when the database is fresh (0) — the - // baseline DDL already shipped the latest shape. Otherwise - // dispatch each registered migrator until we hit the - // target. - if (onDiskVersion > 0 && onDiskVersion < SCHEMA_VERSION) { - runMigrations(db, onDiskVersion, SCHEMA_VERSION); - } - - // 4. Stamp the version and seed the boot rows. Both shapes - // (insert-if-missing, then update) keep this idempotent so - // repeat calls do nothing. - db.run("INSERT OR IGNORE INTO vfs_meta (k, v) VALUES (?, ?)", "schema_version", SCHEMA_VERSION); - db.run("UPDATE vfs_meta SET v = ? WHERE k = ?", SCHEMA_VERSION, "schema_version"); - db.run("INSERT OR IGNORE INTO vfs_meta (k, v) VALUES (?, ?)", "rev", 1); - db.run( - "INSERT OR IGNORE INTO _vfs_watermark (k, backend, v) VALUES (?, 'default', ?)", - "pushRev", - 0, - ); - db.run( - "INSERT OR IGNORE INTO _vfs_watermark (k, backend, v) VALUES (?, 'default', ?)", - "fetchRev", - 0, - ); - db.run( - "INSERT OR IGNORE INTO _vfs_fetch_cursor (k, backend, path) VALUES (?, 'default', ?)", - "fetch", - null, - ); - - db.run( - `INSERT OR IGNORE INTO vfs_nodes - (inode, type, mode, mtime, rev) - VALUES (?, 'dir', ?, ?, 0)`, - ROOT_INODE, - 0o755, - now(), - ); - }); -} diff --git a/spikes/349-dofs/vendor/dofs/src/schema/migrations.ts b/spikes/349-dofs/vendor/dofs/src/schema/migrations.ts deleted file mode 100644 index d2425ada..00000000 --- a/spikes/349-dofs/vendor/dofs/src/schema/migrations.ts +++ /dev/null @@ -1,169 +0,0 @@ -// Schema migration runner. -// -// The schema's "CREATE TABLE IF NOT EXISTS" baseline handles fresh -// databases. When a schema column changes shape — added, dropped, -// renamed, retyped — IF NOT EXISTS does nothing and the older -// rows stay incompatible. Migrations close that gap. -// -// Shape: an ordered list of `(from, to, migrator)` tuples. The -// runner reads `vfs_meta.schema_version` (defaulting to 0 when the -// row is absent), picks every migration whose `from === current`, -// runs it, advances `current`, and repeats until `current >= -// SCHEMA_VERSION`. The whole pass runs inside the caller's -// transactionSync so a partial migration rolls back. -// -// Each migrator is a `(db: Database) => void` and may assume the -// previous version's schema is in place. Migrators land schema -// changes only; they don't touch user data unless the column shape -// requires it. - -import type { Database } from "../storage.js"; - -export interface Migration { - readonly from: number; - readonly to: number; - readonly migrator: (db: Database) => void; -} - -// v1 → v2 — add `_vfs_mounts.mode` so dofs can enforce read-only -// mounts at the data layer. Existing rows default to 'read-only'; -// the workspace re-stamps them with the registered mount's mode on -// the next index pass. -// -// The CHECK constraint is duplicated in `sync.ts`'s fresh-install -// DDL; both paths must keep the same allowed set. -function v1_to_v2_add_mounts_mode(db: Database): void { - db.run( - `ALTER TABLE _vfs_mounts - ADD COLUMN mode TEXT NOT NULL DEFAULT 'read-only' - CHECK(mode IN ('read-only', 'read-write'))`, - ); -} - -// v2 → v3 — denormalise file size onto vfs_nodes so stat doesn't -// have to SUM the chunk rows on every call. The column is -// backfilled from existing vfs_chunks; later writes maintain it. -function v2_to_v3_add_size_column(db: Database): void { - const hasColumn = db - .all<{ name: string }>("PRAGMA table_info(vfs_nodes)") - .some((column) => column.name === "size"); - if (!hasColumn) { - db.run("ALTER TABLE vfs_nodes ADD COLUMN size INTEGER NOT NULL DEFAULT 0"); - } - db.run( - `UPDATE vfs_nodes - SET size = COALESCE( - (SELECT SUM(size) FROM vfs_chunks WHERE vfs_chunks.inode = vfs_nodes.inode), - 0 - ) - WHERE type = 'file'`, - ); -} - -// v3 → v4 — add a `backend` column to `_vfs_watermark` so a -// workspace can host more than one backend with independent sync -// cursors. SQLite's ALTER TABLE can't change a primary key; copy -// existing rows into a fresh table with the composite -// (k, backend) primary key, then swap the tables. -// -// Existing rows land under the `default` backend id, which the -// dofs sync helpers also use as the fallback when a caller -// doesn't pass an id. Pre-multi-backend workspaces keep their -// pushRev / fetchRev cursors intact through the upgrade. -function v3_to_v4_watermark_backend_column(db: Database): void { - db.run(`ALTER TABLE _vfs_watermark RENAME TO _vfs_watermark_v3`); - db.run( - `CREATE TABLE _vfs_watermark ( - k TEXT NOT NULL, - backend TEXT NOT NULL DEFAULT 'default', - v INTEGER NOT NULL, - PRIMARY KEY (k, backend) - )`, - ); - db.run( - `INSERT INTO _vfs_watermark (k, backend, v) - SELECT k, 'default', v FROM _vfs_watermark_v3`, - ); - db.run(`DROP TABLE _vfs_watermark_v3`); -} - -// v4 → v5 — rebuild `vfs_dirents` and `vfs_chunks` as WITHOUT ROWID. -// SQLite can't convert a table to WITHOUT ROWID in place, so for each -// table: rename it aside, create the WITHOUT ROWID replacement, copy -// the rows, drop the old table. -// -// Both targets are FK-inert (neither is an FK parent or child; the -// schema's only foreign key is vfs_blob_bytes -> vfs_blobs) and have -// composite primary keys with no AUTOINCREMENT, so WITHOUT ROWID is -// legal and sqlite_sequence is untouched. `vfs_blob_bytes` is left -// alone on purpose — it holds the large blob payloads and the FK. -// -// A RENAME carries the table's secondary index along to the temp -// name, and the following DROP takes the index with it. The baseline -// `CREATE INDEX IF NOT EXISTS` in initializeSchema already ran, before -// migrations, and does not re-run — so this migrator must recreate -// vfs_dirents_by_child and vfs_chunks_by_hash itself, or upgraded -// databases silently lose them. Keep the CREATE bodies in lockstep -// with the fresh-install DDL in core.ts. -function v4_to_v5_without_rowid(db: Database): void { - // vfs_dirents - db.run(`ALTER TABLE vfs_dirents RENAME TO vfs_dirents_v4`); - db.run( - `CREATE TABLE vfs_dirents ( - parent_inode INTEGER NOT NULL, - name TEXT NOT NULL, - child_inode INTEGER NOT NULL, - PRIMARY KEY (parent_inode, name) - ) WITHOUT ROWID`, - ); - db.run( - `INSERT INTO vfs_dirents (parent_inode, name, child_inode) - SELECT parent_inode, name, child_inode FROM vfs_dirents_v4`, - ); - db.run(`DROP TABLE vfs_dirents_v4`); - db.run(`CREATE INDEX vfs_dirents_by_child ON vfs_dirents(child_inode)`); - - // vfs_chunks - db.run(`ALTER TABLE vfs_chunks RENAME TO vfs_chunks_v4`); - db.run( - `CREATE TABLE vfs_chunks ( - inode INTEGER NOT NULL, - idx INTEGER NOT NULL, - hash BLOB NOT NULL, - size INTEGER NOT NULL, - PRIMARY KEY (inode, idx) - ) WITHOUT ROWID`, - ); - db.run( - `INSERT INTO vfs_chunks (inode, idx, hash, size) - SELECT inode, idx, hash, size FROM vfs_chunks_v4`, - ); - db.run(`DROP TABLE vfs_chunks_v4`); - db.run(`CREATE INDEX vfs_chunks_by_hash ON vfs_chunks(hash)`); -} - -export const MIGRATIONS: readonly Migration[] = [ - { from: 1, to: 2, migrator: v1_to_v2_add_mounts_mode }, - { from: 2, to: 3, migrator: v2_to_v3_add_size_column }, - { from: 3, to: 4, migrator: v3_to_v4_watermark_backend_column }, - { from: 4, to: 5, migrator: v4_to_v5_without_rowid }, -] as const; - -// Apply every migration whose `from` matches the current version, -// in order, until we reach the target. The caller has already -// wrapped this in a transactionSync; failures here roll the whole -// initializeSchema call back. -export function runMigrations(db: Database, current: number, target: number): number { - let version = current; - while (version < target) { - const next = MIGRATIONS.find((m) => m.from === version); - if (next === undefined) { - // No migration registered for this jump. This is a bug — the - // version was bumped without a matching migration. - throw new Error(`dofs schema: no migration registered for v${version} -> v${target}`); - } - next.migrator(db); - version = next.to; - } - return version; -} diff --git a/spikes/349-dofs/vendor/dofs/src/schema/sync.ts b/spikes/349-dofs/vendor/dofs/src/schema/sync.ts deleted file mode 100644 index fdbac101..00000000 --- a/spikes/349-dofs/vendor/dofs/src/schema/sync.ts +++ /dev/null @@ -1,59 +0,0 @@ -// Sync-protocol tables. Populated by the sync module; the FS module -// only writes to vfs_changes (via sync/changes.ts) on rm. The rest -// of these tables stay empty until the sync task is implemented. - -export const SYNC_STATEMENTS = [ - `CREATE TABLE IF NOT EXISTS vfs_manifests ( - hash BLOB PRIMARY KEY, - size INTEGER NOT NULL, - encoded BLOB NOT NULL, - last_seen INTEGER NOT NULL DEFAULT 0 - )`, - `CREATE TABLE IF NOT EXISTS vfs_changes ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - rev INTEGER NOT NULL, - path TEXT NOT NULL, - op TEXT NOT NULL CHECK(op IN ('delete')) - )`, - `CREATE INDEX IF NOT EXISTS vfs_changes_by_rev ON vfs_changes(rev)`, - // changes.ts looks up the latest op for a path via - // `WHERE path = ? ORDER BY id DESC LIMIT 1`. Without the index - // SQLite falls back to a full scan; with (path, id DESC) the - // lookup is O(log n) and the ORDER BY drains straight from the - // index. Used on every recordDelete and on every push-tick that - // processes tombstones. - `CREATE INDEX IF NOT EXISTS vfs_changes_by_path ON vfs_changes(path, id DESC)`, - // Watermarks are keyed by (k, backend) so a workspace hosting - // multiple backends keeps each backend's sync cursors - // independent. The `backend` column was added at schema v3; - // `schema/migrations.ts` owns the ALTER for existing - // databases. Fresh installs land the composite key directly. - `CREATE TABLE IF NOT EXISTS _vfs_watermark ( - k TEXT NOT NULL, - backend TEXT NOT NULL DEFAULT 'default', - v INTEGER NOT NULL, - PRIMARY KEY (k, backend) - )`, - // The fetch cursor's same-rev `path` component, keyed by - // (k, backend) so each backend resumes a partially-drained rev - // independently. The rev component lives in _vfs_watermark under - // 'fetchRev'; this table only holds the in-rev path. `backend` - // mirrors _vfs_watermark and defaults to 'default'. - `CREATE TABLE IF NOT EXISTS _vfs_fetch_cursor ( - k TEXT NOT NULL CHECK(k = 'fetch'), - backend TEXT NOT NULL DEFAULT 'default', - path TEXT, - PRIMARY KEY (k, backend) - )`, - // The `mode` column was added at schema v2; `schema/migrations.ts` - // owns the ALTER for existing databases. Keep the CHECK - // constraint here aligned with the migration's CHECK so fresh - // installs and upgrades enforce the same allowed set. - `CREATE TABLE IF NOT EXISTS _vfs_mounts ( - root TEXT PRIMARY KEY, - kind TEXT NOT NULL, - indexed INTEGER NOT NULL DEFAULT 0, - mode TEXT NOT NULL DEFAULT 'read-only' - CHECK(mode IN ('read-only', 'read-write')) - )`, -] as const; diff --git a/spikes/349-dofs/vendor/dofs/src/storage.ts b/spikes/349-dofs/vendor/dofs/src/storage.ts deleted file mode 100644 index becce5a7..00000000 --- a/spikes/349-dofs/vendor/dofs/src/storage.ts +++ /dev/null @@ -1,115 +0,0 @@ -import type { DurableObjectStorageLike, SQLStorageLike } from "./types.js"; - -export class Database { - readonly sql: SQLStorageLike; - readonly transactionSync: (closure: () => T) => T; - // Depth counter so reentrant transactionSync() calls work. The - // outer call uses the storage adapter's transactionSync (or - // BEGIN/COMMIT under the hood); nested calls use SAVEPOINTs - // through sql.exec directly. SQLite forbids a real BEGIN inside - // an active transaction. - #txDepth = 0; - - constructor(storage: DurableObjectStorageLike) { - this.sql = storage.sql; - this.transactionSync = (closure: () => T): T => { - if (this.#txDepth > 0) { - // Reentrant call: use a savepoint. SQLite's RELEASE on a - // savepoint inside an outer transaction commits the inner - // work without ending the outer one. - const sp = `_t${this.#txDepth}`; - this.sql.exec(`SAVEPOINT ${sp}`); - this.#txDepth++; - try { - const result = closure(); - this.sql.exec(`RELEASE ${sp}`); - return result; - } catch (error) { - this.sql.exec(`ROLLBACK TO ${sp}`); - this.sql.exec(`RELEASE ${sp}`); - throw error; - } finally { - this.#txDepth--; - } - } - // Outer call: hand off to the storage adapter so the DO - // runtime's transaction semantics apply. - this.#txDepth++; - try { - if (storage.transactionSync !== undefined) { - return storage.transactionSync(closure); - } - if (storage.transaction !== undefined) { - const result = storage.transaction(closure); - if ( - result !== undefined && - result !== null && - typeof result === "object" && - "then" in result - ) { - throw new Error("Durable Object storage adapter requires synchronous transactions"); - } - return result; - } - return closure(); - } finally { - this.#txDepth--; - } - }; - } - - // True while a transactionSync closure is on the stack. The resolve - // cache uses this to refuse populating entries mid-transaction, so a - // rolled-back mutation can never leave the cache reflecting - // uncommitted state. (Invalidation still runs freely inside a - // transaction — dropping an entry is always safe.) - // - // Invariant: #txDepth only tracks transactionSync. A raw - // BEGIN/SAVEPOINT issued through run() would open a transaction this - // flag can't see, letting the cache populate mid-transaction and - // survive a rollback — so transactionSync is the only sanctioned way - // to open one. - get inTransaction(): boolean { - return this.#txDepth > 0; - } - - run(query: string, ...bindings: unknown[]): void { - this.sql.exec(query, ...bindings); - } - - all(query: string, ...bindings: unknown[]): Row[] { - const rows = this.sql.exec(query, ...bindings).toArray(); - return rows.map((row) => normalizeRow(row as Record)) as Row[]; - } - - one(query: string, ...bindings: unknown[]): Row | undefined { - return this.all(query, ...bindings)[0]; - } - - scalar(query: string, ...bindings: unknown[]): T | undefined { - const row = this.one>(query, ...bindings); - if (row === undefined) { - return undefined; - } - - const [value] = Object.values(row); - return value; - } -} - -// Cloudflare's DO SqlStorage returns BLOB columns as ArrayBuffer, -// whereas node:sqlite returns Uint8Array. Normalise to Uint8Array so -// the rest of the code only has to handle one shape. -function normalizeRow(row: Record): Record { - // node:sqlite hands back rows with a null prototype; the DO SQL - // flavour returns ArrayBuffer for BLOB columns. Re-key into a plain - // {} so consumers get Object.prototype-shaped rows (capnweb's - // serializer keys off Object.prototype to detect "object") and - // convert any ArrayBuffer to Uint8Array in the same pass. - const out: Record = {}; - for (const key of Object.keys(row)) { - const value = row[key]; - out[key] = value instanceof ArrayBuffer ? new Uint8Array(value) : value; - } - return out; -} diff --git a/spikes/349-dofs/vendor/dofs/src/sync/apply.test.ts b/spikes/349-dofs/vendor/dofs/src/sync/apply.test.ts deleted file mode 100644 index 5ced0d0e..00000000 --- a/spikes/349-dofs/vendor/dofs/src/sync/apply.test.ts +++ /dev/null @@ -1,1120 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { link } from "../fs/link.js"; -import { mkdir } from "../fs/mkdir.js"; -import { invalidateReadOnlyMountCache } from "../fs/mount-guard.js"; -import { readFile } from "../fs/readFile.js"; -import { readlink } from "../fs/readlink.js"; -import { rename } from "../fs/rename.js"; -import { resolveInode } from "../fs/resolve.js"; -import { rm } from "../fs/rm.js"; -import { symlink } from "../fs/symlink.js"; -import { withDB, withTwoDBs } from "../fs/with-db.js"; -import { writeFile, writeFileSync } from "../fs/writeFile.js"; -import { applyChanges, applyChangesSync } from "./apply.js"; -import type { ChangeEntry } from "./changes.js"; -import { coalesceChanges } from "./coalesce.js"; -import { fetchObjects } from "./fetch.js"; -import { currentRev } from "./watermarks.js"; - -async function drain(it: AsyncIterable): Promise { - const out: T[] = []; - for await (const x of it) out.push(x); - return out; -} - -function hex(bytes: Uint8Array): string { - let s = ""; - for (let i = 0; i < bytes.byteLength; i++) s += bytes[i].toString(16).padStart(2, "0"); - return s; -} - -function deepChildPath(depth: number): string { - return `/dst/${Array.from({ length: depth }, () => "d").join("/")}/old.txt`; -} - -async function collectObjects( - db: import("../storage.js").Database, - entries: ChangeEntry[], -): Promise> { - const hashes: Uint8Array[] = []; - const seen = new Set(); - for (const e of entries) { - if (e.kind !== "file") continue; - for (const c of e.chunks) { - const k = hex(c.hash); - if (!seen.has(k)) { - seen.add(k); - hashes.push(c.hash); - } - } - } - const out = new Map(); - for await (const { hash, bytes } of fetchObjects(db, hashes)) { - out.set(hex(hash), bytes); - } - return out; -} - -describe("applyChanges", () => { - it("converges across a mixed stream", async () => { - await withTwoDBs( - async (a) => { - await writeFile(a, "/a.txt", "alpha", {}, () => 1); - await writeFile(a, "/b.txt", "beta", {}, () => 2); - const entries = await drain(coalesceChanges(a, 0)); - return { entries, objects: await collectObjects(a, entries) }; - }, - async (b, { entries, objects }) => { - await applyChanges(b, entries, objects); - expect(await readFile(b, "/a.txt", "utf8")).toBe("alpha"); - expect(await readFile(b, "/b.txt", "utf8")).toBe("beta"); - }, - ); - }); - - it("commits in batches capped by byte budget", async () => { - // Force many small files; with a tiny byte budget the apply - // path should still converge, just across more batches. We - // verify convergence rather than batch count (batch count is - // an implementation detail). - await withTwoDBs( - async (a) => { - for (let i = 0; i < 10; i++) { - await writeFile(a, `/f${i}.txt`, `payload ${i}`, {}, () => 100 + i); - } - const entries = await drain(coalesceChanges(a, 0)); - return { entries, objects: await collectObjects(a, entries) }; - }, - async (b, { entries, objects }) => { - await applyChanges(b, entries, objects, { maxBytesPerBatch: 16 }); - for (let i = 0; i < 10; i++) { - expect(await readFile(b, `/f${i}.txt`, "utf8")).toBe(`payload ${i}`); - } - }, - ); - }); - - it("applies a file rename over an existing file", async () => { - await withTwoDBs( - async (a) => { - await writeFile(a, "/src", "new", {}, () => 1); - await writeFile(a, "/dst", "old", {}, () => 2); - const cursor = currentRev(a); - - rename(a, "/src", "/dst"); - - const entries = await drain(coalesceChanges(a, cursor)); - return { entries, objects: await collectObjects(a, entries) }; - }, - async (b, { entries, objects }) => { - await writeFile(b, "/src", "new", {}, () => 1); - await writeFile(b, "/dst", "old", {}, () => 2); - - await applyChanges(b, entries, objects); - - expect(resolveInode(b, "/src")).toBeNull(); - expect(await readFile(b, "/dst", "utf8")).toBe("new"); - }, - ); - }); - - it("applies a directory rename over an existing empty directory", async () => { - await withTwoDBs( - async (a) => { - mkdir(a, "/src", { mode: 0o700, recursive: true }, () => 1); - await writeFile(a, "/src/inside", "x", {}, () => 2); - mkdir(a, "/dst", { mode: 0o755 }, () => 3); - const cursor = currentRev(a); - - rename(a, "/src", "/dst"); - - const entries = await drain(coalesceChanges(a, cursor)); - return { entries, objects: await collectObjects(a, entries) }; - }, - async (b, { entries, objects }) => { - mkdir(b, "/src", { mode: 0o700, recursive: true }, () => 1); - await writeFile(b, "/src/inside", "x", {}, () => 2); - mkdir(b, "/dst", { mode: 0o755 }, () => 3); - - await applyChanges(b, entries, objects); - - expect(resolveInode(b, "/src")).toBeNull(); - expect(resolveInode(b, "/dst", { followSymlinks: false })).toMatchObject({ - type: "dir", - mode: 0o700, - mtime: 1, - }); - expect(await readFile(b, "/dst/inside", "utf8")).toBe("x"); - }, - ); - }); - - it("applyChangesSync updates existing directory metadata", async () => { - await withDB((db) => { - mkdir(db, "/dst", { mode: 0o755 }, () => 3); - - applyChangesSync( - db, - [{ kind: "dir", rev: 99, path: "/dst", mode: 0o700, mtime: 1 }], - new Map(), - ); - - expect(resolveInode(db, "/dst", { followSymlinks: false })).toMatchObject({ - type: "dir", - mode: 0o700, - mtime: 1, - }); - }); - }); - - it("does not update existing directory mtime for upstream entries with matching mode", async () => { - await withDB(async (db) => { - mkdir(db, "/dst", { mode: 0o700 }, () => 3); - - await applyChanges( - db, - [{ kind: "dir", rev: 99, path: "/dst", mode: 0o700, mtime: 1 }], - new Map(), - { source: "upstream" }, - ); - - expect(resolveInode(db, "/dst", { followSymlinks: false })).toMatchObject({ - type: "dir", - mode: 0o700, - mtime: 3, - }); - }); - }); - - it("applies a file rename over an existing symlink", async () => { - await withTwoDBs( - async (a) => { - await writeFile(a, "/target", "target", {}, () => 1); - await writeFile(a, "/src", "new", {}, () => 2); - symlink(a, "/target", "/dst", () => 3); - const cursor = currentRev(a); - - rename(a, "/src", "/dst"); - - const entries = await drain(coalesceChanges(a, cursor)); - return { entries, objects: await collectObjects(a, entries) }; - }, - async (b, { entries, objects }) => { - await writeFile(b, "/target", "target", {}, () => 1); - await writeFile(b, "/src", "new", {}, () => 2); - symlink(b, "/target", "/dst", () => 3); - - await applyChanges(b, entries, objects); - - expect(resolveInode(b, "/src")).toBeNull(); - expect(resolveInode(b, "/dst", { followSymlinks: false })?.type).toBe("file"); - expect(await readFile(b, "/dst", "utf8")).toBe("new"); - expect(await readFile(b, "/target", "utf8")).toBe("target"); - }, - ); - }); - - it("applies a symlink rename over an existing file", async () => { - await withTwoDBs( - async (a) => { - await writeFile(a, "/target", "target", {}, () => 1); - symlink(a, "/target", "/src", () => 2); - await writeFile(a, "/dst", "old", {}, () => 3); - const cursor = currentRev(a); - - rename(a, "/src", "/dst"); - - const entries = await drain(coalesceChanges(a, cursor)); - return { entries, objects: await collectObjects(a, entries) }; - }, - async (b, { entries, objects }) => { - await writeFile(b, "/target", "target", {}, () => 1); - symlink(b, "/target", "/src", () => 2); - await writeFile(b, "/dst", "old", {}, () => 3); - - await applyChanges(b, entries, objects); - - expect(resolveInode(b, "/src", { followSymlinks: false })).toBeNull(); - expect(resolveInode(b, "/dst", { followSymlinks: false })?.type).toBe("symlink"); - expect(readlink(b, "/dst")).toBe("/target"); - expect(await readFile(b, "/target", "utf8")).toBe("target"); - }, - ); - }); - - it("applies a symlink rename over an existing symlink", async () => { - await withTwoDBs( - async (a) => { - await writeFile(a, "/target", "target", {}, () => 1); - await writeFile(a, "/other", "other", {}, () => 2); - symlink(a, "/target", "/src", () => 3); - symlink(a, "/other", "/dst", () => 4); - const cursor = currentRev(a); - - rename(a, "/src", "/dst"); - - const entries = await drain(coalesceChanges(a, cursor)); - return { entries, objects: await collectObjects(a, entries) }; - }, - async (b, { entries, objects }) => { - await writeFile(b, "/target", "target", {}, () => 1); - await writeFile(b, "/other", "other", {}, () => 2); - symlink(b, "/target", "/src", () => 3); - symlink(b, "/other", "/dst", () => 4); - - await applyChanges(b, entries, objects); - - expect(resolveInode(b, "/src", { followSymlinks: false })).toBeNull(); - expect(resolveInode(b, "/dst", { followSymlinks: false })?.type).toBe("symlink"); - expect(readlink(b, "/dst")).toBe("/target"); - expect(await readFile(b, "/target", "utf8")).toBe("target"); - expect(await readFile(b, "/other", "utf8")).toBe("other"); - }, - ); - }); - - it("applies a directory rename containing a symlink without deleting its target", async () => { - await withTwoDBs( - async (a) => { - await writeFile(a, "/target", "target", {}, () => 1); - mkdir(a, "/src", {}, () => 2); - symlink(a, "/target", "/src/link", () => 3); - const cursor = currentRev(a); - - rename(a, "/src", "/dst"); - - const entries = await drain(coalesceChanges(a, cursor)); - return { entries, objects: await collectObjects(a, entries) }; - }, - async (b, { entries, objects }) => { - await writeFile(b, "/target", "target", {}, () => 1); - mkdir(b, "/src", {}, () => 2); - symlink(b, "/target", "/src/link", () => 3); - - await applyChanges(b, entries, objects); - - expect(resolveInode(b, "/src/link", { followSymlinks: false })).toBeNull(); - expect(resolveInode(b, "/dst/link", { followSymlinks: false })?.type).toBe("symlink"); - expect(readlink(b, "/dst/link")).toBe("/target"); - expect(await readFile(b, "/target", "utf8")).toBe("target"); - }, - ); - }); - - it("applies a file over an existing directory subtree", async () => { - await withDB(async (db) => { - mkdir(db, "/dst/sub", { recursive: true }, () => 1); - await writeFile(db, "/dst/sub/old.txt", "old", {}, () => 2); - - await applyChanges( - db, - [ - { - kind: "file", - rev: 99, - path: "/dst", - mode: 0o644, - mtime: 3, - size: 0, - chunks: [], - }, - ], - new Map(), - ); - - expect(resolveInode(db, "/dst", { followSymlinks: false })?.type).toBe("file"); - expect(resolveInode(db, "/dst/sub/old.txt")).toBeNull(); - }); - }); - - it("applies a file over a deeply nested existing directory subtree", async () => { - await withDB(async (db) => { - const oldPath = deepChildPath(12_000); - mkdir(db, oldPath.slice(0, -"/old.txt".length), { recursive: true }, () => 1); - await writeFile(db, oldPath, "old", {}, () => 2); - - await applyChanges( - db, - [ - { - kind: "file", - rev: 99, - path: "/dst", - mode: 0o644, - mtime: 3, - size: 0, - chunks: [], - }, - ], - new Map(), - ); - - expect(resolveInode(db, "/dst", { followSymlinks: false })?.type).toBe("file"); - expect(resolveInode(db, oldPath, { followSymlinks: false })).toBeNull(); - }); - }); - - it("applies a symlink over an existing directory subtree", async () => { - await withDB((db) => { - mkdir(db, "/dst/sub", { recursive: true }, () => 1); - writeFileSync(db, "/dst/sub/old.txt", new Uint8Array(), {}, () => 2); - - applyChangesSync( - db, - [ - { - kind: "symlink", - rev: 99, - path: "/dst", - mode: 0o777, - mtime: 3, - target: "/target", - }, - ], - new Map(), - ); - - expect(resolveInode(db, "/dst", { followSymlinks: false })?.type).toBe("symlink"); - expect(readlink(db, "/dst")).toBe("/target"); - expect(resolveInode(db, "/dst/sub/old.txt")).toBeNull(); - }); - }); - - it("applies a symlink over one hardlink without removing sibling hardlinks", async () => { - await withDB(async (db) => { - await writeFile(db, "/dst", "shared", {}, () => 1); - link(db, "/dst", "/other"); - - await applyChanges( - db, - [ - { - kind: "symlink", - rev: 99, - path: "/dst", - mode: 0o777, - mtime: 2, - target: "/target", - }, - ], - new Map(), - ); - - expect(resolveInode(db, "/dst", { followSymlinks: false })?.type).toBe("symlink"); - expect(readlink(db, "/dst")).toBe("/target"); - expect(await readFile(db, "/other", "utf8")).toBe("shared"); - }); - }); - - it("applies a directory over one hardlink without removing sibling hardlinks", async () => { - await withDB(async (db) => { - await writeFile(db, "/dst", "shared", {}, () => 1); - link(db, "/dst", "/other"); - - await applyChanges( - db, - [{ kind: "dir", rev: 99, path: "/dst", mode: 0o755, mtime: 2 }], - new Map(), - ); - - expect(resolveInode(db, "/dst", { followSymlinks: false })?.type).toBe("dir"); - expect(await readFile(db, "/other", "utf8")).toBe("shared"); - }); - }); - - it("applies a file over a directory whose subtree hardlinks a file outside it", async () => { - await withDB(async (db) => { - await mkdir(db, "/dir", { mode: 0o755 }, () => 1); - await writeFile(db, "/dir/inner", "shared", {}, () => 1); - // /outside is a second name for /dir/inner; replacing /dir must - // walk the subtree by name and keep /outside alive. - link(db, "/dir/inner", "/outside"); - - await applyChanges( - db, - [ - { - kind: "file", - rev: 99, - path: "/dir", - mode: 0o644, - mtime: 2, - size: 0, - chunks: [], - }, - ], - new Map(), - ); - - expect(resolveInode(db, "/dir", { followSymlinks: false })?.type).toBe("file"); - expect(resolveInode(db, "/dir/inner", { followSymlinks: false })).toBeNull(); - expect(await readFile(db, "/outside", "utf8")).toBe("shared"); - }); - }); - - it("applyChangesSync applies a symlink over a deeply nested existing directory subtree", async () => { - await withDB((db) => { - const oldPath = deepChildPath(12_000); - mkdir(db, oldPath.slice(0, -"/old.txt".length), { recursive: true }, () => 1); - writeFileSync(db, oldPath, new Uint8Array(), {}, () => 2); - - applyChangesSync( - db, - [ - { - kind: "symlink", - rev: 99, - path: "/dst", - mode: 0o777, - mtime: 3, - target: "/target", - }, - ], - new Map(), - ); - - expect(resolveInode(db, "/dst", { followSymlinks: false })?.type).toBe("symlink"); - expect(readlink(db, "/dst")).toBe("/target"); - expect(resolveInode(db, oldPath, { followSymlinks: false })).toBeNull(); - }); - }); - - it("applies a directory over an existing file", async () => { - await withDB(async (db) => { - await writeFile(db, "/dst", "old", {}, () => 1); - - await applyChanges( - db, - [{ kind: "dir", rev: 99, path: "/dst", mode: 0o755, mtime: 2 }], - new Map(), - ); - - expect(resolveInode(db, "/dst", { followSymlinks: false })).toMatchObject({ - type: "dir", - mode: 0o755, - mtime: 2, - }); - }); - }); - - it("applies a directory over an existing symlink", async () => { - await withDB((db) => { - symlink(db, "/target", "/dst", () => 1); - - applyChangesSync( - db, - [{ kind: "dir", rev: 99, path: "/dst", mode: 0o755, mtime: 2 }], - new Map(), - ); - - expect(resolveInode(db, "/dst", { followSymlinks: false })).toMatchObject({ - type: "dir", - mode: 0o755, - mtime: 2, - }); - }); - }); - - it("applies a coalesced file-to-directory replacement", async () => { - await withTwoDBs( - async (a) => { - await writeFile(a, "/dst", "old", {}, () => 1); - const cursor = currentRev(a); - rm(a, "/dst", { force: true }); - mkdir(a, "/dst", {}, () => 2); - const entries = await drain(coalesceChanges(a, cursor)); - expect(entries).toEqual([expect.objectContaining({ kind: "dir", path: "/dst" })]); - return { entries, objects: await collectObjects(a, entries) }; - }, - async (b, { entries, objects }) => { - await writeFile(b, "/dst", "old", {}, () => 1); - - await applyChanges(b, entries, objects); - - expect(resolveInode(b, "/dst", { followSymlinks: false })?.type).toBe("dir"); - }, - ); - }); - - it("handles delete entries", async () => { - await withDB(async (db) => { - await writeFile(db, "/gone.txt", "bye", {}, () => 1); - await applyChanges(db, [{ kind: "delete", rev: 99, path: "/gone.txt" }], new Map()); - expect(resolveInode(db, "/gone.txt")).toBeNull(); - }); - }); -}); - -describe("applyChanges loopback suppression", () => { - it("does not advance pushRev locally on upstream apply", async () => { - await withDB(async (db) => { - // Pre-existing local state: a write the container already - // pushed. pushRev sits at currentRev. - await writeFile(db, "/local.txt", "local", {}, () => 1); - const { currentRev, readWatermark, writeWatermark } = await import("./watermarks.js"); - writeWatermark(db, "pushRev", currentRev(db)); - const beforePushRev = readWatermark(db, "pushRev"); - expect(beforePushRev).toBeGreaterThan(0); - - // Apply an entry as if it came from upstream. The apply's - // writeFile bumps the local rev counter, but pushRev must - // *not* advance with it — advancing locally would move our - // pushRev past entries the remote does not know we have - // shipped, breaking the cross-side invariant on the next - // pull. The next pushOnce re-ships these rev bumps and the - // receiver's alreadyApplied() check drops them. - await applyChanges( - db, - [ - { - kind: "file", - rev: 100, - path: "/from-upstream.txt", - mode: 0o644, - mtime: 2, - size: 0, - chunks: [], - }, - ], - new Map(), - { source: "upstream" }, - ); - - const afterCurrent = currentRev(db); - const afterPushRev = readWatermark(db, "pushRev"); - expect(afterCurrent).toBeGreaterThan(beforePushRev); - expect(afterPushRev).toBe(beforePushRev); - }); - }); - - it("source=local (default) does not advance pushRev", async () => { - await withDB(async (db) => { - const { readWatermark } = await import("./watermarks.js"); - await applyChanges( - db, - [ - { - kind: "file", - rev: 100, - path: "/local.txt", - mode: 0o644, - mtime: 1, - size: 0, - chunks: [], - }, - ], - new Map(), - ); - expect(readWatermark(db, "pushRev")).toBe(0); - }); - }); - - it("upstream entries surface on the next coalesce and rely on receiver-side alreadyApplied", async () => { - await withDB(async (db) => { - const { coalesceChanges } = await import("./coalesce.js"); - const { currentRev, readWatermark, writeWatermark } = await import("./watermarks.js"); - // Seed pushRev at the current point. - writeWatermark(db, "pushRev", currentRev(db)); - - // Upstream sends a file. After apply, pushRev stays where it - // was (the local advance was unsound — see the test above). - // The next coalesceChanges(db, pushRev) re-emits the entry; - // the receiver's alreadyApplied() check drops it. One extra - // round trip per apply, watermarks stay in lockstep. - await applyChanges( - db, - [ - { - kind: "file", - rev: 100, - path: "/upstream.txt", - mode: 0o644, - mtime: 1, - size: 0, - chunks: [], - }, - ], - new Map(), - { source: "upstream" }, - ); - const cursor = readWatermark(db, "pushRev"); - const drained = []; - for await (const e of coalesceChanges(db, cursor)) drained.push(e); - expect(drained.map((e) => e.path)).toContain("/upstream.txt"); - }); - }); -}); - -describe("applyChanges loopback suppression — F1", () => { - // Regression for F1: when local writes are sitting at - // rev > pushRev (i.e. queued for the next push) and an - // upstream pull arrives, the old code advanced pushRev to - // currentRev unconditionally. That stranded the local - // writes — the next pushOnce skipped them as already- - // shipped. Fix: only advance pushRev when the existing - // value already covers everything that existed before this - // apply. - it("does not advance pushRev past unpushed local writes", async () => { - await withDB(async (db) => { - const { currentRev, readWatermark } = await import("./watermarks.js"); - // Simulate an unpushed local write: pushRev stays at - // its initial value (1) but currentRev climbs. - await writeFile(db, "/local.txt", new Uint8Array([1, 2, 3]), { mode: 0o644 }, () => 1); - const revBeforeApply = currentRev(db); - const pushRevBefore = readWatermark(db, "pushRev"); - expect(pushRevBefore).toBeLessThan(revBeforeApply); - - // Upstream sends an entry. alreadyApplied skips it - // (we don't have it locally, so it actually writes — - // pick a path that won't conflict). - await applyChanges( - db, - [ - { - kind: "file", - rev: 100, - path: "/from-upstream.txt", - mode: 0o644, - mtime: 2, - size: 0, - chunks: [], - }, - ], - new Map(), - { source: "upstream" }, - ); - // pushRev must NOT have jumped past the unpushed - // local write. The local write is at revBeforeApply; - // we want pushRev still < revBeforeApply so the next - // pushOnce drains it. - const pushRevAfter = readWatermark(db, "pushRev"); - expect(pushRevAfter).toBeLessThan(revBeforeApply); - // The local write should still appear in coalesce. - const drained = []; - for await (const e of coalesceChanges(db, pushRevAfter)) drained.push(e); - const paths = drained.map((e) => (e.kind === "delete" ? e.path : e.path)); - expect(paths).toContain("/local.txt"); - }); - }); - - it("leaves pushRev alone even when the caller had no unpushed locals", async () => { - await withDB(async (db) => { - const { currentRev, readWatermark, writeWatermark } = await import("./watermarks.js"); - // pushRev already caught up to currentRev: caller has no - // pending local writes. The old apply path advanced pushRev - // here as an optimization; we no longer do that because it - // desynced our pushRev from the remote's fetchRev (echoed - // back as appliedPushRev on the wire). The next pushOnce - // re-ships the apply's rev bump and the receiver's - // alreadyApplied() check drops it. - const before = currentRev(db); - writeWatermark(db, "pushRev", before); - await applyChanges( - db, - [ - { - kind: "file", - rev: 100, - path: "/from-upstream.txt", - mode: 0o644, - mtime: 1, - size: 0, - chunks: [], - }, - ], - new Map(), - { source: "upstream" }, - ); - const after = currentRev(db); - expect(after).toBeGreaterThan(before); - expect(readWatermark(db, "pushRev")).toBe(before); - }); - }); -}); - -describe("applyChanges with read-only mount roots", () => { - function stageReadOnly(db: import("../storage.js").Database, root: string): void { - db.run( - "INSERT INTO _vfs_mounts (root, kind, indexed, mode) VALUES (?, ?, 1, 'read-only')", - root, - "test", - ); - invalidateReadOnlyMountCache(db); - } - - it("skips a write entry under a read-only mount and reports it", async () => { - await withDB(async (db) => { - mkdir(db, "/workspace/r2", { recursive: true }, () => 0); - stageReadOnly(db, "/workspace/r2"); - - const result = await applyChanges( - db, - [ - { - kind: "file", - rev: 100, - path: "/workspace/r2/hello.txt", - mode: 0o644, - mtime: 1, - size: 0, - chunks: [], - }, - ], - new Map(), - ); - - expect(result.applied).toBe(0); - expect(result.skipped).toEqual([ - { - path: "/workspace/r2/hello.txt", - mountRoot: "/workspace/r2", - op: "write", - reason: "read-only", - }, - ]); - // The skipped path is not on disk. - expect(resolveInode(db, "/workspace/r2/hello.txt")).toBeNull(); - }); - }); - - it("skips a delete entry under a read-only mount and reports op:delete", async () => { - await withDB(async (db) => { - mkdir(db, "/workspace/r2", { recursive: true }, () => 0); - stageReadOnly(db, "/workspace/r2"); - - const result = await applyChanges( - db, - [{ kind: "delete", rev: 101, path: "/workspace/r2/gone.txt" }], - new Map(), - ); - - expect(result.applied).toBe(0); - expect(result.skipped).toEqual([ - { - path: "/workspace/r2/gone.txt", - mountRoot: "/workspace/r2", - op: "delete", - reason: "read-only", - }, - ]); - }); - }); - - it("skips an entry whose path is exactly the mount root", async () => { - await withDB(async (db) => { - mkdir(db, "/workspace/r2", { recursive: true }, () => 0); - stageReadOnly(db, "/workspace/r2"); - - const result = await applyChanges( - db, - [ - { - kind: "dir", - rev: 102, - path: "/workspace/r2", - mode: 0o755, - mtime: 1, - }, - ], - new Map(), - ); - - expect(result.applied).toBe(0); - expect(result.skipped[0]?.mountRoot).toBe("/workspace/r2"); - }); - }); - - it("applies entries that lie outside any mount and reports an empty skipped list", async () => { - await withDB(async (db) => { - mkdir(db, "/workspace/r2", { recursive: true }, () => 0); - stageReadOnly(db, "/workspace/r2"); - - mkdir(db, "/scratch", { recursive: true }, () => 0); - - const result = await applyChanges( - db, - [ - { - kind: "file", - rev: 103, - path: "/scratch/ok.txt", - mode: 0o644, - mtime: 1, - size: 0, - chunks: [], - }, - ], - new Map(), - ); - - expect(result.applied).toBe(1); - expect(result.skipped).toEqual([]); - expect(resolveInode(db, "/scratch/ok.txt")).not.toBeNull(); - }); - }); - - it("does not skip entries under a read-write mount", async () => { - await withDB(async (db) => { - mkdir(db, "/workspace/rw", { recursive: true }, () => 0); - db.run( - "INSERT INTO _vfs_mounts (root, kind, indexed, mode) VALUES (?, ?, 1, 'read-write')", - "/workspace/rw", - "test", - ); - invalidateReadOnlyMountCache(db); - - const result = await applyChanges( - db, - [ - { - kind: "file", - rev: 104, - path: "/workspace/rw/ok.txt", - mode: 0o644, - mtime: 1, - size: 0, - chunks: [], - }, - ], - new Map(), - ); - - expect(result.applied).toBe(1); - expect(result.skipped).toEqual([]); - }); - }); - - it("folds skip + apply across a mixed batch", async () => { - await withDB(async (db) => { - mkdir(db, "/workspace/r2", { recursive: true }, () => 0); - mkdir(db, "/scratch", { recursive: true }, () => 0); - stageReadOnly(db, "/workspace/r2"); - - const result = await applyChanges( - db, - [ - { - kind: "file", - rev: 200, - path: "/scratch/a.txt", - mode: 0o644, - mtime: 1, - size: 0, - chunks: [], - }, - { - kind: "file", - rev: 201, - path: "/workspace/r2/blocked.txt", - mode: 0o644, - mtime: 1, - size: 0, - chunks: [], - }, - { - kind: "file", - rev: 202, - path: "/scratch/b.txt", - mode: 0o644, - mtime: 1, - size: 0, - chunks: [], - }, - ], - new Map(), - ); - - expect(result.applied).toBe(2); - expect(result.skipped).toEqual([ - { - path: "/workspace/r2/blocked.txt", - mountRoot: "/workspace/r2", - op: "write", - reason: "read-only", - }, - ]); - }); - }); - - it("applyChangesSync emits the same SkippedEntry shape", async () => { - await withDB(async (db) => { - mkdir(db, "/workspace/r2", { recursive: true }, () => 0); - stageReadOnly(db, "/workspace/r2"); - - const result = applyChangesSync( - db, - [ - { - kind: "file", - rev: 300, - path: "/workspace/r2/blocked.txt", - mode: 0o644, - mtime: 1, - size: 0, - chunks: [], - }, - ], - new Map(), - ); - - expect(result.applied).toBe(0); - expect(result.skipped).toEqual([ - { - path: "/workspace/r2/blocked.txt", - mountRoot: "/workspace/r2", - op: "write", - reason: "read-only", - }, - ]); - }); - }); -}); - -describe("applyChanges mtime propagation (auto_cache contract)", () => { - // The FUSE driver mounts with COMPUTERD_FUSE_AUTO_CACHE=1 in the - // production-safe profile. auto_cache tells the kernel to keep - // file data in the page cache until the file is reopened with a - // different mtime or size; the kernel then drops the cached - // pages and re-reads through FUSE. The whole story rests on - // mtime moving forward whenever the bytes change, including the - // tricky cases where size stays the same. - // - // These tests pin that contract on the sync apply path. If a - // future apply refactor stops propagating mtime, a container - // with auto_cache enabled would keep serving the old bytes - // after a remote push and the cache-coherency story would - // silently break. - - it("bumps destination mtime when a same-size file changes content", async () => { - await withTwoDBs( - async (a) => { - // Source: write the first version, push, then overwrite - // with bytes of the same length but a different value - // and a strictly later mtime. - await writeFile(a, "/note.txt", "alpha", {}, () => 1000); - const first = await drain(coalesceChanges(a, 0)); - const firstObjects = await collectObjects(a, first); - - await writeFile(a, "/note.txt", "OMEGA", {}, () => 2000); - const second = await drain(coalesceChanges(a, Math.max(...first.map((e) => e.rev)))); - const secondObjects = await collectObjects(a, second); - return { first, firstObjects, second, secondObjects }; - }, - async (b, { first, firstObjects, second, secondObjects }) => { - await applyChanges(b, first, firstObjects); - const beforeInode = resolveInode(b, "/note.txt"); - expect(beforeInode).not.toBeNull(); - expect(beforeInode?.mtime).toBe(1000); - expect(await readFile(b, "/note.txt", "utf8")).toBe("alpha"); - - await applyChanges(b, second, secondObjects); - const afterInode = resolveInode(b, "/note.txt"); - expect(afterInode).not.toBeNull(); - // Bytes changed: the kernel must see a strictly newer - // mtime so auto_cache drops the page cache on reopen. - expect(afterInode?.mtime).toBeGreaterThan(beforeInode?.mtime ?? 0); - expect(afterInode?.mtime).toBe(2000); - expect(await readFile(b, "/note.txt", "utf8")).toBe("OMEGA"); - }, - ); - }); - - it("skips an apply when bytes are identical even if the source mtime is newer", async () => { - // The mirror of the test above. If a sender pushes the same - // content with a fresher mtime, the apply path takes the - // alreadyApplied fast path and does not touch the local row. - // The local mtime stays put. This is by design: auto_cache - // only needs to invalidate when bytes change. A pure mtime - // bump would still be safe (the kernel would invalidate and - // re-read identical bytes), but it would burn a local rev - // for nothing. - await withTwoDBs( - async (a) => { - await writeFile(a, "/same.txt", "static", {}, () => 1000); - const first = await drain(coalesceChanges(a, 0)); - const firstObjects = await collectObjects(a, first); - // Re-stamp the same bytes with a newer mtime on the - // source. This isn't something writeFile normally - // produces (it bumps rev and emits the same chunks), but - // it's the worst case a future sender might present. - const reissued = first.map((entry): ChangeEntry => { - if (entry.kind !== "file" || entry.path !== "/same.txt") return entry; - return { ...entry, mtime: 9999, rev: entry.rev + 1000 }; - }); - return { first, firstObjects, reissued }; - }, - async (b, { first, firstObjects, reissued }) => { - // First apply is from upstream too, so the local rev - // counter doesn't claim the bytes as a local write. - await applyChanges(b, first, firstObjects, { source: "upstream" }); - const beforeInode = resolveInode(b, "/same.txt"); - expect(beforeInode?.mtime).toBe(1000); - - // Reapply with a fresh mtime but identical chunks. The - // upstream-source guard runs alreadyApplied(), which - // matches on manifest hash and drops the entry. - await applyChanges(b, reissued, firstObjects, { source: "upstream" }); - const afterInode = resolveInode(b, "/same.txt"); - // alreadyApplied caught the no-op; the mtime stays at - // 1000 because the row never moved. auto_cache's safety - // story is unaffected — the bytes are still the same, - // so a stale page cache would still return correct bytes. - expect(afterInode?.mtime).toBe(1000); - }, - ); - }); - - it("bumps destination mtime when a file shrinks", async () => { - // Size change alone is enough to invalidate auto_cache, but - // the kernel still consults mtime first. Make sure the - // shrinking case carries the new mtime through so a fast - // mtime-cache check at the kernel layer can short-circuit. - await withTwoDBs( - async (a) => { - await writeFile(a, "/shrink.txt", "original-content", {}, () => 1000); - const first = await drain(coalesceChanges(a, 0)); - const firstObjects = await collectObjects(a, first); - - await writeFile(a, "/shrink.txt", "x", {}, () => 2000); - const second = await drain(coalesceChanges(a, Math.max(...first.map((e) => e.rev)))); - const secondObjects = await collectObjects(a, second); - return { first, firstObjects, second, secondObjects }; - }, - async (b, { first, firstObjects, second, secondObjects }) => { - await applyChanges(b, first, firstObjects); - await applyChanges(b, second, secondObjects); - const inode = resolveInode(b, "/shrink.txt"); - expect(inode?.mtime).toBe(2000); - expect(await readFile(b, "/shrink.txt", "utf8")).toBe("x"); - }, - ); - }); - - it("bumps mtime on the sync-style synchronous apply path too", async () => { - // applyChangesSync is the alternate entry point used by - // SyncRPC.fetch's commit phase. It walks the same - // alreadyApplied / writeFileSync path as applyChanges, but - // the two paths are easy to drift apart in a refactor. Lock - // the same mtime-propagation contract on both. - await withTwoDBs( - async (a) => { - await writeFile(a, "/sync.txt", "first", {}, () => 1000); - const first = await drain(coalesceChanges(a, 0)); - const firstObjects = await collectObjects(a, first); - - await writeFile(a, "/sync.txt", "SECOND", {}, () => 2000); - const second = await drain(coalesceChanges(a, Math.max(...first.map((e) => e.rev)))); - const secondObjects = await collectObjects(a, second); - return { first, firstObjects, second, secondObjects }; - }, - async (b, { first, firstObjects, second, secondObjects }) => { - applyChangesSync(b, first, firstObjects); - expect(resolveInode(b, "/sync.txt")?.mtime).toBe(1000); - applyChangesSync(b, second, secondObjects); - expect(resolveInode(b, "/sync.txt")?.mtime).toBe(2000); - expect(await readFile(b, "/sync.txt", "utf8")).toBe("SECOND"); - }, - ); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/sync/apply.ts b/spikes/349-dofs/vendor/dofs/src/sync/apply.ts deleted file mode 100644 index 4986c98e..00000000 --- a/spikes/349-dofs/vendor/dofs/src/sync/apply.ts +++ /dev/null @@ -1,485 +0,0 @@ -import { mkdir } from "../fs/mkdir.js"; -import { readOnlyRootFor } from "../fs/mount-guard.js"; -import { resolveInode } from "../fs/resolve.js"; -import { invalidateResolveSubtree } from "../fs/resolveCache.js"; -import { rm } from "../fs/rm.js"; -import { symlink } from "../fs/symlink.js"; -import { unlinkDirent } from "../fs/unlink.js"; -import { writeFile, writeFileSync } from "../fs/writeFile.js"; -import { canonicalizePath } from "../path.js"; -import { incrementRev } from "../rev.js"; -import type { Database } from "../storage.js"; -import type { ChangeEntry } from "./changes.js"; -import { computeManifestHash } from "./manifests.js"; - -// One container-side change that landed under a read-only mount and -// was therefore skipped rather than applied. Callers (the workspace -// pull surface, the shell exec bracket) surface these so the user -// learns the mount stayed authoritative. -export interface SkippedEntry { - // Absolute VFS path the change targeted. - path: string; - // Mount root that owns the path (the one whose mode is - // read-only). Lets callers group skipped entries by mount. - mountRoot: string; - // 'write' covers file / dir / symlink create-or-update; 'delete' - // covers tombstones. The single field is enough for callers to - // decide messaging. - op: "write" | "delete"; - // Open shape: future skip reasons can join this union without - // breaking callers that match on 'read-only' today. - reason: "read-only"; -} - -// Return shape of applyChanges / applyChangesSync. Existing callers -// that only wanted a count read `result.applied`; new callers can -// surface `result.skipped`. -export interface ApplyResult { - // Entries written through writeFile / mkdir / symlink / rm. - applied: number; - // Entries dropped because they targeted a read-only mount root. - // Empty when no such mounts are registered or the stream stayed - // clear of them. - skipped: SkippedEntry[]; -} - -export interface ApplyOptions { - // Soft cap on bytes written per transactionSync batch. Default 64 - // MiB; matches docs/02_sync_protocol.md. The cap is advisory: a - // single large file is always one batch. - maxBytesPerBatch?: number; - // Soft cap on entries per batch. Default 1024 paths. - maxPathsPerBatch?: number; - // Where the entries came from. 'local' (default) treats the apply - // path like any other mutation: writeFile/mkdir/etc bump - // vfs_meta.rev and the push loop later ships those new revs - // upstream. 'upstream' is informational: the apply still bumps - // rev so readers see fresh data, and the next pushOnce ships - // those rev bumps back to the sender. Loop convergence is the - // receiver's job — the apply path on the original sender uses - // alreadyApplied() to drop the redundant entries without bumping - // rev further, bounding the echo at one extra round trip per - // upstream apply. - source?: "local" | "upstream"; - // Backend id whose watermark row this apply should touch. The - // DO hosts independent sync cursors per backend; threading the - // id through here keeps a pull from backend A from bumping - // backend B's pushRev. Defaults to the dofs `default` slot, - // which is fine for the container backend the package shipped - // with first. - backend?: string; -} - -const DEFAULT_MAX_BYTES = 64 * 1024 * 1024; -const DEFAULT_MAX_PATHS = 1024; - -type NodeType = "file" | "dir" | "symlink"; - -function hex(bytes: Uint8Array): string { - let s = ""; - for (let i = 0; i < bytes.byteLength; i++) s += bytes[i].toString(16).padStart(2, "0"); - return s; -} - -function removeReplaceableFinalEntry( - db: Database, - path: string, - incomingKind: "file" | "symlink", -): void { - const existing = resolveInode(db, path, { followSymlinks: false }); - if (existing === null) return; - if (incomingKind === "file" && existing.type === "file") return; - - removeInodeTreeAtPath(db, path, existing.inode, existing.type); -} - -// Structural conflict cleanup for upstream applies. This removes -// the local shape without recording tombstones because the incoming -// entry is the authoritative state for this path. -function removeInodeTreeAtPath(db: Database, path: string, inode: number, type: NodeType): void { - // Structural subtree removal that bypasses rm() and calls unlinkDirent - // directly, so it must drop cached resolutions itself. One subtree - // drop at the root covers every descendant the walk unlinks. - // Canonicalize to the exact key readers cache under (every other hook - // already passes a canonical path; this one takes an entry path). - invalidateResolveSubtree(db, canonicalizePath(path).path); - const root = direntForPath(db, path, inode); - const stack: Array<{ - path: string; - parentInode: number; - name: string; - inode: number; - type: NodeType; - expanded: boolean; - }> = [ - { - path, - parentInode: root.parentInode, - name: root.name, - inode, - type, - expanded: false, - }, - ]; - - while (stack.length > 0) { - const current = stack.pop(); - if (current === undefined) break; - - if (current.type === "dir" && !current.expanded) { - const children = db.all<{ name: string; child_inode: number; type: NodeType }>( - `SELECT d.name AS name, d.child_inode AS child_inode, n.type AS type - FROM vfs_dirents d - JOIN vfs_nodes n ON n.inode = d.child_inode - WHERE d.parent_inode = ?`, - current.inode, - ); - stack.push({ ...current, expanded: true }); - for (let i = children.length - 1; i >= 0; i--) { - const child = children[i]; - const childPath = current.path === "/" ? `/${child.name}` : `${current.path}/${child.name}`; - stack.push({ - path: childPath, - parentInode: current.inode, - name: child.name, - inode: child.child_inode, - type: child.type, - expanded: false, - }); - } - continue; - } - - // Unlink this one name and reap the inode only when its last link - // disappears, so a sibling hardlink (inside or outside the subtree) - // keeps the file alive. (parent, name) is unique, so this removes - // exactly the dirent the walk is visiting. - unlinkDirent(db, current.parentInode, current.name, current.inode, current.type); - } -} - -function direntForPath( - db: Database, - path: string, - inode: number, -): { parentInode: number; name: string } { - const { parts, path: canonical } = canonicalizePath(path); - if (parts.length === 0) { - throw new Error(`applyChanges: cannot structurally replace root ${canonical}`); - } - const name = parts[parts.length - 1]; - const parentPath = parts.length === 1 ? "/" : `/${parts.slice(0, -1).join("/")}`; - const parent = resolveInode(db, parentPath, { followSymlinks: false }); - if (parent === null || parent.type !== "dir") { - throw new Error(`applyChanges: parent missing for structural replacement ${canonical}`); - } - const child = db.scalar( - "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", - parent.inode, - name, - ); - if (child !== inode) { - throw new Error(`applyChanges: dirent mismatch for structural replacement ${canonical}`); - } - return { parentInode: parent.inode, name }; -} - -function applyDirectoryEntry(db: Database, entry: Extract): void { - const mode = entry.mode & 0o7777; - const existing = resolveInode(db, entry.path, { followSymlinks: false }); - if (existing === null) { - mkdir(db, entry.path, { mode, recursive: true }, () => entry.mtime); - return; - } - if (existing.type !== "dir") { - removeInodeTreeAtPath(db, entry.path, existing.inode, existing.type); - mkdir(db, entry.path, { mode, recursive: true }, () => entry.mtime); - return; - } - - db.transactionSync(() => { - const rev = incrementRev(db); - db.run( - "UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ? WHERE inode = ?", - mode, - entry.mtime, - rev, - existing.inode, - ); - }); -} - -// Drive a ChangeEntry stream against `db`, batching writes so peak -// memory stays bounded and a crash mid-apply leaves the DB in a -// consistent state. Each batch runs inside a single transactionSync -// from the underlying FS helpers — mkdir, writeFile, symlink, -// rm all wrap their own transactionSync, so a batch is in practice -// a sequence of independently-committed mutations rather than one -// fat transaction. The bounded-batch contract still holds because -// fetchRev only advances after the stream drains. -// -// `objects` is a hash-keyed map of chunk bytes the sender shipped -// via pushObjects / fetchObjects. File entries reassemble their -// chunks from this map; missing entries throw. -export async function applyChanges( - db: Database, - entries: Iterable | AsyncIterable, - objects: Map, - options: ApplyOptions = {}, -): Promise { - const maxBytes = options.maxBytesPerBatch ?? DEFAULT_MAX_BYTES; - const maxPaths = options.maxPathsPerBatch ?? DEFAULT_MAX_PATHS; - - let bytesInBatch = 0; - let pathsInBatch = 0; - let applied = 0; - const skipped: SkippedEntry[] = []; - const flush = () => { - bytesInBatch = 0; - pathsInBatch = 0; - }; - - for await (const entry of entries) { - // Idempotent skip: if the entry already matches the local - // state, drop it on the floor. The check is what stops a - // pull from bumping vfs_meta.rev for entries that are - // already in place, which in turn stops the next push from - // re-shipping them. - if (options.source === "upstream" && entry.kind !== "delete") { - if (alreadyApplied(db, entry)) continue; - } - // Read-only mount guard. Entries under a registered read-only - // mount root are surfaced via the return value and not applied. - // The owning workspace's surface (Workspace.pull, exec()) folds - // these into its own return so callers see what stayed - // authoritative on the mount. - const blockingRoot = readOnlyRootFor(db, entry.path); - if (blockingRoot !== undefined) { - skipped.push({ - path: entry.path, - mountRoot: blockingRoot, - op: entry.kind === "delete" ? "delete" : "write", - reason: "read-only", - }); - continue; - } - if (entry.kind === "delete") { - try { - rm(db, entry.path, { recursive: true, force: true }); - } catch { - // Already gone is fine — idempotent apply. - } - applied++; - pathsInBatch++; - if (pathsInBatch >= maxPaths) flush(); - continue; - } - if (entry.kind === "dir") { - applyDirectoryEntry(db, entry); - applied++; - pathsInBatch++; - if (pathsInBatch >= maxPaths) flush(); - continue; - } - if (entry.kind === "symlink") { - removeReplaceableFinalEntry(db, entry.path, "symlink"); - symlink(db, entry.target, entry.path, () => entry.mtime); - applied++; - pathsInBatch++; - if (pathsInBatch >= maxPaths) flush(); - continue; - } - // file: assemble chunk bytes. First check the in-memory map - // (the streaming hand-off); fall back to vfs_blob_bytes (the - // staged-via-pushObjects path). - const parts: Uint8Array[] = []; - let total = 0; - for (const c of entry.chunks) { - const k = hex(c.hash); - let bytes = objects.get(k); - if (bytes === undefined) { - const row = db.one<{ bytes: Uint8Array }>( - "SELECT bytes FROM vfs_blob_bytes WHERE hash = ?", - c.hash, - ); - bytes = row?.bytes; - } - if (bytes === undefined) { - throw new Error(`applyChanges: missing object ${k} for ${entry.path}`); - } - parts.push(bytes); - total += bytes.byteLength; - } - const buf = new Uint8Array(total); - let off = 0; - for (const p of parts) { - buf.set(p, off); - off += p.byteLength; - } - removeReplaceableFinalEntry(db, entry.path, "file"); - await writeFile(db, entry.path, buf, { mode: entry.mode }, () => entry.mtime); - applied++; - bytesInBatch += total; - pathsInBatch++; - if (bytesInBatch >= maxBytes || pathsInBatch >= maxPaths) flush(); - } - - // Loopback suppression used to advance pushRev locally after an - // upstream apply so the next push tick wouldn't re-ship the rev - // bumps the apply produced. That optimization is unsound: it - // moves the *local* pushRev past entries the remote does not - // know we have shipped, while the remote's fetchRev (echoed back - // as appliedPushRev on every fetchChanges) stays where it was. - // The cross-side invariant check in pullOnce then trips on the - // very next pull and the post-drain pullOnce in the exec bracket - // swallows the error, leaving every subsequent container-side - // write invisible to the host until something reconciles. - // - // The bounded "redundant round-trip" the old comment promised is - // still bounded, and the receiver's alreadyApplied() check still - // suppresses the entries on the next pushOnce. We just pay one - // extra push per upstream apply to keep the two sides in lockstep. - return { applied, skipped }; -} - -// Synchronous variant of applyChanges. Same semantics; takes an -// in-memory entry array instead of an iterable. Used on the push -// receiver so the whole batch can run inside a single transactionSync -// and a mid-stream failure rolls back every prior entry. -// -// Stays separate from applyChanges so the streaming pull path -// (which can't hold a sync transaction across network I/O) keeps -// its async semantics. -export function applyChangesSync( - db: Database, - entries: readonly ChangeEntry[], - objects: Map, - options: ApplyOptions = {}, -): ApplyResult { - const maxBytes = options.maxBytesPerBatch ?? DEFAULT_MAX_BYTES; - const maxPaths = options.maxPathsPerBatch ?? DEFAULT_MAX_PATHS; - - let bytesInBatch = 0; - let pathsInBatch = 0; - let applied = 0; - const skipped: SkippedEntry[] = []; - const flush = () => { - bytesInBatch = 0; - pathsInBatch = 0; - }; - - for (const entry of entries) { - if (options.source === "upstream" && entry.kind !== "delete") { - if (alreadyApplied(db, entry)) continue; - } - const blockingRoot = readOnlyRootFor(db, entry.path); - if (blockingRoot !== undefined) { - skipped.push({ - path: entry.path, - mountRoot: blockingRoot, - op: entry.kind === "delete" ? "delete" : "write", - reason: "read-only", - }); - continue; - } - if (entry.kind === "delete") { - try { - rm(db, entry.path, { recursive: true, force: true }); - } catch { - // Already gone is fine — idempotent apply. - } - applied++; - pathsInBatch++; - if (pathsInBatch >= maxPaths) flush(); - continue; - } - if (entry.kind === "dir") { - applyDirectoryEntry(db, entry); - applied++; - pathsInBatch++; - if (pathsInBatch >= maxPaths) flush(); - continue; - } - if (entry.kind === "symlink") { - removeReplaceableFinalEntry(db, entry.path, "symlink"); - symlink(db, entry.target, entry.path, () => entry.mtime); - applied++; - pathsInBatch++; - if (pathsInBatch >= maxPaths) flush(); - continue; - } - const parts: Uint8Array[] = []; - let total = 0; - for (const c of entry.chunks) { - const k = hex(c.hash); - let bytes = objects.get(k); - if (bytes === undefined) { - const row = db.one<{ bytes: Uint8Array }>( - "SELECT bytes FROM vfs_blob_bytes WHERE hash = ?", - c.hash, - ); - bytes = row?.bytes; - } - if (bytes === undefined) { - throw new Error(`applyChanges: missing object ${k} for ${entry.path}`); - } - parts.push(bytes); - total += bytes.byteLength; - } - const buf = new Uint8Array(total); - let off = 0; - for (const p of parts) { - buf.set(p, off); - off += p.byteLength; - } - removeReplaceableFinalEntry(db, entry.path, "file"); - writeFileSync(db, entry.path, buf, { mode: entry.mode }, () => entry.mtime); - applied++; - bytesInBatch += total; - pathsInBatch++; - if (bytesInBatch >= maxBytes || pathsInBatch >= maxPaths) flush(); - } - - // See applyChanges() for why pushRev no longer advances locally - // on upstream applies. The receiver's alreadyApplied() check - // suppresses the redundant entries on the next pushOnce; one - // extra push per apply keeps the cross-side invariant intact. - - return { applied, skipped }; -} - -// Compare an entry against the local node graph. Returns true when -// the entry would be a no-op apply: the manifest hash (files), mode -// (dirs), or mode + symlink target (symlinks) already matches. -function alreadyApplied(db: Database, entry: Exclude): boolean { - const live = resolveInode(db, entry.path, { followSymlinks: false }); - if (live === null) return false; - - if (entry.kind === "file") { - if (live.type !== "file") return false; - const row = db.one<{ manifest_hash: Uint8Array | null }>( - "SELECT manifest_hash FROM vfs_nodes WHERE inode = ?", - live.inode, - ); - if (!row?.manifest_hash) return false; - const wanted = computeManifestHash(entry.chunks); - return uint8Equal(row.manifest_hash, wanted); - } - if (entry.kind === "dir") { - return live.type === "dir" && (live.mode & 0o7777) === (entry.mode & 0o7777); - } - // symlink - return ( - live.type === "symlink" && - live.linkTarget === entry.target && - (live.mode & 0o7777) === (entry.mode & 0o7777) - ); -} - -function uint8Equal(a: Uint8Array, b: Uint8Array): boolean { - if (a.byteLength !== b.byteLength) return false; - for (let i = 0; i < a.byteLength; i++) { - if (a[i] !== b[i]) return false; - } - return true; -} diff --git a/spikes/349-dofs/vendor/dofs/src/sync/blobs.test.ts b/spikes/349-dofs/vendor/dofs/src/sync/blobs.test.ts deleted file mode 100644 index e9568596..00000000 --- a/spikes/349-dofs/vendor/dofs/src/sync/blobs.test.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { createHash } from "node:crypto"; -import { describe, expect, it } from "vitest"; -import { gc } from "../fs/gc.js"; -import { withDB } from "../fs/with-db.js"; -import { writeFile } from "../fs/writeFile.js"; -import { Database } from "../storage.js"; -import { stageBlob } from "./blobs.js"; - -function sha256(bytes: Uint8Array): Uint8Array { - return new Uint8Array(createHash("sha256").update(bytes).digest()); -} - -describe("stageBlob", () => { - it("lands a chunk into vfs_blobs + vfs_blob_bytes", async () => { - await withDB(async (db) => { - const bytes = new TextEncoder().encode("payload"); - const hash = sha256(bytes); - stageBlob(db, hash, bytes, 1234); - const blob = db.one<{ size: number; last_seen: number }>( - "SELECT size, last_seen FROM vfs_blobs WHERE hash = ?", - hash, - ); - expect(blob?.size).toBe(7); - expect(blob?.last_seen).toBe(1234); - const row = db.one<{ bytes: Uint8Array }>( - "SELECT bytes FROM vfs_blob_bytes WHERE hash = ?", - hash, - ); - expect(row?.bytes).toEqual(bytes); - }); - }); - - it("rolls back metadata and bytes when storing the bytes fails", async () => { - await withDB(async (db) => { - const bytes = new TextEncoder().encode("payload"); - const hash = sha256(bytes); - const failingDb = new Database({ - sql: { - exec: (query: string, ...bindings: unknown[]) => { - if (query.startsWith("INSERT INTO vfs_blob_bytes")) { - throw new Error("injected bytes failure"); - } - return db.sql.exec(query, ...bindings); - }, - }, - transactionSync: (closure) => db.transactionSync(closure), - }); - - expect(() => stageBlob(failingDb, hash, bytes, 1234)).toThrow("injected bytes failure"); - expect(db.scalar("SELECT COUNT(*) FROM vfs_blobs")).toBe(0); - expect(db.scalar("SELECT COUNT(*) FROM vfs_blob_bytes")).toBe(0); - }); - }); - - it("is idempotent: a second call refreshes last_seen but leaves bytes alone", async () => { - await withDB(async (db) => { - const bytes = new TextEncoder().encode("same"); - const hash = sha256(bytes); - stageBlob(db, hash, bytes, 100); - stageBlob(db, hash, bytes, 200); - const blob = db.one<{ last_seen: number }>( - "SELECT last_seen FROM vfs_blobs WHERE hash = ?", - hash, - ); - expect(blob?.last_seen).toBe(200); - // Still one row. - const count = db.scalar("SELECT COUNT(*) FROM vfs_blob_bytes"); - expect(count).toBe(1); - }); - }); - - it("a staged-but-unreferenced blob is reaped by gc outside the safety window", async () => { - await withDB(async (db) => { - const bytes = new TextEncoder().encode("orphan"); - const hash = sha256(bytes); - stageBlob(db, hash, bytes, 100); - // Inside the safety window, gc preserves it. - expect(gc(db, { now: () => 200, safetyWindowMs: 1000 }).blobsFreed).toBe(0); - // Outside, gc reaps it. - expect(gc(db, { now: () => 5000, safetyWindowMs: 1000 }).blobsFreed).toBe(1); - }); - }); - - it("dedups against a blob already written by writeFile", async () => { - await withDB(async (db) => { - const bytes = new TextEncoder().encode("shared"); - const hash = sha256(bytes); - await writeFile(db, "/a.txt", "shared", {}, () => 1); - // The writeFile path already wrote the blob with that hash. - const before = db.scalar("SELECT COUNT(*) FROM vfs_blob_bytes"); - stageBlob(db, hash, bytes, 5000); - const after = db.scalar("SELECT COUNT(*) FROM vfs_blob_bytes"); - expect(after).toBe(before); - // last_seen on the blob row has been bumped. - const ls = db.scalar("SELECT last_seen FROM vfs_blobs WHERE hash = ?", hash); - expect(ls).toBe(5000); - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/sync/blobs.ts b/spikes/349-dofs/vendor/dofs/src/sync/blobs.ts deleted file mode 100644 index d51fabe5..00000000 --- a/spikes/349-dofs/vendor/dofs/src/sync/blobs.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { clearBlobCache } from "../fs/blobCache.js"; -import type { Database } from "../storage.js"; - -// Stage a chunk directly into vfs_blobs + vfs_blob_bytes without -// creating a node or a manifest. The receiver-side push path uses -// this to land bytes the sender shipped via pushObjects so a -// subsequent applyChanges call can find them by hash. -// -// Idempotent: a second call with the same hash refreshes -// last_seen so the bytes don't get reaped by an interleaved gc. -// Conflict updates also repair incomplete or size-mismatched rows -// left by an interrupted or corrupt write. -// -// Callers are expected to have verified that hash === sha256(bytes) -// before calling. The function trusts the caller; a mismatched -// pair would silently land under the wrong key. -export function stageBlob(db: Database, hash: Uint8Array, bytes: Uint8Array, now: number): void { - db.transactionSync(() => { - db.run( - "INSERT INTO vfs_blobs (hash, size, last_seen) VALUES (?, ?, ?) ON CONFLICT(hash) DO UPDATE SET size = excluded.size, last_seen = excluded.last_seen", - hash, - bytes.byteLength, - now, - ); - db.run( - "INSERT INTO vfs_blob_bytes (hash, bytes) VALUES (?, ?) ON CONFLICT(hash) DO UPDATE SET bytes = excluded.bytes", - hash, - bytes, - ); - }); - clearBlobCache(db); -} diff --git a/spikes/349-dofs/vendor/dofs/src/sync/changes.test.ts b/spikes/349-dofs/vendor/dofs/src/sync/changes.test.ts deleted file mode 100644 index e685b6c4..00000000 --- a/spikes/349-dofs/vendor/dofs/src/sync/changes.test.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { mkdir } from "../fs/mkdir.js"; -import { rm } from "../fs/rm.js"; -import { symlink } from "../fs/symlink.js"; -import { withDB } from "../fs/with-db.js"; -import { writeFile } from "../fs/writeFile.js"; -import { materialiseChange } from "./changes.js"; - -describe("materialiseChange", () => { - it("returns a file entry with chunk hashes for a written file", async () => { - await withDB(async (db) => { - await writeFile(db, "/hello.txt", "hello world", { mode: 0o644 }, () => 1234); - const entry = materialiseChange(db, "/hello.txt"); - expect(entry).toMatchObject({ - kind: "file", - path: "/hello.txt", - mode: 0o644, - mtime: 1234, - size: 11, - }); - if (entry?.kind !== "file") throw new Error("expected file"); - expect(entry.rev).toBeGreaterThan(0); - if (entry?.kind !== "file") throw new Error("expected file"); - expect(entry.chunks).toHaveLength(1); - expect(entry.chunks[0].size).toBe(11); - expect(entry.chunks[0].hash).toBeInstanceOf(Uint8Array); - expect(entry.chunks[0].hash.byteLength).toBe(32); - }); - }); - - it("returns a dir entry for a created directory", async () => { - await withDB(async (db) => { - mkdir(db, "/sub", { mode: 0o755 }, () => 999); - expect(materialiseChange(db, "/sub")).toEqual({ - kind: "dir", - rev: expect.any(Number), - path: "/sub", - mode: 0o755, - mtime: 999, - }); - }); - }); - - it("returns a symlink entry for a symlink", async () => { - await withDB(async (db) => { - await writeFile(db, "/target.txt", "x", {}, () => 1); - symlink(db, "/target.txt", "/link", () => 2); - expect(materialiseChange(db, "/link")).toEqual({ - kind: "symlink", - rev: expect.any(Number), - path: "/link", - target: "/target.txt", - // chmod on a symlink itself is platform-specific; we record - // whatever symlink() stamped on vfs_nodes. - mode: expect.any(Number), - mtime: 2, - }); - }); - }); - - it("returns a delete entry for a tombstoned path", async () => { - await withDB(async (db) => { - await writeFile(db, "/gone.txt", "bye", {}, () => 1); - rm(db, "/gone.txt", {}); - expect(materialiseChange(db, "/gone.txt")).toEqual({ - kind: "delete", - rev: expect.any(Number), - path: "/gone.txt", - }); - }); - }); - - it("returns null for a path that was never written and has no tombstone", async () => { - await withDB(async (db) => { - expect(materialiseChange(db, "/never")).toBeNull(); - }); - }); - - it("size reflects the sum of chunk sizes", async () => { - await withDB(async (db) => { - // Force two chunks: 600 KiB > CHUNK_SIZE (512 KiB). - const bytes = new Uint8Array(600 * 1024); - for (let i = 0; i < bytes.byteLength; i++) bytes[i] = i & 0xff; - await writeFile(db, "/big.bin", bytes, {}, () => 1); - const entry = materialiseChange(db, "/big.bin"); - if (entry?.kind !== "file") throw new Error("expected file"); - expect(entry.chunks).toHaveLength(2); - expect(entry.size).toBe(600 * 1024); - expect(entry.chunks[0].size + entry.chunks[1].size).toBe(600 * 1024); - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/sync/changes.ts b/spikes/349-dofs/vendor/dofs/src/sync/changes.ts deleted file mode 100644 index 2cef2326..00000000 --- a/spikes/349-dofs/vendor/dofs/src/sync/changes.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { resolveInode } from "../fs/resolve.js"; -import { canonicalizePath } from "../path.js"; -import type { Database } from "../storage.js"; - -// Record a tombstone for a deleted path so the next push to the -// container learns the path is gone. Called by fs/rm inside the same -// transaction that bumped rev and removed the inode rows; the caller -// passes the post-bump rev value. -export function recordDelete(db: Database, rev: number, path: string): void { - db.run("INSERT INTO vfs_changes (rev, path, op) VALUES (?, ?, 'delete')", rev, path); -} - -// One row of the sync wire. The DO pushes these to the container -// and the container fetches them back. Bytes are never inline: -// file entries carry chunk hashes and the receiver does its own -// hasObjects probe + fetchObjects pull for the bytes it lacks. -// -// `rev` is the sender's currentRev at the moment this entry was -// stamped — vfs_nodes.rev for live mutations, vfs_changes.rev for -// tombstones. The puller uses it as a per-entry cursor so it can -// advance fetchRev per committed batch instead of waiting for the -// whole stream to drain. -export type ChangeEntry = - | { - kind: "file"; - rev: number; - path: string; - mode: number; - mtime: number; - size: number; - chunks: { hash: Uint8Array; size: number }[]; - } - | { kind: "dir"; rev: number; path: string; mode: number; mtime: number } - | { - kind: "symlink"; - rev: number; - path: string; - target: string; - mode: number; - mtime: number; - } - | { kind: "delete"; rev: number; path: string }; - -// Read the current state of `path` and turn it into a wire entry. -// Returns null when the path was never touched (no live inode and no -// tombstone in vfs_changes). Live inodes win over tombstones, which -// handles the delete-then-recreate case correctly. -// -// Symlinks are returned as symlink entries; we never follow them on -// the sync wire. Callers that want "the file the link points at" -// resolve it themselves after applying the symlink entry. -export function materialiseChange(db: Database, path: string): ChangeEntry | null { - const canonical = canonicalizePath(path).path; - const live = resolveInode(db, canonical, { followSymlinks: false }); - if (live !== null) { - // Read the rev stamped on this inode. Used as the per-entry - // cursor on the sync wire; coalesceChanges yields entries in - // ascending rev order so the puller can checkpoint per batch. - const revRow = db.one<{ rev: number }>("SELECT rev FROM vfs_nodes WHERE inode = ?", live.inode); - const rev = revRow?.rev ?? 0; - if (live.type === "dir") { - return { kind: "dir", rev, path: canonical, mode: live.mode, mtime: live.mtime }; - } - if (live.type === "symlink") { - return { - kind: "symlink", - rev, - path: canonical, - target: live.linkTarget ?? "", - mode: live.mode, - mtime: live.mtime, - }; - } - // file: collect chunk rows in index order. Each row carries hash - // and size so the receiver can probe hasObjects without a - // separate manifest lookup. An empty file has zero chunk rows - // and reports size 0. - const chunks = db.all<{ hash: Uint8Array; size: number }>( - "SELECT hash, size FROM vfs_chunks WHERE inode = ? ORDER BY idx", - live.inode, - ); - let size = 0; - for (const c of chunks) size += c.size; - return { - kind: "file", - rev, - path: canonical, - mode: live.mode, - mtime: live.mtime, - size, - chunks, - }; - } - // No live inode — check for a tombstone. The last row wins if the - // path was deleted and never recreated; an indexed scan by path is - // cheap because vfs_changes is bounded by the watermark window. - const tomb = db.one<{ rev: number; op: string }>( - "SELECT rev, op FROM vfs_changes WHERE path = ? ORDER BY id DESC LIMIT 1", - canonical, - ); - if (tomb?.op === "delete") { - return { kind: "delete", rev: tomb.rev, path: canonical }; - } - return null; -} diff --git a/spikes/349-dofs/vendor/dofs/src/sync/coalesce.test.ts b/spikes/349-dofs/vendor/dofs/src/sync/coalesce.test.ts deleted file mode 100644 index 3bbc9749..00000000 --- a/spikes/349-dofs/vendor/dofs/src/sync/coalesce.test.ts +++ /dev/null @@ -1,342 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { link } from "../fs/link.js"; -import { mkdir } from "../fs/mkdir.js"; -import { rename } from "../fs/rename.js"; -import { rm } from "../fs/rm.js"; -import { symlink } from "../fs/symlink.js"; -import { withDB } from "../fs/with-db.js"; -import { writeFile, writeFileSync } from "../fs/writeFile.js"; -import { coalesceChanges } from "./coalesce.js"; -import { currentRev } from "./watermarks.js"; - -// Drain an async iterable into an array. Tests stay synchronous-looking -// while the production code can stream. -async function drain(it: AsyncIterable): Promise { - const out: T[] = []; - for await (const x of it) out.push(x); - return out; -} - -describe("coalesceChanges", () => { - it("yields nothing for an empty rev window", async () => { - await withDB(async (db) => { - const entries = await drain(coalesceChanges(db, 1_000_000)); - expect(entries).toEqual([]); - }); - }); - - it("yields one entry per touched path after the cursor", async () => { - await withDB(async (db) => { - mkdir(db, "/d", { mode: 0o755 }, () => 1); - await writeFile(db, "/d/a.txt", "alpha", {}, () => 2); - const entries = await drain(coalesceChanges(db, 0)); - // root mkdir is implicit (already exists); we should see /d and /d/a.txt. - const paths = entries.map((e) => e.path).sort(); - expect(paths).toContain("/d"); - expect(paths).toContain("/d/a.txt"); - }); - }); - - it("coalesces five rewrites of the same path into one entry", async () => { - await withDB(async (db) => { - for (let i = 0; i < 5; i++) { - await writeFile(db, "/log.txt", `pass ${i}`, {}, () => 100 + i); - } - const entries = await drain(coalesceChanges(db, 0)); - const log = entries.filter((e) => e.path === "/log.txt"); - expect(log).toHaveLength(1); - // Latest state wins: the entry carries the size of "pass 4" (6 bytes). - expect(log[0]).toMatchObject({ kind: "file", size: 6 }); - }); - }); - - it("emits delete entries for tombstoned paths", async () => { - await withDB(async (db) => { - await writeFile(db, "/gone.txt", "x", {}, () => 1); - rm(db, "/gone.txt", {}); - const entries = await drain(coalesceChanges(db, 0)); - const gone = entries.find((e) => e.path === "/gone.txt"); - expect(gone).toEqual({ kind: "delete", rev: expect.any(Number), path: "/gone.txt" }); - }); - }); - - it("emits resolved delete paths for removes through intermediate symlinks", async () => { - await withDB(async (db) => { - mkdir(db, "/real", {}, () => 1); - await writeFile(db, "/real/file.txt", "x", {}, () => 2); - symlink(db, "/real", "/link", () => 3); - - rm(db, "/link/file.txt", {}); - - const entries = await drain(coalesceChanges(db, 0)); - expect(entries).toContainEqual({ - kind: "delete", - rev: expect.any(Number), - path: "/real/file.txt", - }); - expect(entries).not.toContainEqual({ - kind: "delete", - rev: expect.any(Number), - path: "/link/file.txt", - }); - }); - }); - - it("delete-then-recreate yields a single live entry, not a delete", async () => { - await withDB(async (db) => { - await writeFile(db, "/x.txt", "first", {}, () => 1); - rm(db, "/x.txt", {}); - await writeFile(db, "/x.txt", "second", {}, () => 2); - const entries = await drain(coalesceChanges(db, 0)); - const x = entries.filter((e) => e.path === "/x.txt"); - expect(x).toHaveLength(1); - expect(x[0].kind).toBe("file"); - }); - }); - - it("cursor rev filters out changes the receiver has already seen", async () => { - await withDB(async (db) => { - await writeFile(db, "/old.txt", "old", {}, () => 1); - // Read current rev counter to use as the cursor. - const cursor = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; - await writeFile(db, "/new.txt", "new", {}, () => 2); - const entries = await drain(coalesceChanges(db, cursor)); - const paths = entries.map((e) => e.path); - expect(paths).toContain("/new.txt"); - expect(paths).not.toContain("/old.txt"); - }); - }); - - it("yields entries in ascending rev order", async () => { - // pullOnce uses entry.rev as a per-batch checkpoint cursor. The - // contract is monotonicity: if entry N has rev R, every entry - // already emitted has rev <= R. Without that, the puller can't - // advance fetchRev mid-stream without risking a skip on the next - // resume. - await withDB(async (db) => { - await writeFile(db, "/a.txt", "a", {}, () => 1); - mkdir(db, "/d", {}, () => 1); - await writeFile(db, "/d/b.txt", "b", {}, () => 1); - await writeFile(db, "/c.txt", "c", {}, () => 1); - rm(db, "/a.txt", {}); - await writeFile(db, "/d/b.txt", "b2", {}, () => 2); - const entries = await drain(coalesceChanges(db, 0)); - const revs = entries.map((e) => e.rev); - for (let i = 1; i < revs.length; i++) { - expect(revs[i]).toBeGreaterThanOrEqual(revs[i - 1]); - } - }); - }); - - it("resumes after a path within the same rev", async () => { - await withDB(async (db) => { - mkdir(db, "/src", {}, () => 1); - await writeFile(db, "/src/a.txt", "a", {}, () => 2); - await writeFile(db, "/src/b.txt", "b", {}, () => 3); - const beforeRename = currentRev(db); - - rename(db, "/src", "/dst"); - const renameRev = currentRev(db); - expect(renameRev).toBeGreaterThan(beforeRename); - - const allSameRev = await drain(coalesceChanges(db, { rev: beforeRename, path: null })); - const paths = allSameRev.map((entry) => entry.path); - expect(paths).toEqual([...paths].sort()); - expect(new Set(allSameRev.map((entry) => entry.rev))).toEqual(new Set([renameRev])); - - const resumed = await drain(coalesceChanges(db, { rev: renameRev, path: "/dst/a.txt" })); - expect(resumed.map((entry) => entry.path)).toEqual( - paths.filter((path) => path > "/dst/a.txt"), - ); - }); - }); - - it("skips a whole rev when the cursor path is null", async () => { - await withDB(async (db) => { - mkdir(db, "/src", {}, () => 1); - await writeFile(db, "/src/a.txt", "a", {}, () => 2); - const beforeRename = currentRev(db); - - rename(db, "/src", "/dst"); - const renameRev = currentRev(db); - expect(await drain(coalesceChanges(db, { rev: beforeRename, path: null }))).not.toEqual([]); - expect(await drain(coalesceChanges(db, { rev: renameRev, path: null }))).toEqual([]); - }); - }); - - it("orders entries deterministically by rev then path", async () => { - await withDB(async (db) => { - mkdir(db, "/src", {}, () => 1); - await writeFile(db, "/src/b.txt", "b", {}, () => 2); - await writeFile(db, "/src/a.txt", "a", {}, () => 3); - rename(db, "/src", "/dst"); - - const entries = await drain(coalesceChanges(db, { rev: 0, path: null })); - const pairs = entries.map((entry) => [entry.rev, entry.path] as const); - expect(pairs).toEqual( - [...pairs].sort((a, b) => { - if (a[0] !== b[0]) return a[0] - b[0]; - return a[1].localeCompare(b[1]); - }), - ); - }); - }); - - it("excludes entries newer than the through rev", async () => { - await withDB(async (db) => { - await writeFile(db, "/included.txt", "included", {}, () => 1); - const throughRev = currentRev(db); - await writeFile(db, "/excluded.txt", "excluded", {}, () => 2); - - const entries = await drain( - coalesceChanges(db, { rev: 0, path: null }, { through: { rev: throughRev, path: null } }), - ); - - expect(entries.map((entry) => entry.path)).toContain("/included.txt"); - expect(entries.map((entry) => entry.path)).not.toContain("/excluded.txt"); - }); - }); - - it("excludes same-rev entries after the through path", async () => { - await withDB(async (db) => { - mkdir(db, "/src", {}, () => 1); - await writeFile(db, "/src/a.txt", "a", {}, () => 2); - await writeFile(db, "/src/b.txt", "b", {}, () => 3); - const beforeRename = currentRev(db); - - rename(db, "/src", "/dst"); - const renameRev = currentRev(db); - - const entries = await drain( - coalesceChanges( - db, - { rev: beforeRename, path: null }, - { - through: { rev: renameRev, path: "/dst/a.txt" }, - }, - ), - ); - - expect(entries.map((entry) => entry.path)).toEqual(["/dst", "/dst/a.txt"]); - }); - }); - - it("skips an entry that materializes beyond the through cursor", async () => { - await withDB(async (db) => { - await writeFile(db, "/file.txt", "first", {}, () => 1); - const throughRev = currentRev(db); - - const originalOne = db.one.bind(db); - let rewrote = false; - db.one = ((query: string, ...bindings: unknown[]) => { - if (!rewrote && query === "SELECT rev FROM vfs_nodes WHERE inode = ?") { - rewrote = true; - writeFileSync(db, "/file.txt", new TextEncoder().encode("second"), {}, () => 2); - } - return originalOne(query, ...bindings); - }) as typeof db.one; - - const entries = await drain( - coalesceChanges(db, { rev: 0, path: null }, { through: { rev: throughRev, path: null } }), - ); - - expect(rewrote).toBe(true); - expect(entries).toEqual([]); - }); - }); -}); - -describe("coalesceChanges (ignore)", () => { - it("drops entries whose path contains an ignored segment", async () => { - await withDB(async (db) => { - mkdir(db, "/src", {}, () => 0); - mkdir(db, "/node_modules", {}, () => 0); - mkdir(db, "/node_modules/lodash", {}, () => 0); - mkdir(db, "/a", {}, () => 0); - mkdir(db, "/a/node_modules", {}, () => 0); - mkdir(db, "/a/node_modules/p", {}, () => 0); - await writeFile(db, "/src/index.ts", "x", {}, () => 1); - await writeFile(db, "/node_modules/lodash/index.js", "y", {}, () => 2); - await writeFile(db, "/a/node_modules/p/q.js", "z", {}, () => 3); - const entries = await drain(coalesceChanges(db, 0, { ignore: ["node_modules"] })); - const paths = entries.map((e) => e.path); - expect(paths).toContain("/src/index.ts"); - for (const p of paths) { - expect(p.includes("node_modules")).toBe(false); - } - }); - }); - - it("an empty ignore list is the default behaviour", async () => { - await withDB(async (db) => { - mkdir(db, "/node_modules", {}, () => 0); - await writeFile(db, "/node_modules/x.js", "x", {}, () => 1); - const entries = await drain(coalesceChanges(db, 0)); - expect(entries.some((e) => e.path === "/node_modules/x.js")).toBe(true); - }); - }); - - it("emits an entry for every hardlink name of a touched inode", async () => { - await withDB(async (db) => { - await writeFile(db, "/a", "shared", {}, () => 1); - link(db, "/a", "/b"); - - const entries = await drain(coalesceChanges(db, 0)); - const files = entries.filter((e) => e.kind === "file").map((e) => e.path); - // Both names share one inode; pathOf would pick only one of them. - // The wire has to carry both so the receiver materialises each. - expect(files).toContain("/a"); - expect(files).toContain("/b"); - }); - }); - - it("defers a path whose live state raced past the snapshot, then redelivers it", async () => { - // Pins the documented cursor contract (docs/02_sync_protocol.md): - // `through` is a resume bound, not a point-in-time snapshot. A path - // deleted at the snapshot rev but recreated above it is dropped from - // the bounded scan, because materialiseChange reads the live state - // and inCursorWindow filters it out. The omission is not loss — the - // recreate's rev is above the snapshot, so the next scan redelivers - // the path. Convergence holds without the store keeping history. - await withDB(async (db) => { - await writeFile(db, "/x", "v1", {}, () => 1); - rm(db, "/x", {}); - const snapshot = currentRev(db); - await writeFile(db, "/x", "v2", {}, () => 2); - - // Bounded by the snapshot: the rev-`snapshot` delete is a - // candidate, but the live entry now sits above the window, so /x - // is omitted from this snapshot rather than frozen at the delete. - const bounded = await drain( - coalesceChanges(db, 0, { through: { rev: snapshot, path: null } }), - ); - expect(bounded.some((e) => e.path === "/x")).toBe(false); - - // The next scan resumes after the snapshot rev and redelivers /x - // at its current state. The rev that caused the drop is above the - // snapshot, so a later window always covers it. - const next = await drain(coalesceChanges(db, snapshot)); - expect(next.find((e) => e.path === "/x")).toMatchObject({ kind: "file", path: "/x" }); - }); - }); - - it("emits the new name when a hardlinked file is renamed", async () => { - await withDB(async (db) => { - await writeFile(db, "/a", "shared", {}, () => 1); - link(db, "/a", "/b"); - // /a and /b now share an inode. Renaming /a to /c leaves the - // inode named /b and /c; pathOf might resolve it to /b and never - // emit /c, dropping the renamed name on the wire. - const baseline = currentRev(db); - rename(db, "/a", "/c"); - - const entries = await drain(coalesceChanges(db, baseline)); - const live = entries.filter((e) => e.kind !== "delete").map((e) => e.path); - const deletes = entries.filter((e) => e.kind === "delete").map((e) => e.path); - expect(live).toContain("/c"); - expect(live).toContain("/b"); - expect(deletes).toContain("/a"); - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/sync/coalesce.ts b/spikes/349-dofs/vendor/dofs/src/sync/coalesce.ts deleted file mode 100644 index bfd4b304..00000000 --- a/spikes/349-dofs/vendor/dofs/src/sync/coalesce.ts +++ /dev/null @@ -1,141 +0,0 @@ -import type { Database } from "../storage.js"; -import { type ChangeEntry, materialiseChange } from "./changes.js"; -import { isIgnored } from "./ignore.js"; -import { pathsOf } from "./paths.js"; -import { type ChangeCursor, compareChangeCursors } from "./watermarks.js"; - -// Yield one ChangeEntry per path touched after `after`. Per-path -// coalescing: five rewrites of the same path between watermarks -// produce one entry (the latest state wins). Tombstoned paths get a -// delete entry unless they have been recreated, in which case the -// live entry wins. -// -// Entries are emitted in ascending rev order. pullOnce relies on -// this so it can advance fetchRev per committed batch — if entry N -// has rev R, every entry already emitted has rev <= R, so -// checkpointing fetchRev at R is safe. -// -// Streaming, not buffering at the wire: the per-path coalesce step -// holds at most one slot per dirty path in memory (sorted by rev), -// which is the same bound the live + tombstone scans already pay. -export interface CoalesceOptions { - // Path-segment patterns to drop before yielding. The wire never - // carries entries under an ignored segment. - ignore?: string[]; - // Internal snapshot bound. When set, entries are limited to the - // cursor window `after < entry <= through`. The bound is applied to - // each path's *live* rev at materialise time, not to a frozen - // snapshot: a path whose state moves past `through` after the scan - // (a concurrent rewrite or delete) is dropped here and redelivered - // under a later cursor, since the cursor never advances past the rev - // that caused the drop. See the redelivery note on the yield loop. - through?: ChangeCursor; -} - -export async function* coalesceChanges( - db: Database, - after: ChangeCursor | number, - options: CoalesceOptions = {}, -): AsyncIterable { - const ignore = options.ignore ?? []; - const cursor = typeof after === "number" ? { rev: after, path: null } : after; - const through = options.through; - - // Build the per-path candidate set in two passes, keeping the - // highest rev seen for each path. A live mutation that landed - // after a tombstone wins; a tombstone that landed after a write - // wins; the highest rev is the rev we stamp on the wire and the - // rev pullOnce checkpoints to. - type Candidate = { path: string; rev: number }; - const candidates = new Map(); - - // Live mutations: every mkdir / writeFile / symlink bumps - // vfs_nodes.rev. The by_rev index makes this a range scan. - const lowerRev = cursor.path === null ? cursor.rev : cursor.rev - 1; - const touched = - through === undefined - ? db.all<{ inode: number; rev: number }>( - "SELECT inode, rev FROM vfs_nodes WHERE rev > ? ORDER BY rev", - lowerRev, - ) - : db.all<{ inode: number; rev: number }>( - "SELECT inode, rev FROM vfs_nodes WHERE rev > ? AND rev <= ? ORDER BY rev", - lowerRev, - through.rev, - ); - for (const { inode, rev } of touched) { - // One inode can carry several hardlink names; every name has to - // become a candidate so the wire materialises each, not just the - // arbitrary one pathOf would return. - for (const path of pathsOf(db, inode)) { - if (!inCursorWindow({ rev, path }, cursor, through)) continue; - if (isIgnored(path, ignore)) continue; - const prior = candidates.get(path); - if (prior === undefined || rev > prior.rev) { - candidates.set(path, { path, rev }); - } - } - } - - // Tombstones: each rm appends a row to vfs_changes with the - // post-bump rev. The highest rev per path wins (a path can be - // deleted-recreated-deleted; we want the last rm's rev). - const tombs = - through === undefined - ? db.all<{ path: string; rev: number }>( - "SELECT path, MAX(rev) AS rev FROM vfs_changes WHERE rev > ? AND op = 'delete' GROUP BY path", - lowerRev, - ) - : db.all<{ path: string; rev: number }>( - "SELECT path, MAX(rev) AS rev FROM vfs_changes WHERE rev > ? AND rev <= ? AND op = 'delete' GROUP BY path", - lowerRev, - through.rev, - ); - for (const { path, rev } of tombs) { - if (!inCursorWindow({ rev, path }, cursor, through)) continue; - if (isIgnored(path, ignore)) continue; - const prior = candidates.get(path); - if (prior === undefined || rev > prior.rev) { - candidates.set(path, { path, rev }); - } - } - - // Sort by rev ascending so pullOnce can checkpoint per batch. - // Ties on rev (same transactionSync touching multiple paths) - // break on path so the wire order is deterministic. - const ordered = Array.from(candidates.values()).sort((a, b) => { - if (a.rev !== b.rev) return a.rev - b.rev; - return a.path < b.path ? -1 : a.path > b.path ? 1 : 0; - }); - - // materialiseChange reads each path's *current* state, not the state - // it held at the candidate's rev — the VFS keeps no content history. - // If a path was rewritten or deleted again after the scan, its live - // rev can now sit above `through`; inCursorWindow drops it. The - // dropped change is not lost: the rev that pushed it past `through` - // is, by definition, greater than `through.rev` (the cursor the - // puller persists), so the next pull's `after < entry` scan finds the - // path again and delivers its then-current state. The consequence is - // that a `{rev, null}` cursor means "every change committed at or - // before rev has been *offered*"; it does not guarantee the receiver - // tree byte-matches the rev snapshot for a path that raced ahead. - // Convergence is preserved because the cursor never advances past the - // racing rev. See docs/02_sync_protocol.md. - for (const { path } of ordered) { - const entry = materialiseChange(db, path); - if (entry !== null && inCursorWindow(entry, cursor, through)) { - yield entry; - } - } -} - -function inCursorWindow( - entry: ChangeCursor & { path: string }, - after: ChangeCursor, - through?: ChangeCursor, -): boolean { - return ( - compareChangeCursors(entry, after) > 0 && - (through === undefined || compareChangeCursors(entry, through) <= 0) - ); -} diff --git a/spikes/349-dofs/vendor/dofs/src/sync/fetch.test.ts b/spikes/349-dofs/vendor/dofs/src/sync/fetch.test.ts deleted file mode 100644 index 01b295e2..00000000 --- a/spikes/349-dofs/vendor/dofs/src/sync/fetch.test.ts +++ /dev/null @@ -1,155 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { withDB } from "../fs/with-db.js"; -import { createFileSync, writeFile, writeRangeSync } from "../fs/writeFile.js"; -import { coalesceChanges } from "./coalesce.js"; -import { fetchChanges, fetchObjects, hasObjects } from "./fetch.js"; - -async function drain(it: AsyncIterable): Promise { - const out: T[] = []; - for await (const x of it) out.push(x); - return out; -} - -describe("fetch wire", () => { - it("fetchChanges yields the same entries as coalesceChanges", async () => { - await withDB(async (db) => { - await writeFile(db, "/a.txt", "alpha", {}, () => 1); - await writeFile(db, "/b.txt", "beta", {}, () => 2); - const viaCoalesce = await drain(coalesceChanges(db, 0)); - const viaFetch = await drain(fetchChanges(db, 0)); - expect(viaFetch).toEqual(viaCoalesce); - }); - }); - - it("fetchChanges and fetchObjects include small direct writes", async () => { - await withDB(async (db) => { - createFileSync(db, "/inline.txt", {}, () => 1); - writeRangeSync(db, "/inline.txt", new TextEncoder().encode("inline direct"), 0, {}, () => 2); - - const entries = await drain(fetchChanges(db, 0)); - const file = entries.find((entry) => entry.kind === "file" && entry.path === "/inline.txt"); - expect(file).toMatchObject({ kind: "file", size: "inline direct".length }); - expect(file?.kind === "file" ? file.chunks : []).toHaveLength(1); - const hash = file?.kind === "file" ? file.chunks[0].hash : new Uint8Array(); - const objects = await drain(fetchObjects(db, [hash])); - expect(objects).toHaveLength(1); - expect(new TextDecoder().decode(objects[0].bytes)).toBe("inline direct"); - }); - }); - - it("fetchChanges resumes from a rev/path cursor", async () => { - await withDB(async (db) => { - await writeFile(db, "/a.txt", "alpha", {}, () => 1); - await writeFile(db, "/b.txt", "beta", {}, () => 2); - const entries = await drain(fetchChanges(db, { rev: 0, path: null })); - const first = entries[0]; - const resumed = await drain(fetchChanges(db, { rev: first.rev, path: first.path })); - expect(resumed).toEqual(entries.slice(1)); - }); - }); - - it("fetchObjects yields each hash exactly once", async () => { - await withDB(async (db) => { - await writeFile(db, "/a.txt", "shared", {}, () => 1); - await writeFile(db, "/b.txt", "shared", {}, () => 2); - // Pick the single chunk hash from the file entry. - const entries = await drain(fetchChanges(db, 0)); - const hashes: Uint8Array[] = []; - for (const e of entries) { - if (e.kind === "file") hashes.push(...e.chunks.map((c) => c.hash)); - } - // Two files, same content, so two references to one hash. - expect(hashes).toHaveLength(2); - const seen = new Set(); - for await (const { hash, bytes } of fetchObjects(db, [hashes[0]])) { - seen.add(Array.from(hash).join(",")); - expect(new TextDecoder().decode(bytes)).toBe("shared"); - } - expect(seen.size).toBe(1); - }); - }); -}); - -describe("hasObjects", () => { - it("returns the subset of inputs the receiver already holds", async () => { - await withDB(async (db) => { - await writeFile(db, "/a.txt", "alpha", {}, () => 1); - const entries = await drain(fetchChanges(db, 0)); - const known = (entries.find((e) => e.kind === "file") as { chunks: { hash: Uint8Array }[] }) - .chunks[0].hash; - const unknown = new Uint8Array(32); - unknown.fill(0xff); - const got = hasObjects(db, [known, unknown]); - expect(got).toHaveLength(1); - expect(got[0]).toEqual(known); - }); - }); - - it("treats blob metadata without bytes as missing", async () => { - await withDB(async (db) => { - await writeFile(db, "/a.txt", "alpha", {}, () => 1); - const entries = await drain(fetchChanges(db, 0)); - const known = ( - entries.find((e) => e.kind === "file") as { - chunks: { hash: Uint8Array }[]; - } - ).chunks[0].hash; - db.run("DELETE FROM vfs_blob_bytes WHERE hash = ?", known); - - expect(hasObjects(db, [known])).toEqual([]); - }); - }); - - it("treats blob bytes with the wrong length as missing", async () => { - await withDB(async (db) => { - await writeFile(db, "/a.txt", "alpha", {}, () => 1); - const entries = await drain(fetchChanges(db, 0)); - const known = ( - entries.find((e) => e.kind === "file") as { - chunks: { hash: Uint8Array }[]; - } - ).chunks[0].hash; - db.run("UPDATE vfs_blob_bytes SET bytes = ? WHERE hash = ?", new Uint8Array([1]), known); - - expect(hasObjects(db, [known])).toEqual([]); - }); - }); - - it("returns an empty array when nothing matches", async () => { - await withDB(async (db) => { - const zero = new Uint8Array(32); - expect(hasObjects(db, [zero])).toEqual([]); - }); - }); - - it("returns an empty array when no hashes are passed", async () => { - await withDB(async (db) => { - expect(hasObjects(db, [])).toEqual([]); - }); - }); - - it("preserves input order and duplicates across mixed inputs", async () => { - await withDB(async (db) => { - await writeFile(db, "/a.txt", "alpha", {}, () => 1); - await writeFile(db, "/b.txt", "beta", {}, () => 2); - const entries = await drain(fetchChanges(db, 0)); - const hashOf = (path: string): Uint8Array => { - const e = entries.find((x) => x.kind === "file" && x.path === path); - return e?.kind === "file" ? e.chunks[0].hash : new Uint8Array(); - }; - const a = hashOf("/a.txt"); - const b = hashOf("/b.txt"); - const missing = new Uint8Array(32).fill(0xff); - - // All present, returned in input order. - expect(hasObjects(db, [a, b])).toEqual([a, b]); - // Mixed: only present hashes, still in input order. - expect(hasObjects(db, [missing, b, a])).toEqual([b, a]); - // A duplicated present hash keeps every occurrence. - expect(hasObjects(db, [a, a, missing])).toEqual([a, a]); - // Duplicated absent hashes drop entirely. - expect(hasObjects(db, [missing, missing])).toEqual([]); - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/sync/fetch.ts b/spikes/349-dofs/vendor/dofs/src/sync/fetch.ts deleted file mode 100644 index 317540fe..00000000 --- a/spikes/349-dofs/vendor/dofs/src/sync/fetch.ts +++ /dev/null @@ -1,64 +0,0 @@ -import type { Database } from "../storage.js"; -import type { ChangeEntry } from "./changes.js"; -import { coalesceChanges } from "./coalesce.js"; -import { pushObjects } from "./push.js"; -import type { ChangeCursor } from "./watermarks.js"; - -// The fetch wire is the mirror of the push wire: same SQL, -// opposite direction. The DO calls fetchChanges / fetchObjects on -// the container; the container calls push / pushObjects on the DO. -// Both names exist so call sites read in their own direction. - -export function fetchChanges( - db: Database, - after: ChangeCursor | number, - options: { ignore?: string[] } = {}, -): AsyncIterable { - return coalesceChanges(db, after, options); -} - -export function fetchObjects( - db: Database, - hashes: Uint8Array[], -): AsyncIterable<{ hash: Uint8Array; bytes: Uint8Array }> { - return pushObjects(db, hashes); -} - -// Stable hex key for JS-side membership tests. Only content matters -// here; the SQL match is on the raw hash blob. -function toHex(bytes: Uint8Array): string { - let out = ""; - for (const b of bytes) out += b.toString(16).padStart(2, "0"); - return out; -} - -// Largest hash list bound into one IN (…) probe. Comfortably under -// SQLite's bound-parameter limit, so a large probe splits into a few -// index-backed lookups instead of one oversized statement. -const PROBE_BATCH = 256; - -// Subset-test the input hashes against complete local objects. A -// metadata row alone is not enough: the payload must exist and its -// byte length must match the advertised blob size. -// -// Matches the raw hash blobs through an IN (…) list so the lookup -// rides the primary-key index. Present hashes are returned in input -// order, preserving any duplicates the caller passed. -export function hasObjects(db: Database, hashes: Uint8Array[]): Uint8Array[] { - if (hashes.length === 0) return []; - const present = new Set(); - for (let i = 0; i < hashes.length; i += PROBE_BATCH) { - const window = hashes.slice(i, i + PROBE_BATCH); - const placeholders = window.map(() => "?").join(", "); - const rows = db.all<{ hash: Uint8Array }>( - `SELECT b.hash - FROM vfs_blobs b - JOIN vfs_blob_bytes bb ON bb.hash = b.hash - WHERE b.hash IN (${placeholders}) - AND length(bb.bytes) = b.size`, - ...window, - ); - for (const row of rows) present.add(toHex(row.hash)); - } - return hashes.filter((h) => present.has(toHex(h))); -} diff --git a/spikes/349-dofs/vendor/dofs/src/sync/ignore.test.ts b/spikes/349-dofs/vendor/dofs/src/sync/ignore.test.ts deleted file mode 100644 index 37975760..00000000 --- a/spikes/349-dofs/vendor/dofs/src/sync/ignore.test.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { isIgnored } from "./ignore.js"; - -describe("isIgnored", () => { - const list = ["node_modules", ".next", "target"]; - - it("returns false for paths that don't intersect the list", () => { - expect(isIgnored("/src/index.ts", list)).toBe(false); - expect(isIgnored("/README.md", list)).toBe(false); - expect(isIgnored("/", list)).toBe(false); - }); - - it("matches the segment exactly, not as a substring", () => { - expect(isIgnored("/node_modules", list)).toBe(true); - expect(isIgnored("/node_modules_old", list)).toBe(false); - expect(isIgnored("/my_node_modules", list)).toBe(false); - }); - - it("matches anywhere in the path", () => { - expect(isIgnored("/a/b/node_modules", list)).toBe(true); - expect(isIgnored("/a/b/node_modules/c.js", list)).toBe(true); - expect(isIgnored("/packages/x/node_modules/y/index.js", list)).toBe(true); - }); - - it("matches nested ignored dirs too", () => { - expect(isIgnored("/a/.next/cache", list)).toBe(true); - expect(isIgnored("/rust/target/debug/foo", list)).toBe(true); - }); - - it("returns false for an empty list", () => { - expect(isIgnored("/anywhere/node_modules", [])).toBe(false); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/sync/ignore.ts b/spikes/349-dofs/vendor/dofs/src/sync/ignore.ts deleted file mode 100644 index 91f7b841..00000000 --- a/spikes/349-dofs/vendor/dofs/src/sync/ignore.ts +++ /dev/null @@ -1,24 +0,0 @@ -// Path segment matcher for the container-side ignore list. The -// container uses this to drop paths from coalesceChanges before -// they hit the wire; the DO's Workspace.fs surface uses the same -// helper to make ignored paths invisible to API consumers. -// -// Matching is whole-segment: "node_modules" matches the segment -// node_modules anywhere in the path but does not match -// node_modules_old or my_node_modules. Patterns are plain strings, -// not globs; we can extend to globs later if a real case demands it. - -export const DEFAULT_IGNORE = ["node_modules"]; - -export function isIgnored(path: string, patterns: string[]): boolean { - if (patterns.length === 0) return false; - // canonicalizePath strips the trailing slash and leaves a leading - // "/" for non-root paths; split skips the empty leading segment. - const segments = path.split("/").filter((s) => s.length > 0); - for (const segment of segments) { - for (const p of patterns) { - if (segment === p) return true; - } - } - return false; -} diff --git a/spikes/349-dofs/vendor/dofs/src/sync/invariant.test.ts b/spikes/349-dofs/vendor/dofs/src/sync/invariant.test.ts deleted file mode 100644 index 7e40d9f3..00000000 --- a/spikes/349-dofs/vendor/dofs/src/sync/invariant.test.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { assertAppliedPushCursor } from "./invariant.js"; - -describe("assertAppliedPushCursor", () => { - it("passes when applied covers pushed", () => { - expect(() => - assertAppliedPushCursor({ rev: 10, path: null }, { rev: 10, path: null }), - ).not.toThrow(); - expect(() => - assertAppliedPushCursor({ rev: 11, path: "/partial.txt" }, { rev: 10, path: null }), - ).not.toThrow(); - }); - - it("passes at zero", () => { - expect(() => - assertAppliedPushCursor({ rev: 0, path: null }, { rev: 0, path: null }), - ).not.toThrow(); - }); - - it("throws when the receiver only partially applied the pushed rev", () => { - expect(() => - assertAppliedPushCursor({ rev: 10, path: "/partial.txt" }, { rev: 10, path: null }), - ).toThrowError(/appliedPushCursor.*pushCursor/i); - }); - - it("throws when the receiver is behind the sender's push cursor", () => { - expect(() => - assertAppliedPushCursor({ rev: 5, path: null }, { rev: 10, path: null }), - ).toThrowError(/appliedPushCursor.*pushCursor/i); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/sync/invariant.ts b/spikes/349-dofs/vendor/dofs/src/sync/invariant.ts deleted file mode 100644 index 036fd0e7..00000000 --- a/spikes/349-dofs/vendor/dofs/src/sync/invariant.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { type ChangeCursor, compareChangeCursors } from "./watermarks.js"; - -// Cross-side invariant: every fetchChanges and push response carries -// the receiver's current applied push cursor. The sender asserts that -// cursor covers its local push cursor on every response. -// -// The two sides never share a single clock, but echoing the largest -// applied sender cursor makes the "receiver is caught up with our -// pushes" invariant inspectable on the wire instead of load-bearing -// in-process state. A regression in the suppress-dirty-tracking apply -// path trips the assertion immediately rather than corrupting data -// silently. -// -// Throwing an Error is the right escalation: a violation means the -// protocol is broken; the connection should tear down and rebuild -// rather than soldiering on with stale state. - -export function assertAppliedPushCursor( - appliedPushCursor: ChangeCursor, - pushCursor: ChangeCursor, -): void { - if (compareChangeCursors(appliedPushCursor, pushCursor) < 0) { - throw new Error( - `cross-side invariant violated: appliedPushCursor (${JSON.stringify( - appliedPushCursor, - )}) < pushCursor (${JSON.stringify(pushCursor)})`, - ); - } -} diff --git a/spikes/349-dofs/vendor/dofs/src/sync/manifests.test.ts b/spikes/349-dofs/vendor/dofs/src/sync/manifests.test.ts deleted file mode 100644 index 15a649d5..00000000 --- a/spikes/349-dofs/vendor/dofs/src/sync/manifests.test.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { withDB } from "../fs/with-db.js"; -import { writeFile } from "../fs/writeFile.js"; - -describe("manifests", () => { - it("writeFile sets vfs_nodes.manifest_hash to a non-null hash", async () => { - await withDB(async (db) => { - await writeFile(db, "/hello.txt", "hello", {}, () => 1); - const row = db.one<{ manifest_hash: Uint8Array | null }>( - "SELECT manifest_hash FROM vfs_nodes WHERE inode = (SELECT child_inode FROM vfs_dirents WHERE parent_inode = 1 AND name = 'hello.txt')", - ); - expect(row?.manifest_hash).toBeInstanceOf(Uint8Array); - expect(row?.manifest_hash?.byteLength).toBe(32); - }); - }); - - it("identical content at two paths shares one manifest row", async () => { - await withDB(async (db) => { - await writeFile(db, "/a.txt", "same bytes", {}, () => 1); - await writeFile(db, "/b.txt", "same bytes", {}, () => 1); - const count = db.scalar("SELECT COUNT(*) AS n FROM vfs_manifests"); - expect(count).toBe(1); - // Both inodes point at the same manifest_hash. - const hashes = db.all<{ manifest_hash: Uint8Array | null }>( - "SELECT manifest_hash FROM vfs_nodes WHERE type = 'file' ORDER BY inode", - ); - expect(hashes).toHaveLength(2); - expect(hashes[0].manifest_hash).toEqual(hashes[1].manifest_hash); - }); - }); - - it("different content produces different manifest hashes", async () => { - await withDB(async (db) => { - await writeFile(db, "/a.txt", "one", {}, () => 1); - await writeFile(db, "/b.txt", "two", {}, () => 1); - const count = db.scalar("SELECT COUNT(*) AS n FROM vfs_manifests"); - expect(count).toBe(2); - }); - }); - - it("vfs_manifests row carries the chunk list as JSON", async () => { - await withDB(async (db) => { - await writeFile(db, "/hi.txt", "hi", {}, () => 1); - const row = db.one<{ encoded: Uint8Array; size: number }>( - "SELECT encoded, size FROM vfs_manifests LIMIT 1", - ); - expect(row).toBeDefined(); - expect(row?.size).toBe(2); - const decoded = JSON.parse(new TextDecoder().decode(row?.encoded)); - expect(decoded.version).toBe(1); - expect(decoded.chunks).toHaveLength(1); - expect(decoded.chunks[0].size).toBe(2); - expect(typeof decoded.chunks[0].hash).toBe("string"); - expect(decoded.chunks[0].hash).toMatch(/^[0-9a-f]{64}$/); - }); - }); - - it("overwriting a file updates manifest_hash and may leave the old manifest orphaned", async () => { - await withDB(async (db) => { - await writeFile(db, "/x.txt", "first", {}, () => 1); - const before = db.one<{ manifest_hash: Uint8Array | null }>( - "SELECT manifest_hash FROM vfs_nodes WHERE type = 'file'", - ); - await writeFile(db, "/x.txt", "second", {}, () => 2); - const after = db.one<{ manifest_hash: Uint8Array | null }>( - "SELECT manifest_hash FROM vfs_nodes WHERE type = 'file'", - ); - expect(after?.manifest_hash).not.toEqual(before?.manifest_hash); - // Two manifest rows now exist; the old one is orphaned and will - // be reaped by gc(). - const count = db.scalar("SELECT COUNT(*) AS n FROM vfs_manifests"); - expect(count).toBe(2); - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/sync/manifests.ts b/spikes/349-dofs/vendor/dofs/src/sync/manifests.ts deleted file mode 100644 index 98dfcfbc..00000000 --- a/spikes/349-dofs/vendor/dofs/src/sync/manifests.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { createHash } from "node:crypto"; - -import type { Database } from "../storage.js"; - -// A manifest names the ordered chunk list for a single file. Two -// files whose bytes chunk identically share one manifest row, which -// is what lets the sync wire say "this file is the same as the one -// I just sent you" by hash alone. -// -// Encoding is JSON for now — readable, debuggable, and structurally -// identical to casync's `.caidx`. A future commit can swap the -// encoding to the `.caidx` byte layout without a schema change. - -export interface ManifestChunk { - hash: Uint8Array; - size: number; -} - -export const MANIFEST_VERSION = 1; - -interface EncodedManifest { - version: number; - chunks: { hash: string; size: number }[]; -} - -function toHex(bytes: Uint8Array): string { - let out = ""; - for (let i = 0; i < bytes.byteLength; i++) { - out += bytes[i].toString(16).padStart(2, "0"); - } - return out; -} - -function sha256(bytes: Uint8Array): Uint8Array { - return new Uint8Array(createHash("sha256").update(bytes).digest()); -} - -// Serialize a chunk list into the canonical manifest bytes. The -// hash is taken over these bytes and the same bytes are stored, so -// producing them once keeps the two in step. -function encodeManifest(chunks: ManifestChunk[]): Uint8Array { - const encoded: EncodedManifest = { - version: MANIFEST_VERSION, - chunks: chunks.map((c) => ({ hash: toHex(c.hash), size: c.size })), - }; - return new TextEncoder().encode(JSON.stringify(encoded)); -} - -// Compute the manifest hash for a chunk list without touching the -// DB. Used by the apply path to short-circuit when an upstream -// entry already matches the local node — the manifest hash is -// content-addressed so identical chunks always produce the same -// hash. -export function computeManifestHash(chunks: ManifestChunk[]): Uint8Array { - return sha256(encodeManifest(chunks)); -} - -// Build a manifest row for the given chunk list. Idempotent: a -// second call with the same chunks no-ops on the UNIQUE(hash). The -// returned hash is what the caller writes onto -// `vfs_nodes.manifest_hash`. -export function buildManifest(db: Database, chunks: ManifestChunk[], now: number): Uint8Array { - const bytes = encodeManifest(chunks); - const hash = sha256(bytes); - const size = chunks.reduce((acc, c) => acc + c.size, 0); - db.run( - "INSERT INTO vfs_manifests (hash, size, encoded, last_seen) VALUES (?, ?, ?, ?) ON CONFLICT(hash) DO UPDATE SET last_seen = excluded.last_seen", - hash, - size, - bytes, - now, - ); - return hash; -} diff --git a/spikes/349-dofs/vendor/dofs/src/sync/paths.ts b/spikes/349-dofs/vendor/dofs/src/sync/paths.ts deleted file mode 100644 index d68a1e14..00000000 --- a/spikes/349-dofs/vendor/dofs/src/sync/paths.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { ROOT_INODE } from "../schema/index.js"; -import type { Database } from "../storage.js"; - -// Walk vfs_dirents from `inode` up to ROOT_INODE, gathering the path -// segments along the way. Returns null when the inode is unreachable. -export function pathOf(db: Database, inode: number): string | null { - if (inode === ROOT_INODE) return "/"; - const segments: string[] = []; - let current = inode; - // Bound the walk: a million levels deep is well past any real FS; - // anything beyond that is corruption and should not loop forever. - for (let i = 0; i < 1_000_000; i++) { - const row = db.one<{ parent_inode: number; name: string }>( - "SELECT parent_inode, name FROM vfs_dirents WHERE child_inode = ?", - current, - ); - if (row === undefined) return null; - segments.push(row.name); - if (row.parent_inode === ROOT_INODE) { - segments.reverse(); - return `/${segments.join("/")}`; - } - current = row.parent_inode; - } - return null; -} - -// Every path that currently names `inode`. A file may carry several -// hardlink names; pathOf collapses them to one arbitrary name, which -// is wrong for the change stream — every name has to reach the wire so -// the receiver materialises each. Directories cannot be hardlinked, so -// each parent walk is unambiguous. -export function pathsOf(db: Database, inode: number): string[] { - if (inode === ROOT_INODE) return ["/"]; - const dirents = db.all<{ parent_inode: number; name: string }>( - "SELECT parent_inode, name FROM vfs_dirents WHERE child_inode = ?", - inode, - ); - const paths: string[] = []; - for (const { parent_inode, name } of dirents) { - const parent = pathOf(db, parent_inode); - if (parent === null) continue; - paths.push(parent === "/" ? `/${name}` : `${parent}/${name}`); - } - return paths; -} diff --git a/spikes/349-dofs/vendor/dofs/src/sync/push.test.ts b/spikes/349-dofs/vendor/dofs/src/sync/push.test.ts deleted file mode 100644 index 3f22e455..00000000 --- a/spikes/349-dofs/vendor/dofs/src/sync/push.test.ts +++ /dev/null @@ -1,166 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { mkdir } from "../fs/mkdir.js"; -import { readFile } from "../fs/readFile.js"; -import { resolveInode } from "../fs/resolve.js"; -import { rm } from "../fs/rm.js"; -import { symlink } from "../fs/symlink.js"; -import { withDB, withTwoDBs } from "../fs/with-db.js"; -import { writeFile } from "../fs/writeFile.js"; -import type { Database } from "../storage.js"; -import type { ChangeEntry } from "./changes.js"; -import { coalesceChanges } from "./coalesce.js"; -import { pushObjects } from "./push.js"; - -// Minimal hand-rolled apply loop. We just need to prove that -// coalesceChanges + pushObjects together transfer enough information -// for the receiver to converge. -async function apply(db: Database, entries: ChangeEntry[], objects: Map) { - for (const entry of entries) { - if (entry.kind === "delete") { - try { - rm(db, entry.path, { recursive: true, force: true }); - } catch { - // Path may already be gone if the receiver was empty. - } - continue; - } - if (entry.kind === "dir") { - mkdir(db, entry.path, { mode: entry.mode, recursive: true }, () => entry.mtime); - continue; - } - if (entry.kind === "symlink") { - // Best-effort: writeFile to a path the symlink replaces won't - // round-trip through symlink. For this test we never overwrite - // a symlink with a non-symlink, so a fresh create is fine. - symlink(db, entry.target, entry.path, () => entry.mtime); - continue; - } - // file: assemble bytes from the chunks the sender shipped. - const parts: Uint8Array[] = []; - for (const c of entry.chunks) { - const key = hex(c.hash); - const bytes = objects.get(key); - if (bytes === undefined) throw new Error(`missing object for ${key}`); - parts.push(bytes); - } - const total = parts.reduce((acc, p) => acc + p.byteLength, 0); - const buf = new Uint8Array(total); - let off = 0; - for (const p of parts) { - buf.set(p, off); - off += p.byteLength; - } - await writeFile(db, entry.path, buf, { mode: entry.mode }, () => entry.mtime); - } -} - -function hex(bytes: Uint8Array): string { - let s = ""; - for (let i = 0; i < bytes.byteLength; i++) s += bytes[i].toString(16).padStart(2, "0"); - return s; -} - -async function drain(it: AsyncIterable): Promise { - const out: T[] = []; - for await (const x of it) out.push(x); - return out; -} - -describe("push", () => { - it("transfers a single file end-to-end", async () => { - await withTwoDBs( - async (a) => { - await writeFile(a, "/hello.txt", "hello world", { mode: 0o644 }, () => 100); - const entries = await drain(coalesceChanges(a, 0)); - const objects = await pull(a, collectHashes(entries)); - return { entries, objects }; - }, - async (b, { entries, objects }) => { - await apply(b, entries, objects); - const node = resolveInode(b, "/hello.txt"); - expect(node?.type).toBe("file"); - const got = await readFile(b, "/hello.txt", "utf8"); - expect(got).toBe("hello world"); - }, - ); - }); - - it("converges across mixed mutations", async () => { - await withTwoDBs( - async (a) => { - mkdir(a, "/d", { mode: 0o755 }, () => 1); - await writeFile(a, "/d/a.txt", "alpha", {}, () => 2); - await writeFile(a, "/d/b.txt", "beta", {}, () => 3); - symlink(a, "/d/a.txt", "/link", () => 4); - await writeFile(a, "/tmp.txt", "scratch", {}, () => 5); - rm(a, "/tmp.txt", {}); - const entries = await drain(coalesceChanges(a, 0)); - const objects = await pull(a, collectHashes(entries)); - return { entries, objects }; - }, - async (b, { entries, objects }) => { - await apply(b, entries, objects); - expect(await readFile(b, "/d/a.txt", "utf8")).toBe("alpha"); - expect(await readFile(b, "/d/b.txt", "utf8")).toBe("beta"); - expect(resolveInode(b, "/tmp.txt")).toBeNull(); - const linked = await readFile(b, "/link", "utf8"); - expect(linked).toBe("alpha"); - }, - ); - }); - - it("pushObjects yields each requested hash exactly once", async () => { - await withDB(async (a) => { - await writeFile(a, "/a.txt", "same", {}, () => 1); - await writeFile(a, "/b.txt", "same", {}, () => 2); - const entries = await drain(coalesceChanges(a, 0)); - const hashes = collectHashes(entries); - // Both files reuse one chunk hash; collectHashes already - // dedups, so we should see exactly one object on the wire. - expect(hashes).toHaveLength(1); - const objects = await pull(a, hashes); - expect(objects.size).toBe(1); - }); - }); - - it("pushObjects throws EUNKNOWN_HASH when a hash is not in vfs_blob_bytes", async () => { - await withDB(async (a) => { - const unknown = new Uint8Array(32); - unknown.fill(0xff); - let caught: unknown; - try { - for await (const _ of pushObjects(a, [unknown])) { - // drain - } - } catch (e) { - caught = e; - } - expect(caught).toBeInstanceOf(Error); - expect((caught as { code?: string }).code).toBe("EUNKNOWN_HASH"); - }); - }); -}); - -function collectHashes(entries: ChangeEntry[]): Uint8Array[] { - const seen = new Set(); - const out: Uint8Array[] = []; - for (const e of entries) { - if (e.kind !== "file") continue; - for (const c of e.chunks) { - const key = hex(c.hash); - if (!seen.has(key)) { - seen.add(key); - out.push(c.hash); - } - } - } - return out; -} - -async function pull(db: Database, hashes: Uint8Array[]): Promise> { - const out = new Map(); - for await (const { hash, bytes } of pushObjects(db, hashes)) { - out.set(hex(hash), bytes); - } - return out; -} diff --git a/spikes/349-dofs/vendor/dofs/src/sync/push.ts b/spikes/349-dofs/vendor/dofs/src/sync/push.ts deleted file mode 100644 index cf84a446..00000000 --- a/spikes/349-dofs/vendor/dofs/src/sync/push.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { createWorkspaceError } from "../errors.js"; -import type { Database } from "../storage.js"; - -// Stream chunk bytes by hash. The receiver collects these into the -// keyed map it uses when assembling files from ChangeEntry chunks. -// Missing hashes throw — the caller is supposed to have probed -// hasObjects() first to avoid asking for what the sender doesn't have. -// -// The push direction (DO → container) and the fetch direction -// (container → DO) both use this same shape; on the wire it is -// fetchObjects on one side and pushObjects on the other. Both names -// resolve to the same SQL. -export async function* pushObjects( - db: Database, - hashes: Uint8Array[], -): AsyncIterable<{ hash: Uint8Array; bytes: Uint8Array }> { - for (const hash of hashes) { - const row = db.one<{ bytes: Uint8Array }>( - "SELECT bytes FROM vfs_blob_bytes WHERE hash = ?", - hash, - ); - if (row === undefined) { - throw createWorkspaceError("EUNKNOWN_HASH", "pushObjects: missing blob for requested hash"); - } - yield { hash, bytes: row.bytes }; - } -} diff --git a/spikes/349-dofs/vendor/dofs/src/sync/watermarks.test.ts b/spikes/349-dofs/vendor/dofs/src/sync/watermarks.test.ts deleted file mode 100644 index 7b9213b0..00000000 --- a/spikes/349-dofs/vendor/dofs/src/sync/watermarks.test.ts +++ /dev/null @@ -1,143 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { withDB } from "../fs/with-db.js"; -import { writeFile } from "../fs/writeFile.js"; -import { - compareChangeCursors, - currentRev, - readFetchCursor, - readWatermark, - writeFetchCursor, - writeWatermark, -} from "./watermarks.js"; - -describe("watermarks", () => { - it("readWatermark returns 0 for a fresh DB", async () => { - await withDB(async (db) => { - expect(readWatermark(db, "pushRev")).toBe(0); - }); - }); - - it("writeWatermark persists across reads", async () => { - await withDB(async (db) => { - writeWatermark(db, "pushRev", 42); - expect(readWatermark(db, "pushRev")).toBe(42); - }); - }); - - it("persists the fetch cursor rev and path separately", async () => { - await withDB(async (db) => { - expect(readFetchCursor(db)).toEqual({ rev: 0, path: null }); - writeFetchCursor(db, { rev: 12, path: "/dir/file.txt" }); - expect(readFetchCursor(db)).toEqual({ rev: 12, path: "/dir/file.txt" }); - writeFetchCursor(db, { rev: 13, path: null }); - expect(readFetchCursor(db)).toEqual({ rev: 13, path: null }); - }); - }); - - it("does not persist an intermediate full-rev cursor when a partial cursor write fails", async () => { - await withDB(async (db) => { - writeFetchCursor(db, { rev: 12, path: null }); - - const originalRun = db.run.bind(db); - db.run = ((query: string, ...bindings: unknown[]) => { - if (query.includes("_vfs_fetch_cursor")) { - throw new Error("forced cursor path failure"); - } - return originalRun(query, ...bindings); - }) as typeof db.run; - - expect(() => writeFetchCursor(db, { rev: 13, path: "/partial.txt" })).toThrow( - "forced cursor path failure", - ); - expect(readFetchCursor(db)).toEqual({ rev: 12, path: null }); - }); - }); - - it("returns a fresh start cursor for a zero fetchRev", async () => { - await withDB(async (db) => { - const cursor = readFetchCursor(db); - cursor.rev = 99; - cursor.path = "/mutated.txt"; - - expect(readFetchCursor(db)).toEqual({ rev: 0, path: null }); - }); - }); - - it("orders partial cursors before full same-rev cursors", () => { - expect(compareChangeCursors({ rev: 0, path: null }, { rev: 0, path: null })).toBe(0); - expect(compareChangeCursors({ rev: 12, path: null }, { rev: 12, path: "/partial.txt" })).toBe( - 1, - ); - expect(compareChangeCursors({ rev: 12, path: "/partial.txt" }, { rev: 12, path: null })).toBe( - -1, - ); - expect(compareChangeCursors({ rev: 13, path: "/partial.txt" }, { rev: 12, path: null })).toBe( - 1, - ); - }); - - it("watermarks advance monotonically (the caller enforces this)", async () => { - await withDB(async (db) => { - writeWatermark(db, "pushRev", 5); - writeWatermark(db, "pushRev", 10); - expect(readWatermark(db, "pushRev")).toBe(10); - // Going backwards is allowed by the helper itself; the sync - // layer's batch-commit logic is what keeps the counter monotonic. - writeWatermark(db, "pushRev", 3); - expect(readWatermark(db, "pushRev")).toBe(3); - }); - }); - - it("currentRev reports the latest rev stamped on a mutation", async () => { - await withDB(async (db) => { - // initializeSchema seeds rev=1 (stamped on the root inode). - const base = currentRev(db); - expect(base).toBeGreaterThanOrEqual(1); - await writeFile(db, "/a.txt", "x", {}, () => 1); - const r1 = currentRev(db); - expect(r1).toBeGreaterThan(base); - await writeFile(db, "/b.txt", "y", {}, () => 2); - expect(currentRev(db)).toBeGreaterThan(r1); - }); - }); - - it("rejects unknown watermark keys at the type level via the helper signature", () => { - // Compile-time only: writeWatermark only accepts "pushRev". - // Fetch progress must go through readFetchCursor/writeFetchCursor. - expect(true).toBe(true); - }); - - describe("per-backend keying", () => { - it("writes under the default backend when the caller omits the id", async () => { - await withDB(async (db) => { - writeWatermark(db, "pushRev", 17); - // The omitted-id read sees the same row. - expect(readWatermark(db, "pushRev")).toBe(17); - // An explicit "default" id also sees it — same slot. - expect(readWatermark(db, "pushRev", "default")).toBe(17); - }); - }); - - it("keeps each backend's cursors independent", async () => { - await withDB(async (db) => { - writeWatermark(db, "pushRev", 10, "worker"); - writeWatermark(db, "pushRev", 20, "container"); - expect(readWatermark(db, "pushRev", "worker")).toBe(10); - expect(readWatermark(db, "pushRev", "container")).toBe(20); - // A push under "worker" doesn't disturb the "container" - // backend's cursor. - writeWatermark(db, "pushRev", 11, "worker"); - expect(readWatermark(db, "pushRev", "worker")).toBe(11); - expect(readWatermark(db, "pushRev", "container")).toBe(20); - }); - }); - - it("unknown backend id reads as 0", async () => { - await withDB(async (db) => { - writeWatermark(db, "pushRev", 5, "worker"); - expect(readWatermark(db, "pushRev", "never-registered")).toBe(0); - }); - }); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/sync/watermarks.ts b/spikes/349-dofs/vendor/dofs/src/sync/watermarks.ts deleted file mode 100644 index 084c3c68..00000000 --- a/spikes/349-dofs/vendor/dofs/src/sync/watermarks.ts +++ /dev/null @@ -1,130 +0,0 @@ -import type { Database } from "../storage.js"; - -// Watermarks owned by the local database. Keyed by (k, backend) so a -// single workspace can host more than one backend and each keeps its -// own sync cursors. The container's appliedPushRev lives in-memory on -// the container side; we don't store it here. -// -// pushRev — last DO-side rev successfully pushed to the backend. -// -// initializeSchema() seeds pushRev at 0 in _vfs_watermark for the -// default backend. The schema table is the durability surface; -// readers and writers always go through this module so the SQL -// stays in one place. -// -// `backend` defaults to DEFAULT_BACKEND_ID so older callers that -// only ran one backend (or ran the package against a schema before -// per-backend keying landed) keep working unchanged. The v3 → v4 -// schema migration backfills the column on existing rows with the -// same default. -// -// Fetch progress is a `{ rev, path }` cursor. Its rev component is -// still stored in _vfs_watermark for schema compatibility, but callers -// must use readFetchCursor() / writeFetchCursor() so rev and path stay -// consistent. -export type WatermarkKey = "pushRev"; - -export const DEFAULT_BACKEND_ID = "default"; - -// Cursor into the remote change stream. `path: null` means `rev` -// is fully drained and the next fetch resumes strictly after that -// rev. A string path means resume inside the same rev after that -// path. The empty string is a real path value, not a sentinel, and -// must not be used to mean "start of rev". -export type ChangeCursor = { rev: number; path: string | null }; - -export function readWatermark( - db: Database, - key: WatermarkKey, - backend: string = DEFAULT_BACKEND_ID, -): number { - return ( - db.scalar("SELECT v FROM _vfs_watermark WHERE k = ? AND backend = ?", key, backend) ?? 0 - ); -} - -function readFetchRev(db: Database, backend: string = DEFAULT_BACKEND_ID): number { - return ( - db.scalar( - "SELECT v FROM _vfs_watermark WHERE k = ? AND backend = ?", - "fetchRev", - backend, - ) ?? 0 - ); -} - -function writeWatermarkValue( - db: Database, - key: WatermarkKey | "fetchRev", - value: number, - backend: string = DEFAULT_BACKEND_ID, -): void { - db.run( - "INSERT INTO _vfs_watermark (k, backend, v) VALUES (?, ?, ?) " + - "ON CONFLICT(k, backend) DO UPDATE SET v = excluded.v", - key, - backend, - value, - ); -} - -function writeFetchCursorPath( - db: Database, - path: string | null, - backend: string = DEFAULT_BACKEND_ID, -): void { - db.run( - "INSERT INTO _vfs_fetch_cursor (k, backend, path) VALUES (?, ?, ?) " + - "ON CONFLICT(k, backend) DO UPDATE SET path = excluded.path", - "fetch", - backend, - path, - ); -} - -export function writeWatermark( - db: Database, - key: WatermarkKey, - value: number, - backend: string = DEFAULT_BACKEND_ID, -): void { - writeWatermarkValue(db, key, value, backend); -} - -export function readFetchCursor(db: Database, backend: string = DEFAULT_BACKEND_ID): ChangeCursor { - const rev = readFetchRev(db, backend); - if (rev === 0) return { rev: 0, path: null }; - const path = db.scalar( - "SELECT path FROM _vfs_fetch_cursor WHERE k = ? AND backend = ?", - "fetch", - backend, - ); - return { rev, path: path ?? null }; -} - -export function writeFetchCursor( - db: Database, - cursor: ChangeCursor, - backend: string = DEFAULT_BACKEND_ID, -): void { - db.transactionSync(() => { - writeWatermarkValue(db, "fetchRev", cursor.rev, backend); - writeFetchCursorPath(db, cursor.path, backend); - }); -} - -export function compareChangeCursors(a: ChangeCursor, b: ChangeCursor): number { - if (a.rev !== b.rev) return a.rev - b.rev; - if (a.path === b.path) return 0; - if (a.path === null) return 1; - if (b.path === null) return -1; - return a.path < b.path ? -1 : 1; -} - -// The latest rev stamped on any DO-side mutation. coalesceChanges -// reads this implicitly via vfs_nodes.rev; the sync layer exposes it -// to callers that want to record the rev component of their next -// cursor. -export function currentRev(db: Database): number { - return db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; -} diff --git a/spikes/349-dofs/vendor/dofs/src/testing-recording.ts b/spikes/349-dofs/vendor/dofs/src/testing-recording.ts deleted file mode 100644 index 04f9cc81..00000000 --- a/spikes/349-dofs/vendor/dofs/src/testing-recording.ts +++ /dev/null @@ -1,78 +0,0 @@ -import type { DurableObjectStorageLike, SQLCursorLike } from "./types.js"; - -// Pure-JS test fixture. Records every SQL statement the production -// code emits, lets the test assert against the trace, and serves a -// tiny in-process subset of vfs_meta semantics so the FS scaffolding -// can boot without dragging node:sqlite in. -// -// Lives in its own file so the workerd test runner can import this -// fixture without loading the SQLiteTestStorage class that wraps -// node:sqlite (which workerd doesn't ship). - -export interface ExecutedStatement { - query: string; - bindings: unknown[]; -} - -class TestCursor implements SQLCursorLike { - private readonly rows: Row[]; - - constructor(rows: Row[]) { - this.rows = rows; - } - - toArray(): Row[] { - return this.rows; - } -} - -export class RecordingStorage implements DurableObjectStorageLike { - readonly statements: ExecutedStatement[] = []; - readonly sql = { - exec: >( - query: string, - ...bindings: unknown[] - ): SQLCursorLike => { - this.statements.push({ query, bindings }); - return new TestCursor(this.rowsFor(query, bindings)); - }, - }; - - private readonly meta = new Map(); - - constructor(seed?: { schemaVersion?: number; rev?: number }) { - if (seed?.schemaVersion !== undefined) { - this.meta.set("schema_version", seed.schemaVersion); - } - if (seed?.rev !== undefined) { - this.meta.set("rev", seed.rev); - } - } - - transactionSync(closure: () => T): T { - return closure(); - } - - private rowsFor(query: string, bindings: unknown[]): Row[] { - const normalized = query.replace(/\s+/g, " ").trim().toLowerCase(); - if (normalized === "select v from vfs_meta where k = ?") { - const key = String(bindings[0]); - const value = this.meta.get(key); - return value === undefined ? [] : ([{ v: value }] as Row[]); - } - - if (normalized.startsWith("insert or ignore into vfs_meta")) { - const key = String(bindings[0]); - const value = Number(bindings[1]); - if (!this.meta.has(key)) { - this.meta.set(key, value); - } - } - - if (normalized.startsWith("update vfs_meta set v = ? where k = ?")) { - this.meta.set(String(bindings[1]), Number(bindings[0])); - } - - return []; - } -} diff --git a/spikes/349-dofs/vendor/dofs/src/testing.test.ts b/spikes/349-dofs/vendor/dofs/src/testing.test.ts deleted file mode 100644 index 507e8c6b..00000000 --- a/spikes/349-dofs/vendor/dofs/src/testing.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { initializeSchema, ROOT_INODE } from "./schema/index.js"; -import { Database } from "./storage.js"; -import { SQLiteTestStorage } from "./testing.js"; - -describe("SQLiteTestStorage", () => { - it("backs a real in-memory database that initializeSchema can apply", () => { - const storage = new SQLiteTestStorage(); - const db = new Database(storage); - - initializeSchema(db, () => 1234); - - const row = db.one<{ inode: number; type: string; mtime: number }>( - "SELECT inode, type, mtime FROM vfs_nodes WHERE inode = ?", - ROOT_INODE, - ); - expect(row).toEqual({ inode: ROOT_INODE, type: "dir", mtime: 1234 }); - }); - - it("runs transactionSync atomically", () => { - const storage = new SQLiteTestStorage(); - const db = new Database(storage); - initializeSchema(db, () => 0); - - expect(() => { - db.transactionSync(() => { - db.run("INSERT INTO vfs_meta (k, v) VALUES (?, ?)", "rollback_probe", 1); - throw new Error("forced"); - }); - }).toThrow("forced"); - - const value = db.scalar("SELECT v FROM vfs_meta WHERE k = ?", "rollback_probe"); - expect(value).toBeUndefined(); - }); -}); diff --git a/spikes/349-dofs/vendor/dofs/src/testing.ts b/spikes/349-dofs/vendor/dofs/src/testing.ts deleted file mode 100644 index 3b42ec02..00000000 --- a/spikes/349-dofs/vendor/dofs/src/testing.ts +++ /dev/null @@ -1,90 +0,0 @@ -// Real-DB DurableObjectStorageLike for unit tests. Backed by Node's -// built-in node:sqlite running an in-memory database. Workers' DO SQL -// surface is a subset of this, so anything that works here works on -// the real platform too. -// -// This module imports node:sqlite at the top level and therefore -// cannot be loaded under workerd. RecordingStorage — the -// pure-JS fixture that also lives in dofs's testing surface -// — has moved to ./testing-recording.ts so it can be imported -// from workerd-runnable tests. We re-export it here so existing -// `import { RecordingStorage } from "@cloudflare/dofs/testing"` -// call sites keep working under node. - -import { DatabaseSync, type StatementSync } from "node:sqlite"; - -import type { DurableObjectStorageLike, SQLCursorLike } from "./types.js"; - -export type { ExecutedStatement } from "./testing-recording.js"; -export { RecordingStorage } from "./testing-recording.js"; - -class TestCursor implements SQLCursorLike { - private readonly rows: Row[]; - - constructor(rows: Row[]) { - this.rows = rows; - } - - toArray(): Row[] { - return this.rows; - } -} - -export class SQLiteTestStorage implements DurableObjectStorageLike { - private readonly db: DatabaseSync; - private readonly cache = new Map(); - readonly sql: { - exec: (query: string, ...bindings: unknown[]) => SQLCursorLike; - }; - - constructor() { - this.db = new DatabaseSync(":memory:"); - this.sql = { - exec: (query: string, ...bindings: unknown[]): SQLCursorLike => { - // node:sqlite refuses statements with trailing whitespace through - // prepare(); also we cache prepared statements per unique query - // string to keep the fixture fast. - const key = query; - let stmt = this.cache.get(key); - if (stmt === undefined) { - stmt = this.db.prepare(query); - this.cache.set(key, stmt); - } - const normalizedBindings = bindings.map(toSQLiteValue); - const rows = (stmt.all(...(normalizedBindings as never[])) as Row[]) ?? []; - return new TestCursor(rows); - }, - }; - } - - transactionSync(closure: () => T): T { - this.db.exec("BEGIN"); - try { - const result = closure(); - this.db.exec("COMMIT"); - return result; - } catch (error) { - this.db.exec("ROLLBACK"); - throw error; - } - } - - close(): void { - // StatementSync instances are released when the database closes. - this.cache.clear(); - this.db.close(); - } -} - -// node:sqlite is strict about input shapes: it accepts strings, numbers, -// bigints, null, and Uint8Array but not undefined, Buffer subclasses -// other than Uint8Array, or booleans. Normalize. -function toSQLiteValue(value: unknown): string | number | bigint | null | Uint8Array { - if (value === undefined || value === null) return null; - if (typeof value === "boolean") return value ? 1 : 0; - if (value instanceof Uint8Array) return value; - if (typeof value === "string" || typeof value === "number" || typeof value === "bigint") { - return value; - } - throw new TypeError(`SQLiteTestStorage cannot bind value of type ${typeof value}`); -} diff --git a/spikes/349-dofs/vendor/dofs/src/types.ts b/spikes/349-dofs/vendor/dofs/src/types.ts deleted file mode 100644 index 2832820e..00000000 --- a/spikes/349-dofs/vendor/dofs/src/types.ts +++ /dev/null @@ -1,16 +0,0 @@ -export interface SQLCursorLike> { - toArray(): Row[]; -} - -export interface SQLStorageLike { - exec>( - query: string, - ...bindings: unknown[] - ): SQLCursorLike; -} - -export interface DurableObjectStorageLike { - sql: SQLStorageLike; - transaction?(closure: () => T | Promise): T | Promise; - transactionSync?(closure: () => T): T; -} diff --git a/spikes/349-dofs/vendor/dofs/tests/worker.ts b/spikes/349-dofs/vendor/dofs/tests/worker.ts deleted file mode 100644 index ef00d508..00000000 --- a/spikes/349-dofs/vendor/dofs/tests/worker.ts +++ /dev/null @@ -1,25 +0,0 @@ -// Minimal worker + DO shim used by the workerd-backed test runner. -// The DO exists solely so vitest-pool-workers can hand a real -// DurableObjectStorage instance to test callbacks via -// runInDurableObject(). The DO doesn't expose any externally useful -// surface; it lives under tests/ so it stays outside the package's -// public exports. - -import { DurableObject } from "cloudflare:workers"; - -export interface TestBindings { - TestStorage: DurableObjectNamespace; -} - -export class TestStorage extends DurableObject { - // No methods of our own; tests reach in via runInDurableObject() and - // use this.ctx.storage directly. -} - -// The pool requires a default export so it can spin up a worker. -// We don't route any traffic through it. -export default { - async fetch(): Promise { - return new Response("dofs test worker", { status: 200 }); - }, -} satisfies ExportedHandler; diff --git a/spikes/349-dofs/vendor/dofs/tests/wrangler.jsonc b/spikes/349-dofs/vendor/dofs/tests/wrangler.jsonc deleted file mode 100644 index f330d766..00000000 --- a/spikes/349-dofs/vendor/dofs/tests/wrangler.jsonc +++ /dev/null @@ -1,20 +0,0 @@ -{ - // Test-only worker. Not deployed. - "name": "dofs-tests", - "main": "./worker.ts", - "compatibility_date": "2026-05-26", - "durable_objects": { - "bindings": [ - { - "name": "TestStorage", - "class_name": "TestStorage" - } - ] - }, - "migrations": [ - { - "tag": "v1", - "new_sqlite_classes": ["TestStorage"] - } - ] -} diff --git a/spikes/349-dofs/vendor/dofs/tsconfig.build.json b/spikes/349-dofs/vendor/dofs/tsconfig.build.json deleted file mode 100644 index e7086b47..00000000 --- a/spikes/349-dofs/vendor/dofs/tsconfig.build.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "extends": "./tsconfig.json", - "compilerOptions": { - "declaration": true, - "emitDeclarationOnly": false, - "noEmit": false, - "outDir": "dist", - "rootDir": "src", - "types": [ - "@cloudflare/workers-types", - "node" - ] - }, - "include": [ - "src/**/*.ts" - ], - "exclude": [ - "src/**/*.test.ts", - "src/fs/with-db.workers.ts", - "src/bench/**" - ] -} diff --git a/spikes/349-dofs/vendor/dofs/tsconfig.json b/spikes/349-dofs/vendor/dofs/tsconfig.json deleted file mode 100644 index 39e35871..00000000 --- a/spikes/349-dofs/vendor/dofs/tsconfig.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2022", - "module": "ES2022", - "moduleResolution": "bundler", - "lib": ["ES2023", "ESNext.Disposable", "WebWorker"], - "strict": true, - "skipLibCheck": true, - "noEmit": true, - "esModuleInterop": true, - "resolveJsonModule": true, - "isolatedModules": true, - "types": [ - "@cloudflare/workers-types", - "vitest/globals", - "node", - "@cloudflare/vitest-pool-workers/types" - ] - }, - "include": ["src/**/*.ts", "tests/**/*.ts", "*.ts"] -} diff --git a/spikes/349-dofs/vendor/dofs/vitest.config.bench.ts b/spikes/349-dofs/vendor/dofs/vitest.config.bench.ts deleted file mode 100644 index b9e25298..00000000 --- a/spikes/349-dofs/vendor/dofs/vitest.config.bench.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { cloudflareTest } from "@cloudflare/vitest-pool-workers"; -import { defineConfig } from "vitest/config"; - -// Benchmark runner. Reuses the workerd-backed pool (same wrangler -// config as the workers test project) so the harness drives a REAL -// Durable Object SqlStorage — NOT the node SQLiteTestStorage fixture, -// which caches prepared statements and would understate per-statement -// cost. Scoped to the *.bench.ts glob so it never runs during -// `npm test`; invoke explicitly via `npm run bench`. -export default defineConfig({ - plugins: [ - cloudflareTest({ - wrangler: { configPath: "./tests/wrangler.jsonc" }, - }), - ], - test: { - globals: true, - include: ["src/bench/**/*.bench.ts"], - // The harness builds large trees and loops tens of thousands of - // synchronous ops; the default 5s timeout is far too tight. - testTimeout: 600_000, - hookTimeout: 600_000, - }, -}); diff --git a/spikes/349-dofs/vendor/dofs/vitest.config.ts b/spikes/349-dofs/vendor/dofs/vitest.config.ts deleted file mode 100644 index e0512789..00000000 --- a/spikes/349-dofs/vendor/dofs/vitest.config.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { defineConfig } from "vitest/config"; - -export default defineConfig({ - test: { - environment: "node", - globals: true, - include: ["src/**/*.test.ts", "test/**/*.test.ts"], - }, -}); diff --git a/spikes/349-dofs/vendor/dofs/vitest.config.workers.ts b/spikes/349-dofs/vendor/dofs/vitest.config.workers.ts deleted file mode 100644 index d912d613..00000000 --- a/spikes/349-dofs/vendor/dofs/vitest.config.workers.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { cloudflareTest } from "@cloudflare/vitest-pool-workers"; -import { defineConfig } from "vitest/config"; - -// Workerd-backed runner. Same .test.ts files as the node project, but -// withDB resolves to the Durable Object-backed implementation via the -// alias below. -export default defineConfig({ - plugins: [ - cloudflareTest({ - wrangler: { configPath: "./tests/wrangler.jsonc" }, - }), - ], - resolve: { - // Match any relative import that lands on src/fs/with-db.js so - // tests under src/fs/, src/, and src/sync/ all resolve to the - // workerd-backed implementation regardless of their depth. - alias: [ - { - find: /^.*\/with-db\.js$/, - replacement: new URL("./src/fs/with-db.workers.ts", import.meta.url).pathname, - }, - ], - }, - test: { - globals: true, - include: ["src/**/*.test.ts"], - // testing.test.ts exercises SQLiteTestStorage directly — the - // node:sqlite-backed fixture has no analogue under workerd, so - // the test is meaningful only against the real node runtime. - // All other tests run under both backends; provider/provider-fd - // use a withProvider helper that delegates to withDB, which the - // workers config aliases to a DO-backed implementation. - exclude: ["src/testing.test.ts"], - }, -}); diff --git a/spikes/351-worker-backends/README.md b/spikes/351-worker-backends/README.md deleted file mode 100644 index 5ddcee16..00000000 --- a/spikes/351-worker-backends/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# Spike 351: Worker Shell and Worker JavaScript natively in Deno - -Tests whether Cloudflare Computer's two imperative execution backends can run -over the Deno-local DOFS Workspace from #349 — with no `workerd`, Wrangler, -Docker, or other JavaScript runtime. Both are **optional imperative extensions** -layered on the declarative durable core, not prerequisites for it; the spike -supplies evidence for #346, not a product decision. Verdicts and measurements -are in [evidence/EVIDENCE.md](evidence/EVIDENCE.md). - -## Run it - -From the repository root: - -```bash -deno task spike:351 # vendor-build + compile + test -``` - -The compiled proof takes a workspace database path and one operation, so each -invocation is a full host restart against the same durable state: - -```bash -dist/proof /tmp/ws.db fs-mkdir /workspace -dist/proof /tmp/ws.db fs-write /workspace/a.txt alpha # host capability -dist/proof /tmp/ws.db shell "cat a.txt" # Worker Shell -dist/proof /tmp/ws.db js 'export default () => 42' # Worker JavaScript -``` - -## Layout - -- `vendor/worker-shell/adapter.ts` — Cloudflare's `WorkspaceFsAdapter`, - byte-identical (MIT; see - [PROVENANCE.md](vendor/worker-shell/PROVENANCE.md)). It is the whole reason - the shell ports: zero `cloudflare:*` imports, and its filesystem contract is - a structural interface rather than an RPC stub. -- `host/shell.ts` — the shell backend with Cloudflare's transport removed: - no `WorkerEntrypoint`, no Workers RPC, no Dynamic Worker loader, because the - workspace is in-process. -- `host/loader.ts` — a Deno port of `WorkspaceRuntimeLoader`: an in-memory - module graph supplied as blob URLs with rewritten specifiers, run in a Deno - Worker, with the filesystem capability crossing as async postMessage RPC. -- `host/workspace.ts`, `host/workspace-fs.ts` — the DOFS workspace and the - small shim supplying the three stub-only methods the adapter expects. -- `tests/` — the scenario suite; `evidence/probes/` — the full probe ledgers - with every command, escape attempt, and measurement. - -The `@cloudflare/dofs` build is shared with the #349 spike rather than vendored -twice: `deno task --cwd spikes/351-worker-backends vendor` builds it there and -links it here. This directory stays outside the workspace via the root -`deno.json` `exclude`, so the four verification gates keep their scope. diff --git a/spikes/351-worker-backends/build.ts b/spikes/351-worker-backends/build.ts deleted file mode 100644 index 49fc3a10..00000000 --- a/spikes/351-worker-backends/build.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { main } from "effection"; -import { exec } from "@effectionx/process"; - -const here = new URL("./", import.meta.url).pathname; - -main(function* () { - yield* exec(Deno.execPath(), { - arguments: [ - "compile", - "--allow-all", - "--unstable-worker-options", - "--node-modules-dir=manual", - "--include", - "host/loader.ts", - "--output", - "dist/proof", - "host/main.ts", - ], - cwd: here, - }).expect(); - console.log("built dist/proof"); -}); diff --git a/spikes/351-worker-backends/deno.json b/spikes/351-worker-backends/deno.json deleted file mode 100644 index 7ef3c600..00000000 --- a/spikes/351-worker-backends/deno.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "exclude": [ - "vendor", - "dist", - "node_modules" - ], - "nodeModulesDir": "manual", - "tasks": { - "all": "deno task vendor && deno task build && deno task test", - "vendor": "deno run --allow-all vendor.ts", - "build": "deno run --allow-all build.ts", - "test": "deno test --allow-all tests/", - "check": "deno check ." - }, - "scopes": { - "./vendor/": { - "@cloudflare/dofs": "./host/dofs-types.ts" - } - } -} \ No newline at end of file diff --git a/spikes/351-worker-backends/evidence/COMPARISON.md b/spikes/351-worker-backends/evidence/COMPARISON.md deleted file mode 100644 index 9233d2a1..00000000 --- a/spikes/351-worker-backends/evidence/COMPARISON.md +++ /dev/null @@ -1,81 +0,0 @@ -# Comparison and recommendation for #346 - -#351 completes the evidence set: #347 measured bundled `workerd`, #349 measured -the Deno-local DOFS Workspace, and this spike measures what imperative -execution the Deno-local topology can carry on its own. The topology decision -has two separate questions. - -## Question 1 — is Deno-local DOFS sufficient for the declarative durable core? - -**Yes, on the evidence available.** The core the workflow needs is a durable, -coherent, identity-scoped filesystem, and #349 proved each property directly: -Cloudflare's unmodified DOFS schema over a file-backed `node:sqlite` adapter, -surviving full process restarts (including deletions and replacements), -isolating workspaces by database path, refusing a newer on-disk schema rather -than recreating it, and checkpointing to a single-file artifact on close — at -110 MB and 0.08 s per operation, in one process with no supervision surface. - -The declarative components map onto primitives that already exist in that -layer: `

` onto `mkdir`, `` onto `writeFile`/`readFile`, and -``/`` onto DOFS's own git support, which Cloudflare drives -through the same filesystem (their `git` shell command is a thin shim over it). -This spike does not implement those components, per its scope; it confirms the -substrate exposes the operations they need, and that a host capability and an -imperative backend observe the same committed state in both directions -**[test]**. - -The one caveat carried forward from #349: real FUSE — needed only for arbitrary -native subprocesses, which `xmd workflow` does not require — is blocked -in-process by a Deno uv-polyfill gap, and the userspace shim is development-only. -Neither bears on the declarative core. - -## Question 2 — which imperative capabilities can safely layer on top? - -**Worker Shell: include initially, scoped.** It runs natively, reuses -Cloudflare's adapter byte-identical, and its containment was measured rather -than assumed — ~50 escape attempts, zero leaks, no native-execution path at -all. Two conditions: run it inside a Deno Worker so a CPU-bound script cannot -starve the host (the measured hazard: 6.4 s of wall time with the abort timer -never firing), and describe it accurately as a workspace-scoped interpreter -with 83 built-ins, never as native command execution. - -**Worker JavaScript: defer.** The full contract works — in-memory module graph -with no materialization, live-streamed output, capability access to DOFS, -returned values, errors, timeouts, cancellation — and it is genuinely useful. -But isolation holds only in a compiled artifact (a `deno run` host lets a -"locked" worker import `jsr:`/`npm:`), and CPU-bound user code cannot be -preempted at all. Since it is an optional extension over a declarative core, -the cost of shipping it now exceeds its value; revisit when Deno offers a CPU -or wall budget on `new Worker(...)`, or accept an out-of-process worker and its -startup cost. - -## Is bundled `workerd` still necessary? - -**Not for this scope.** #347's decisive advantage was that it runs Cloudflare's -real backends; this spike shows both of those execution models also run -natively in Deno over the same workspace, at a fraction of the weight — 139.7 MB -carrying *both* backends and DOFS, versus 191 MB for the workerd host, with -operations in tens of milliseconds rather than tenths of a second, and no child -process to supervise. - -`workerd` remains the only option for the things this topology deliberately -does not provide: Cloudflare Containers, the hosted Workers surface (KV, R2, -Queues, Durable Object identity), and behavior guaranteed to match deployed -Cloudflare. If those enter scope, #347's evidence stands ready; they are not -required by the declarative workflow core. - -## Recommendation - -1. **Adopt** the Deno-local DOFS topology for the declarative durable workflow - core. -2. **Include** Worker Shell initially, scoped: inside a Deno Worker, described - as a workspace-scoped interpreter, with unsupported operations refused - explicitly. -3. **Defer** Worker JavaScript until the preemption gap is resolved or an - out-of-process cost is accepted. -4. **Omit** bundled `workerd` from the local topology, keeping #347 as the - documented path if hosted-Cloudflare fidelity or Containers become - requirements. - -The final selection is recorded on #346; these spikes supply the evidence, not -the decision. diff --git a/spikes/351-worker-backends/evidence/EVIDENCE.md b/spikes/351-worker-backends/evidence/EVIDENCE.md deleted file mode 100644 index 3ae94410..00000000 --- a/spikes/351-worker-backends/evidence/EVIDENCE.md +++ /dev/null @@ -1,193 +0,0 @@ -# Evidence: Worker Shell and Worker JavaScript in Deno (#351) - -Measured on macOS 15 (Darwin 25.5.0, arm64), Deno 2.9.1, against -cloudflare/computer `v0.1.1` (`63d363632e558f7e077794988d36ed75017c2a62`) and -`just-bash@3.0.1`, on 2026-08-06. Claims marked **[test]** are asserted by -`tests/`; **[measured]** were observed directly; **[probe]** carry their -command-by-command ledger in `probes/`. - -Both backends are **optional imperative extensions** over the declarative -durable core. Neither is a prerequisite for the Deno-local DOFS topology, and -neither backend's limits reject it. - -## Verdicts - -| Backend | Verdict | Basis | -| --- | --- | --- | -| **Worker Shell** | **include initially, scoped** | Runs natively; containment is architectural and was measured, not inferred; one availability hazard needs a boundary decision | -| **Worker JavaScript** | **defer** | The full contract works, but isolation depends on shipping compiled, and CPU-bound user code cannot be preempted | - -## Worker Shell - -Cloudflare's shell is the `just-bash` interpreter plus a filesystem adapter. -The adapter is vendored **byte-identical** (sha1 -`c9ec5356d60ee7bc9aa3c5677320bf8c9aa4058c`) — it has zero `cloudflare:*` -imports, and its filesystem contract is a structural interface rather than an -RPC stub, which is the entire reason the backend ports. Everything Cloudflare -wraps around it — `WorkerEntrypoint`, `env.HOST.getWorkspace()`, the Dynamic -Worker loader — exists to reach a filesystem across an isolate boundary, and -is unnecessary when the workspace is in-process. The only new code is a -~60-line shim supplying `exists`/`statOrNull`/`lstatOrNull`, which on -Cloudflare come from the RPC stub. - -**Works** **[test]**: a host `API.Fs` write is visible to shell execution and a -shell write is visible to the host, each across a full process restart; -pipelines, redirection (`>`, `>>`), exit status (`$?`, explicit `exit N`), -environment variables, cwd, stdout/stderr separation, and unknown-command -errors all behave; separate database paths are separate workspaces. - -**Command set** **[probe]**: 83 registered built-ins (`cat`, `grep`, `sed`, -`awk`, `find`, `sort`, `tr`, `cut`, `head`, `tail`, `wc`, `cp`, `mv`, `rm`, -`mkdir`, `chmod`, `test`, …). This is an interpreter, **not** a process -launcher: there is no PATH resolution and no native executable can run -(`/bin/sh`, `id`, `uname` are all `command not found`) **[test]**. Absent: -`trap`, `yes`, `xargs -n`. Worker-backed commands (`js-exec`, `sqlite3`) need a -one-line `process.connected` prelude under Deno; `python3` remains unusable. - -**Security — measured, not inferred** **[probe]**. With canaries planted on the -host filesystem and in the environment, ~50 escape attempts produced **zero -leaks**: - -- Filesystem: `/etc/passwd`, `../../..` traversal, `find /`, `grep -r /` all - resolve inside DOFS only; writes to `/tmp` and `/etc` land in the DOFS - namespace and no host file was created **[test]**. -- Environment: fabricated (`OSTYPE=linux-gnu` on macOS, `uid=1000`, `pid=1`); - the host environment is invisible. -- Native execution: none exists. The QuickJS `child_process.execSync` shim — - the one plausible path — re-enters just-bash itself. -- QuickJS guest: no `Deno`, `Function` constructor blocked, `import("node:fs")` - blocked, `require("fs")` is a workspace-scoped shim, `Atomics`/`WebAssembly` - undefined. -- Network: off by default; when opted in, a strict prefix allowlist plus - private/loopback rejection plus DNS-rebinding pinning — cloud-metadata IPs - and `localhost` are denied even when allowlisted. - -**Defense-in-depth**: just-bash ships its own `DefenseInDepthBox`, and -Cloudflare disables it because workerd cannot register ESM loader hooks. Under -Deno it must *also* be disabled: it is on by default and hard-fails -(`critical patches failed: Module._load`), aborting every exec. So the same -one-line opt-out Cloudflare already ships is required here — **and the -containment above holds without it**, because it is architectural: an injected -filesystem, a fabricated environment, and an interpreter with no process -launcher. - -**The one real hazard is availability, not capability** **[probe]**. just-bash -executes synchronously: a 6.4 s CPU-bound loop starved the host event loop to -0 ticks of an expected 127 — the abort timer never fired, so `AbortSignal` -cancellation only works where the interpreter already awaits, and `while true` -spun 56 s at 102 % CPU unrecoverably. The in-isolate brake is -`maxCommandCount` (default 10 000). - -**Boundary recommendation**: run the shell in a Deno Worker with the filesystem -as an async RPC proxy. The DOFS handle cannot cross (`structuredClone` → -`DataCloneError`), but the adapter's Promise-returning methods tolerate RPC -**unmodified** — 10/10 functional parity including symlinks, `chmod`, `sed -i`, -`grep -r`. Cost: 39.6 ms boot, ~0.04 ms per call (`echo` 0.27 ms vs 0.30 ms -in-isolate). Benefit: the host stays responsive and `worker.terminate()` -reclaims a wedged loop that is otherwise unrecoverable. The committed proof -runs in-isolate, which is why the hazard is stated rather than hidden. - -## Worker JavaScript - -Upstream's `WorkspaceRuntimeLoader` is Cloudflare's Worker Loader binding: -`load({ mainModule, modules })` over an in-memory specifier→source map. The -port replaces it with a Deno Worker. - -**Module supply, without materialization** **[probe]**. A relative -`import "./dep.js"` does not resolve from a `blob:` or `data:` base — Deno -reports `invalid URL: relative URL with a cannot-be-a-base base`. Rewriting -each specifier to the dependency's absolute blob URL, in topological order, -loads the graph and survives `deno compile`. Static import maps are inherited -by workers but there is no per-Worker `importMap` option, so they cannot carry -a runtime-generated map. **Temp-dir materialization was never needed**, so its -coherence and cleanup costs are avoided rather than measured. - -**Works** **[test]**: an entry module with a dependency runs in the isolate, -reaches DOFS through the capability bridge, separates stdout and stderr, -returns a result, and its write is committed — readable by a later process. -A thrown error becomes stderr plus exit 1 without failing the host. Output is -live-streamed, not batched (frames at 6/410/812 ms) **[probe]**. Cold -spawn→exit 13 ms; capability round-trip 0.065 ms **[measured]**. - -**Isolation is adequate only in the compiled artifact** — the decisive caveat. -`permissions: "none"` governs Deno *ops*, not the *module loader*: under -`deno run`, a locked worker still executes `import("jsr:…")`/`import("npm:…")`, -pulling arbitrary third-party code and network egress that `net: false` does -not stop; the `import: false` permission is inert. `deno compile`'s frozen -graph closes this — `jsr:`/`npm:`/`https:` all become `Module not found`. -Host files and the environment are denied in both **[test]**. **A `deno run` -host must never be presented as a sandbox.** - -**Two gaps** **[probe]**: - -1. `Worker.terminate()` cannot preempt CPU-spinning user JavaScript. The - timeout *outcome* is correct (1507 ms for a 1500 ms budget) but - `terminate()` returns in 0 ms while the thread runs on, permanently - blocking process exit. The missing primitive is Deno-side: **a CPU or wall - budget on `new Worker(...)`, or a `terminate()` that interrupts running - JavaScript**. Cloudflare's `limits.cpuMs` has no equivalent. The only - faithful fix is an out-of-process worker that can be SIGKILLed, which - forfeits the 13 ms cold start. -2. `compatibilityDate`/`compatibilityFlags` and retained execution - (`getExec`/`resume`) are not expressible; a Deno Worker's lifetime is bounded - by the host call. Upstream's shell backend has no retention either, so this - only distances the JavaScript backend from hosted Cloudflare. - -## Explicit refusal, never fallback - -An operation the host does not install fails where it is called, at both -refusal points **[test]**: an operation the runner exposes but the host did not -install reports `unsupported filesystem op: `, and one outside the -capability surface fails as `not a function`. The isolate has no path to host -execution, so there is nothing to fall back to — the property workflow mode -requires holds by construction. - -## Compatibility matrix - -| Dimension | Worker Shell (Deno) | Worker JavaScript (Deno) | Cloudflare (both, #347) | -| --- | --- | --- | --- | -| Execution | 83 interpreter built-ins; no native processes | arbitrary user ES modules | same, in a Dynamic Worker | -| Isolation | architectural (injected fs, fabricated env, no launcher) | Deno Worker; **compiled artifact only** | workerd isolate | -| Filesystem coherence | in-process, immediate, bidirectional **[test]** | async capability RPC, committed **[test]** | RPC to host DO | -| Streaming | buffered — upstream synthesizes frames after the run | live-streamed frames | same as each | -| Cancellation | only at interpreter await points; CPU loops unrecoverable in-isolate | terminate() works except on CPU spin | isolate disposal | -| Retention / reattach | none (upstream has none either) | none | JS backend only | -| Platforms | any Deno target | any Deno target | five workerd targets | -| Artifact size | 139.7 MB compiled proof (both backends + DOFS) | — | 191 MB (#347) | -| Startup / operation | shell exec 16-22 ms; fs op 2-5 ms **[measured]** | 13 ms cold spawn; 0.065 ms per capability call | 0.27 s warm op cycle (#347) | - -## Source reuse and licensing - -- `WorkspaceFsAdapter` — vendored byte-identical, **MIT** (Cloudflare), with - provenance and upgrade procedure in `vendor/worker-shell/PROVENANCE.md`. - This is the narrow seam the issue asked for: one file, no broad vendoring. -- `just-bash@3.0.1` — an ordinary npm dependency, **Apache-2.0**, from - `github.com/vercel-labs/just-bash` (Vercel Labs, not Cloudflare). ESM entry - 256 KB. -- `@cloudflare/dofs` — the #349 vendored copy, built once and shared. -- Maintenance delta versus Cloudflare: the shell tracks one vendored file plus - a public npm package; the JavaScript loader is our own ~300 lines against a - 12-line upstream interface, so upstream loader changes do not propagate. - -**Upstream defects found** (affect Cloudflare equally): relative symlinks are -broken in `@cloudflare/dofs` itself (`ln -s t.txt link` → -`Invalid path (must be absolute)`; absolute targets work); a redirect to a -missing parent throws out of `bash.exec()` rather than returning nonzero — the -spike wraps exec as upstream's entrypoint does; `ls -l` reports a stale mode -because the adapter's `readdirWithFileTypes` carries none. - -## Which `API.Process` semantics a Deno-local provider could claim - -Claimable today: run a command string in a workspace-scoped shell; supply cwd -and environment; receive stdout, stderr, and an exit status; run a JavaScript -module graph with a workspace capability and a returned value; refuse anything -unsupported explicitly. - -**Not** claimable: arbitrary native command execution (no launcher exists); -preemptive cancellation of CPU-bound work; live incremental streaming from the -shell; retained or reattachable executions; POSIX process semantics such as -signals, pipes to native processes, or a process table. A provider must not -describe the shell backend as native command execution. - -See [COMPARISON.md](COMPARISON.md) for the topology comparison with #347 and -#349 and the recommendation for #346. diff --git a/spikes/351-worker-backends/evidence/probes/slice1-shell.md b/spikes/351-worker-backends/evidence/probes/slice1-shell.md deleted file mode 100644 index abd0bea0..00000000 --- a/spikes/351-worker-backends/evidence/probes/slice1-shell.md +++ /dev/null @@ -1,657 +0,0 @@ -# Probe: Worker Shell natively in Deno over Deno-local DOFS (#351 slice 1) - -**Verdict: ADOPT** — Cloudflare's Worker Shell runs natively under Deno over the -Deno-local DOFS workspace with no workerd, no Wrangler, no Docker and no Node as -runtime. The `WorkspaceFsAdapter` is reused **verbatim**. Three Deno-specific -defects were found, root-caused and fixed/characterised; none is a blocker for the -core shell. - -Environment: Deno 2.9.1 (stable, aarch64-apple-darwin), macOS 25.5.0. Node 26.5.1 -present only as a comparison oracle for one node-compat probe — never as a runtime -for the shell. - -## Provenance / version facts - -| Artifact | Fact | -| --- | --- | -| `just-bash` | `3.0.1`, `sha512-YVyzCN08fKarUnwqy7rKOAcX+2MLYLnYInuowmUXn3mqhrtd4ieZNBuzdQG+qYV9DqnIWuv9Whiph0WRIWsBtw==` (npm tarball integrity) | -| just-bash origin | `github.com/vercel-labs/just-bash`, Apache-2.0, author "Malte and Claude" — **a Vercel Labs package, not a Cloudflare one** | -| `dist/bundle/index.js` (ESM entry) | sha256 `565b3d15a9b77e0dc77a7ec041f149a06f7aa304155a9509d61231027abec4e5`, 256 124 bytes | -| `dist/bundle/browser.js` (browser entry) | sha256 `0bf49f45d9cb3299e24fd3e98ba4cc9076469e0b727bf6193a58a86fbebcc12c`, 1 098 224 bytes | -| `adapter.ts` (copied verbatim from cf-computer) | sha1 `c9ec5356d60ee7bc9aa3c5677320bf8c9aa4058c`, 355 lines, unmodified | -| `quickjs-emscripten` | 0.32.0 | -| `sql.js` | 1.14.1 | -| DOFS | vendored `@cloudflare/dofs` 0.0.0 from `spikes/349-dofs/vendor/dofs`, built with `npx tsc -p tsconfig.build.json` | - -`just-bash` package export map has `"browser": "./dist/bundle/browser.js"` and -`"./browser"` subpath — a browser build exists (1.07 MB single file, `node:dns` -aliased to a stub). - -## Wiring actually used - -``` -FileSQLiteStorage(dbPath) # node:sqlite DatabaseSync, WAL — from spikes/349-dofs - -> Database(storage) - -> initializeSchema(db, Date.now) - -> WorkspaceFilesystem(db) - -> WorkspaceFsShim # adds exists/statOrNull/lstatOrNull (see below) - -> WorkspaceFsAdapter(shim) # Cloudflare's file, verbatim - -> new Bash({ fs, cwd, defenseInDepth: { enabled: false }, executionLimits }) -``` - -`WorkspaceFsShim` (`workspace.ts`) is 3 real methods of work: DOFS's -`WorkspaceFilesystem` has no `exists`/`statOrNull`/`lstatOrNull` (Cloudflare's -`WorkspaceFilesystemStub` adds them on the RPC host side). `statOrNull = try stat -catch null`; `exists = statOrNull !== null`. Everything else forwards 1:1. - -Nothing from `cloudflare:workers` is needed. `WorkerEntrypoint`, `env.HOST.getWorkspace()` -and the RPC framing are all removed, exactly as the established seam predicted. - ---- - -## Goal 1 — Load + run - -`import { Bash } from "just-bash"` resolves under Deno via `nodeModulesDir: "manual"` -+ `npm i --install-links`. No import shims, no remapping. - -``` -$ deno run -A --node-modules-dir=manual p1-load.ts /tmp/p1.db did-false -{ "result": { "stdout": "hello\n", "stderr": "", "exitCode": 0, ... }, - "timings_ms": { "openWorkspace": 2.54, "bashCtor": 8.21, "firstExec": 14.03, "secondExec": 0.57 } } -``` - -### DEFECT 1 (blocker if unhandled) — `defenseInDepth` is ON BY DEFAULT and hard-fails under Deno - -`defenseInDepth: { enabled: true }` **and the default (option omitted)** both abort -every `exec`: - -``` -[DefenseInDepthBox] Could not patch process.report: Cannot redefine property: report -[DefenseInDepthBox] Could not protect Module._load: Cannot read properties of null (reading 'constructor') -[DefenseInDepthBox] Could not protect Module._resolveFilename: Cannot read properties of null (reading 'constructor') -[DefenseInDepthBox] Could not patch process.execPath: Cannot redefine property: execPath -error: Uncaught (in promise) Error: DefenseInDepthBox: critical patches failed: Module._load - at applyPatches (node_modules/just-bash/dist/bundle/chunks/chunk-TN7HHBQW.js:4:8337) - at activate (…chunk-TN7HHBQW.js:4:2973) - at Ft.exec (node_modules/just-bash/dist/bundle/index.js:700:832) -``` - -**Fix: pass `defenseInDepth: { enabled: false }`.** This is the *same line* -Cloudflare already passes for workerd, for a different underlying reason (workerd's -`node:module.registerHooks` throws "not implemented"; Deno's `node:module` has no -`Module._load` to patch at all). The committed spike must carry this line and a -comment saying it is load-bearing on **both** targets. Security impact is analysed -in Goal 5 — measured answer: none. - ---- - -## Goal 2 — Bidirectional coherence + restart across processes ✅ PASS - -Process 1 (`p2-coherence.ts … write`): - -``` -cat a.txt (API wrote it) exit=0 stdout="written-by-api\n" -echo written-by-shell > b.txt exit=0 - API readFile(/workspace/b.txt) = "written-by-shell\n" - API stat size=17 mode=644 -``` - -Process 2 — **a fresh Deno process against the same db file**, host "restart": - -``` -cat b.txt after restart exit=0 stdout="written-by-shell\n" -cat a.txt after restart exit=0 stdout="written-by-api\n" -ls -la exit=0 -rw-r--r-- … a.txt / b.txt -``` - -API→shell, shell→API, and durability across a process boundary all hold. - ---- - -## Goal 3 — Shell semantics - -Full matrix from `p3-semantics.ts` (48 cases). Everything below is verbatim. - -| Feature | Result | -| --- | --- | -| `pwd`, `cd` within a script | ✅ `/workspace` → `/workspace/sub` | -| cwd resets between `exec` calls | ✅ by design (per-exec `cwd`) | -| env from `exec({env})` | ✅ `echo $FOO` → `bar` | -| `export` inside script | ✅ | -| pipelines, multi-stage | ✅ `echo hi \| tr a-z A-Z` → `HI`; `printf \| sort \| head \| tr` → `A B` | -| `>` `>>` `<` | ✅ | -| `2>` and `2>&1` | ✅ | -| exit status `$?`, `false`/`true` | ✅ `1` / `0` | -| stdout vs stderr separation | ✅ `out="OUT\n" err="ERR\n"` — separate strings on the result | -| `$(...)` and backticks | ✅ | -| arithmetic `$((…))` | ✅ `14` | -| globbing `ls *.txt` | ✅ (no-match → exit 2 + `ls: *.zzz: No such file or directory`) | -| brace expansion | ✅ `a1 a2 b1 b2` | -| multi-line script: `set -e`, `for`, `if`, `case`, function, `while` | ✅ all | -| here-doc (`rawScript: true`), here-string | ✅ | -| nonexistent command | ✅ exit **127**, `bash: definitely_not_a_command: command not found` | -| stdin via `exec({stdin})`, `read` builtin | ✅ | -| subshell `( … )` | ✅ cwd restored | -| `&&` / `\|\|` | ✅ | -| quoting, `${v:1:3}`, `${#v}`, `${v/a/b}`, `${u:-def}` | ✅ | -| arrays `a=(x y z)` | ✅ | -| `cp`/`mv`/`rm`, `chmod`, `ln -s` (absolute) | ✅ | -| `ln` (hard link) | ✅ *fails loudly*: `ln: hard links are not supported by the workspace store` (adapter throws ENOSYS by design) | -| `/dev/null` | ✅ (adapter's virtual `/dev` shim) | -| **`trap`** | ❌ `bash: trap: command not found` — not implemented | -| **`yes`** | ❌ not implemented | -| **`xargs -n1`** | ❌ `xargs: invalid option -- 'n'` (bare `xargs` works) | -| **`ln -s` with a RELATIVE target** | ❌ see Defect 2 | - -### Cancellation — the most important finding of this probe - -| Script | Abort at | Outcome | -| --- | --- | --- | -| `sleep 10; echo never` | 300 ms | ✅ settles at **304 ms**, `exitCode 124` | -| `timeout 0.2 sleep 5` | n/a | ✅ builtin works, 206 ms | -| 200 top-level `sleep 0.02` statements | 300 ms | ✅ exit 124 at 302 ms | -| `for i in $(seq 1 200); do sleep 0.02; done` | 300 ms | ✅ exit 124 at 302 ms | -| `i=0; while [ $i -lt 300000 ]; do i=$((i+1)); done` | 300 ms | ❌ **ignored** — ran 6 378 ms to completion, `exitCode 0`, `signal.aborted === false` | -| `while true; do i=$((i+1)); done` (budgets raised) | 300 ms | ❌ **never returned**; observed 56 s at 102 % CPU, killed by hand | - -The `signal.aborted === false` after 6.4 s is the decisive datum: **the abort -timer itself never fired.** A host `setInterval(…, 50)` running across that same -exec ticked **0 times, expected ~127**. just-bash evaluates a loop body -synchronously with no yield to the event loop, so a CPU-bound shell script starves -the entire isolate — timers, the abort callback, and every other host task. - -- Abort is honoured only where the interpreter already `await`s (a `sleep`, an fs - op). It is *not* a preemption mechanism. -- The only in-isolate guard against a runaway CPU loop is - `executionLimits.maxCommandCount` (default **10 000**), an interpreter-internal - synchronous counter. Measured: `while true; do :; done` stops in **20 ms** with - `exit 126`, `bash: too many commands executed (>10000)`. Raise that limit and - you lose the only brake. -- Abort exit code is **124**, not 130. Cloudflare's entrypoint maps to 130 itself - in its own `catch`; a Deno backend must do the same mapping if it wants 130. - -This is what makes the Worker boundary (Goal 6) a requirement rather than a nicety. - ---- - -## Goal 4 — Command-support matrix - -`just-bash` exports: `Bash, BashTransformPipeline, CommandCollectorPlugin, -DefenseInDepthBox, EMPTY_BYTES, InMemoryFs, MountableFs, NetworkAccessDeniedError, -OverlayFs, ReadWriteFs, RedirectNotAllowedError, Sandbox, SandboxCommand, -SecurityViolationError, SecurityViolationLogger, TeePlugin, TooManyRedirectsError, -bytesOutput, createConsoleViolationCallback, decodeBytesToUtf8, defineCommand, -encodeUtf8ToBytes, getCommandNames, getJavaScriptCommandNames, -getNetworkCommandNames, getPythonCommandNames, latin1FromBytes, parse, serialize, -stdoutAsBytes, stdoutKind, textOutput, unsafeBytesFromLatin1`. - -### Class 1 — built-ins, registered by default (83 names, all present) - -`echo cat printf ls mkdir rmdir touch rm cp mv ln chmod pwd readlink head tail wc -stat grep fgrep egrep rg sed awk sort uniq comm cut paste tr rev nl fold expand -unexpand strings split column join tee find basename dirname tree du env printenv -alias unalias history xargs true false clear bash sh jq base64 diff date sleep -timeout seq expr md5sum sha1sum sha256sum file html-to-markdown help which tac -hostname od gzip gunzip zcat tar yq xan sqlite3 time whoami` - -Functional smoke (all exit 0 unless noted): - -| Command | Observed | -| --- | --- | -| `ls` / `cat` / `head`/`tail` / `wc` | `a.txt d` / `alpha beta` / `alpha…beta` / `2` | -| `grep` / `rg` | `beta` | -| `sed -n 2p` | `beta` | -| `awk 'NR==1{print $1}'` | `alpha` | -| `sort` / `uniq -c` | sorted / `2 a`, `1 b` | -| `cut -d: -f2` / `tr a-c A-C` | `b` / `ABC` | -| `find . -type f` | `./a.txt ./d/x.txt` | -| `du -sh` / `tree` | `13 .` / full tree render | -| `diff` | exit 1 + unified diff (correct) | -| `jq '.k[1]'` / `yq '.a'` | `2` / `1` | -| `base64` round trip | `hi` | -| `md5sum` / `sha256sum` | correct digests | -| `date`/`seq`/`expr` | `1 2 3 7` | -| `tar -cf` + `gzip -c` | archive created in DOFS | -| `od -c` / `file` | hexdump / `a.txt: ASCII text` | -| `tee`, `xargs`, `column`, `nl`, `time` | all work | -| `sh -c` / `bash -c` | `via-sh` / `via-bash` — **re-enters just-bash's own interpreter, not a native shell** | -| `html-to-markdown` | `# T` | -| `xan headers` | CSV headers | -| `sqlite3` | works (after Defect 3 fix) — see below | - -### Class 2 — gated commands, OFF by default (verified) - -With default `BashOptions` all of these are exit 127: - -``` -python3 -c 'print(1)' exit=127 bash: python3: command not available in browser environments… -python --version exit=127 (same) -js-exec -c '…' exit=127 bash: js-exec: command not found -curl https://example.com exit=127 bash: curl: command not found -node -e '…' exit=127 bash: node: command not found -``` - -- `javascript: true` → `js-exec`, `node`. QuickJS-in-WASM inside a - `node:worker_threads` worker. Works under Deno after the Defect 3 fix. -- `python: true` → `python3`, `python`. CPython-Emscripten inside a worker. - **Partially broken under Deno** — see Defect 3. -- `network: {...}` or `fetch:` → `curl` (there is no `wget`). - -### Class 3 — custom commands (host-registered) - -`defineCommand(name, handler)` + `customCommands: [...]`. Cloudflare registers -`git`, `assets`, `artifacts` this way, each backed by host code, not a binary -(`git-command.ts`, `assets-command.ts`, `artifacts-command.ts` in the pinned -tree). Not wired in this probe — noted as available and pure-host. - -### Class 4 — native subprocesses: **NONE** - -No command shells out. Verified three ways: - -1. `which sh; which node; which deno; which python3` → all empty, exit 1. -2. `/bin/sh -c 'id'` → `bash: /bin/sh: No such file or directory`; - `$(/usr/bin/id)` → same. `id`, `uname`, `ps`, `git`, `make`, `sudo`, `ssh`, - `openssl`, `nc` and 13 other native tools: all `command not found`. -3. The QuickJS sandbox's `child_process` shim (`execSync`/`spawnSync`) — - the one plausible native path — **re-enters just-bash**: - `spawnSync("uname", ["-a"])` → `{"stdout":"","stderr":"bash: uname: command not found\n","status":127}`. - The single `child_process` string in the main bundle is inside a - DefenseInDepth *deny* message, not a call site. - ---- - -## Goal 5 — SECURITY (measured, not inferred) - -Canaries planted before every run: host file `/tmp/probe-host-secret.txt` -containing `HOST-SECRET-CANARY`, and `Deno.env HOST_CANARY=HOST-ENV-CANARY`. -Every result below was scanned for those strings plus `root:x:`. **Zero leaks -across every probe in this document.** - -### A. Host filesystem — fully confined - -| Attempt | Outcome | -| --- | --- | -| `cat /etc/passwd` | exit 1 `cat: /etc/passwd: No such file or directory` | -| `cat /tmp/probe-host-secret.txt` | exit 1, no such file | -| `ls /` | `workspace` — the DOFS root only | -| `ls /tmp`, `ls /Users/tarasmankovski` | exit 2, no such file | -| `cat ../../../../../../etc/passwd` | exit 1, no such file (adapter's `normalizePath` clamps `..` at root) | -| `cd / && ls -la` | `.`, `..`, `workspace/` | -| `find / -name 'probe-host-secret*'` | empty | -| `grep -r HOST-SECRET /` | empty | -| `readlink /proc/self/exe`, `cat /proc/self/status` | no such file | -| `echo pwned > /tmp/probe-shell-escape.txt` | `WorkspaceFsError: parent directory missing` — the write went into the **DOFS namespace**, never the host | -| `echo pwned > /etc/probe.txt` | same | -| `tar -cf /tmp/x.tar /etc/passwd` | `tar: /etc/passwd: ENOENT` | -| host-side verification | `/tmp/probe-shell-escape.txt`, `/etc/probe.txt`, `/tmp/host-probe.db`, `/tmp/probe-cp-escape.txt` — **none created** | - -The adapter only ever calls `WorkspaceFs` methods; no built-in was found that -bypasses it. - -### B. Host environment — fully synthetic - -``` -env -> HOME=/ PATH=/usr/bin:/bin IFS=… OSTYPE=linux-gnu - MACHTYPE=x86_64-pc-linux-gnu HOSTTYPE=x86_64 HOSTNAME=localhost - PWD=/workspace OLDPWD=/workspace OPTIND=1 SHELLOPTS=… BASHOPTS=… -printenv HOST_CANARY -> rc=1 (unset) -echo $USER $SHELL -> empty -whoami / hostname -> user / localhost -echo $$ $PPID $UID -> pid=1 ppid=0 uid=1000 (virtual, per `processInfo`) -``` - -The host's real env is invisible. `OSTYPE=linux-gnu` on macOS confirms the env is -fabricated, not inherited. Only explicitly-passed `exec({env})` reaches the script. - -### C. Network - -Default (no `network`/`fetch` option): `curl` and `wget` are **not registered** — -exit 127. `NetworkAccessDeniedError` is exported for the opted-in case. - -Opted in with `network: { allowedUrlPrefixes: ["https://example.com"] }`: - -| Request | Outcome | -| --- | --- | -| `curl https://example.com` | **200 — reaches the real network** | -| `curl https://api.github.com` | exit 7 (denied, not on the allowlist) | -| `curl http://169.254.169.254/latest/meta-data/` | exit 7 (cloud-metadata SSRF denied) | -| `curl file:///etc/passwd` | exit 7 | -| `curl http://localhost:22` | exit 7 — **denied even when `http://localhost` IS on the allowlist** | - -The last row is the interesting one: `dist/network/allow-list.d.ts` has "Check if -a hostname is a private/loopback IP address" and `dns-pin.d.ts` pins DNS to the -preflight-validated IP "defeating DNS rebinding". So the network capability is a -strict prefix allowlist **plus** private/loopback rejection **plus** DNS-rebinding -pinning. Opting in grants exactly the named origins and nothing else. - -### D. QuickJS `js-exec` sandbox (`javascript: true`) - -Every probe below ran **inside** the sandbox via `js-exec -c`: - -| Attempt | Result | -| --- | --- | -| `console.log("js alive", 1+1)` | `js alive 2` — the sandbox works | -| `typeof Deno`, `typeof globalThis.Deno` | `undefined`, `undefined` | -| `Deno.env.get("HOST_CANARY")` | `ReferenceError: 'Deno' is not defined` | -| `Deno.readTextFileSync("/tmp/probe-host-secret.txt")` | `ReferenceError: 'Deno' is not defined` | -| `typeof process / require / module / Buffer` | `object / function / undefined / function` — **shims**, not Node's | -| `process.binding`, `process.dlopen`, `process.getBuiltinModule` | all `undefined` | -| `Object.getOwnPropertyNames(globalThis)` | pure ECMA-262 set (`Error … BigInt64Array`) — no host objects | -| `Function("return this")()` | `TypeError: Function constructor is not allowed` | -| `(function(){}).constructor("…")()` | `TypeError: Function constructor is not allowed` | -| `import("node:fs")` | `ReferenceError: could not load module 'node:fs'` | -| `require("fs")` | returns the **workspace-scoped shim**: `["readFile","readFileBuffer","writeFile","stat","readdir"]` | -| `require("fs").readFileSync("/tmp/probe-host-secret.txt")` | `Error: no such file: ` (path even redacted) | -| `require("fs").writeFileSync("/workspace/from-js.txt", …)` | ✅ lands in DOFS — host verified `"js-wrote-this\n"` | -| `require("child_process").execSync("uname -a")` | `status 127`, `bash: uname: command not found` — re-enters just-bash | -| `child_process.execSync("echo escaped > /tmp/probe-cp-escape.txt")` | `parent directory missing` — host file never created | -| `fetch("https://example.com")` (network not configured) | `TypeError: Network access not configured. Enable network in Bash options.` | -| `typeof Atomics`, `typeof WebAssembly` | `undefined`, `undefined` | -| `typeof SharedArrayBuffer` | `function` (QuickJS's own, not the host's) | -| CPU spin 3 s inside QuickJS | runs 3 006 ms — **but in a worker thread, so the host isolate stays free** | - -quickjs-emscripten is a separate WASM VM with an explicitly-built capability -surface. Nothing in the guest reaches the Deno isolate. Proven, not assumed. - -### E. Python (`python: true`) — CPython-Emscripten - -| Attempt | Result | -| --- | --- | -| `print("py alive")` | `py alive` | -| `open("/etc/passwd").read()` | `FileNotFoundError: [Errno 44] … '/host/etc/passwd'` — reroutes into the sandbox namespace | -| `os.environ.get("HOST_CANARY")` | `None` | -| `subprocess.run(["id"])` | `OSError: [Errno 138] emscripten does not support …` | -| `os.system("id")` | `-1` | -| `socket.connect(("example.com", 80))` | `OSError: [Errno 23] Host is unreachable` | -| `open("/workspace/from-py.txt","w")` | ✅ lands in DOFS | - -Confined — but see Defect 3: python3 is not production-usable under Deno. - -### F. Defense-in-depth verdict - -**`DefenseInDepthBox` is not the thing keeping the shell contained, and it cannot -run under Deno at all.** Measured: - -- With `defenseInDepth: { enabled: false }` — the configuration Cloudflare ships - and the only one that runs under Deno — **every one of the ~50 escape attempts - above failed.** No host file, no host env var, no native process, no network, - no Deno global. -- Containment comes from architecture, not from monkey-patching: (a) the only - filesystem the interpreter has is the injected `IFileSystem`; (b) the env is - fabricated; (c) there is no `child_process`/`spawn` call site anywhere in the - bundle; (d) network commands are not even registered unless configured; (e) - guest code execution happens in a separate WASM VM with a hand-built global set. -- What DID actually patches (`process.dlopen`, `process.getBuiltinModule`, - `Module._load`, `process.exit`, `process.report`, `process.execPath`) is a - hardening layer against *bugs in the above*, aimed at Node hosts. Under Deno - most of those handles do not exist to be patched. - -**Is a Deno Worker boundary necessary for security?** On the evidence: not for -*capability* containment — the interpreter is inherently sandboxed. It **is** -necessary for *availability*: the Goal 3 measurement shows a CPU-bound script -wedges the isolate with no cooperative escape. That is a denial-of-service escape -from the host's control, and only a terminable isolate answers it. - ---- - -## Goal 6 — Deno Worker boundary ✅ VIABLE, and recommended - -**Can the fs capability cross?** Not as a handle: - -``` -structuredClone(WorkspaceFilesystem): DataCloneError: (query, ...bindings)=>{ … } could not be cloned. -structuredClone(Database): DataCloneError: (query, ...bindings)=>{ … } could not be cloned. -``` - -Option (a) — Bash in the main isolate — was implemented and measured (Goals 1–5). -Option (b) — Bash in a `new Worker(url, {type:"module"})`, DOFS staying in the main -isolate, `WorkspaceFs` crossing as an **async postMessage RPC proxy** — was -prototyped (`worker-shell.ts` + `p6-worker.ts`) and works. - -The adapter tolerates the async proxy without modification: every `WorkspaceFs` -method already returns a Promise, so an RPC is indistinguishable from a local -await. `readFile`'s stream overload is the only special case — ship bytes and -rebuild the stream worker-side, which is exactly what the adapter's -`new Response(stream).arrayBuffer()` consumes. - -### Functional parity through the RPC proxy — 10/10 - -``` -"echo hello" exit=0 host=6.7ms worker=6.5ms out="hello\n" -"echo via-worker > w.txt; cat w.txt" exit=0 host=6.8ms out="via-worker\n" -"mkdir -p wd && echo deep > wd/f.txt && find ." exit=0 out="./w.txt\n./wd/f.txt\n" -"printf 'b\na\n' > s.txt; sort s.txt | tr …" exit=0 out="A\nB\n" -"ls -la" exit=0 -"grep -r deep . | head -2" exit=0 out="wd/f.txt:deep\n" -"sed -i.bak 's/deep/DEEP/' wd/f.txt; cat …" exit=0 out="DEEP\n" -"cat missing.txt" exit=1 err="cat: missing.txt: No such file or directory\n" -"chmod 700 w.txt; stat w.txt | head -3" exit=0 out="Access: (0700/-rwx------)" -"ln -s /workspace/w.txt wl.txt; cat wl.txt" exit=0 out="via-worker\n" -(103 fs RPC round trips, 9.7 ms total host-side fs work) -``` - -### Cost - -| Measurement | In-isolate | Worker + RPC | -| --- | --- | --- | -| Worker boot + Bash construction | — | **39.6 ms** (once) | -| `echo x` (mean of 30) | 0.30 ms | **0.27 ms** (4 fs RPCs/exec) | -| write + read a file (mean of 30) | 0.52 ms | **0.97 ms** (11 fs RPCs/exec) | - -≈ **0.04 ms per RPC round trip**. `echo` is a wash; a filesystem-touching command -costs +0.45 ms. Negligible against a 40 ms boot amortised across a session. - -### The decisive benefit - -| | In-isolate | Worker | -| --- | --- | --- | -| Host 50 ms ticks during a worker-side CPU loop | **0** (starved) | **3 of ~3** — host stayed fully responsive | -| Unbounded `while true` loop | unrecoverable; 56 s at 102 % CPU, abort timer never fired | `worker.terminate()` reclaims it in ~1.3 s while the host ticked 9× in 500 ms | - -**Recommendation: the committed spike should run Bash in a Deno Worker** with the -fs proxied by RPC. The security case is a tie (the interpreter is sandboxed either -way); the availability case is decisive, and it costs 40 ms + 0.04 ms/RPC. It also -mirrors Cloudflare's own topology (shell in a separate isolate, fs over RPC to the -host that owns it), so the two backends stay structurally comparable. - ---- - -## Goal 7 — Latency, size, bundling - -Cold path, interpreted: - -``` -openWorkspace + Bash ctor 11.09 ms -cold `echo hello` 5.67 ms -fs command 9.23 ms -warm `echo` 0.17 ms -``` - -`deno compile` — **works**, with `--no-check` (only because this probe's -`workspace.ts` is untyped JS-in-TS; a typed spike will not need it): - -``` -$ deno compile -A --no-check --node-modules-dir=manual --output /tmp/probe-shell-bin p7-compile.ts -Files: 100.76MB → binary 167 MB -$ /tmp/probe-shell-bin /tmp/p7b.db -{"echo":"hello","fs":["body","4"],"warm":"warm", - "ms":{"openWorkspaceAndBash":12.71,"coldEcho":8.03,"fsCommand":10.47,"warmEcho":0.17}} -sqlite3: {"exit":0,"out":"9","err":""} -``` - -The compiled binary runs the shell **and** `sqlite3` — meaning even the -worker-thread commands, which resolve their worker file from a disk path, work -because `deno compile` embedded `node_modules/*`. Compiled timings match -interpreted within noise. - -Size breakdown (unpacked `node_modules`, 111 MB total): - -| Package | Size | -| --- | --- | -| `just-bash` | 21 MB (`dist/bundle` 5.1 MB, `vendor/cpython-emscripten` 9.9 MB) | -| `sql.js` | 18 MB | -| `typescript` (dofs build dep only — droppable) | 24 MB | -| `@cloudflare/dofs` vendored | 12 MB | -| `quickjs-emscripten` (+ `@jitl/*`) | ~9 MB | - -Trimmable: `typescript` is a dofs *build* dependency; `vendor/cpython-emscripten` -(9.9 MB) is dead weight unless `python: true`; `sql.js` (18 MB) unless `sqlite3` -is wanted. The core-shell footprint is the 256 KB `dist/bundle/index.js` plus -runtime deps. A `dist/bundle/browser.js` single-file build (1.07 MB) exists and is -the natural target if a leaner graph is wanted. - -Two optional deps have **native install scripts that npm did not run**: -`@mongodb-js/zstd` (prebuild-install) and `node-liblzma` (node-gyp). The shell -worked fully without them — the spike should keep them unbuilt and note that -`gzip`/`tar` work regardless. - ---- - -## Defects found (all root-caused) - -### Defect 1 — `defenseInDepth` default-on, fatal under Deno -Covered above. **Fix: `defenseInDepth: { enabled: false }`** — one line, already -present in Cloudflare's entrypoint. - -### Defect 2 — relative symlinks are broken *in DOFS itself* (affects Cloudflare too) - -``` -DOFS lstat(rel.link).isSymbolicLink = true -DOFS readlink(rel.link) = t.txt -DOFS stat(rel.link) THREW: Invalid path (must be absolute): t.txt -DOFS readFile(rel.link) THREW: Invalid path (must be absolute): t.txt -DOFS readFile(abs.link) = "body\n" ← absolute target works -shell: cat rel.link -> "cat: rel.link: No such file or directory" -shell: cat abs.link -> "body" -``` - -The failure is in `@cloudflare/dofs`'s own resolver, not in the adapter or in -just-bash: DOFS does not resolve a symlink target relative to the link's parent. -`ln -s t.txt rel.link` — the ordinary form — produces a dangling link. This is -upstream and applies equally to Cloudflare's Worker Shell. Worth reporting. - -### Defect 3 — `process.connected` throws in Deno workers, breaking every worker-backed command - -Root cause, isolated to a 20-line reproduction driving just-bash's own worker: - -``` -$ deno run -A p5c-worker-root-cause.ts -MSG #1: {"success":false,"error":"process.connected is not supported in workers"} -``` - -`js-exec-worker.js` runs `initializeWithDefense()` → `new WorkerDefenseInDepth(...)` -unconditionally (**not** controlled by the host's `defenseInDepth` option — the -config never crosses into the worker). Under Deno, reading `process.connected` -inside a worker throws; the resulting error message carries no `protocolToken`, so -the host's first-message handler rejects it as `Malformed worker response: invalid -protocol token`. Symptoms per command: - -- `js-exec` — code **runs correctly** and stdout arrives over the SharedArrayBuffer - bridge, but exit code is always **1** with that stderr. -- `python3` — **10 s timeout on every invocation**, exit 124. -- `sqlite3` — exit 1, `sqlite3: process.connected is not supported in workers`. - -Runtime differential, confirmed: - -``` -deno: read="THREW process.connected is not supported in workers" write="defineProperty ok -> false" -node: read="undefined" write="defineProperty ok -> false" -``` - -**Fix (verified):** prepend one line to the shipped worker files — - -```js -import process from "node:process"; -try { Object.defineProperty(process, "connected", { value: false, configurable: true, writable: true }); } catch (e) {} -``` - -Applied to `dist/bundle/chunks/js-exec-worker.js`, `worker.js` (python3) and -`sqlite3-worker.js` (originals kept as `*.orig`). After the patch: - -- `js-exec` — **exit 0 everywhere**, all 16 sandbox probes clean. -- `sqlite3` — **exit 0**, `select * from t` → `42`, database written into DOFS, - host path `/tmp/host-probe.db` refused and never created. -- `python3` — runs (`py alive`, confinement all correct) but still exits 1 with - `Fatal Python error: gilstate_tss_clear: failed to clear current tstate (TSS)` / - `Aborted()` / `python3: Security violation: webassembly` on every invocation. - **python3 remains not production-usable under Deno.** It is off by default, so - this does not block anything. - -Placement note: this is a Deno node-compat gap (Deno's worker `process.connected` -getter throws where Node returns `undefined`). Cleanest resolutions, in order: -(1) upstream Deno fix; (2) upstream just-bash guarding the defense init in a -try/catch; (3) a vendored patch in the spike. Since `js-exec`/`python3`/`sqlite3` -are all opt-in and the core shell never touches a worker, the spike can ship -without any of them and record this as a known limitation. - -### Defect 4 (fidelity) — `ls -l` reports a stale mode - -``` -DOFS stat mode = 700 -adapter.stat mode = 700 -adapter.readdirWithFileTypes keys = ["name","isFile","isDirectory","isSymbolicLink"] -shell `ls -l t.txt` = "-rw-r--r-- 1 user user 5 …" ← wrong -shell `stat t.txt` = "Access: (0700/-rwx------)" ← right -shell `[ -x t.txt ]` = "yes" ← right -``` - -`chmod` works end to end; only `ls -l` is wrong, because the adapter's -`readdirWithFileTypes` carries no `mode` and just-bash's `ls` falls back to a -default. This is in Cloudflare's verbatim adapter, so it affects their shell too. - -### Defect 5 (contract) — some fs errors escape `exec()` as exceptions - -``` -echo x > /nonexistent-dir/f.txt *** THREW OUT OF exec(): WorkspaceFsError: parent directory missing -mkdir /a/b/c exit=1 err="mkdir: cannot create directory …" ← correct -cp missing.txt d.txt exit=1 err="cp: cannot stat 'missing.txt' …" ← correct -``` - -A redirection to a missing parent directory rejects the `exec` Promise instead of -returning a non-zero exit. **Callers must wrap `bash.exec` in try/catch** — -Cloudflare's entrypoint already does, mapping to `exitCode: 1`. The spike must -replicate that wrapper or the first bad redirect crashes the host. - ---- - -## What the committed spike must replicate - -1. `WorkspaceFsAdapter` copied **verbatim** (sha1 `c9ec5356…`) with provenance, - plus a ~60-line `WorkspaceFsShim` supplying `exists`/`statOrNull`/`lstatOrNull` - over DOFS's `WorkspaceFilesystem`. -2. `defenseInDepth: { enabled: false }` with a comment stating it is load-bearing - on Deno *and* workerd, and that containment does not depend on it. -3. `executionLimits` explicitly set — `maxOutputSize` **and** `maxCommandCount`. - The default 10 000 command budget is the only in-isolate brake on a runaway - loop; raising it without a Worker boundary removes the last guard. -4. `try/catch` around `bash.exec` mapping a thrown `WorkspaceFsError` to a - non-zero exit (Defect 5), and the abort exit-code mapping 124 → 130 if that - code is part of the contract. -5. Bash inside a **Deno Worker**, `WorkspaceFs` crossing as an async RPC proxy; - `readFile`'s stream overload shipped as bytes and rebuilt worker-side. Cancel - = `worker.terminate()`, not `AbortSignal` alone. -6. Regression tests pinning the measured behaviour: the restart-coherence pair, - `exit 127` for a missing command, `exit 124` on abort of an awaiting script, - the confinement canaries (host file + host env + native exec + network), and - the `sh -c`/`execSync` re-entry proof. -7. Known limitations recorded: relative symlinks (Defect 2, upstream DOFS), - `ls -l` mode (Defect 4, upstream adapter), `trap`/`yes`/`xargs -n` absent, - python3 unusable under Deno (Defect 3), `js-exec`/`sqlite3` needing the - `process.connected` shim if enabled. - -## Files in this probe - -| File | Purpose | -| --- | --- | -| `adapter.ts` | Cloudflare's `WorkspaceFsAdapter`, verbatim | -| `file-storage.ts` | `FileSQLiteStorage` from `spikes/349-dofs` | -| `workspace.ts` | `WorkspaceFsShim` + `openWorkspace()` wiring | -| `lib.ts` | `makeBash`/`open` helpers | -| `p1-load.ts` | Goal 1 — load, construct, defenseInDepth modes | -| `p2-coherence.ts` | Goal 2 — bidirectional + restart | -| `p3-semantics.ts` / `p3b` / `p3c` / `p3d` / `p3e` | Goal 3 — semantics, cancellation, the wedge proof | -| `p4-commands.ts` / `p4b-gating.ts` | Goal 4 — command matrix and gating | -| `p5-security.ts` / `p5b` / `p5c` / `p5d` / `p5e` / `p5f` / `p5g` | Goal 5 — escape attempts, worker root cause, child_process, network | -| `p6-worker.ts` + `worker-shell.ts` | Goal 6 — Deno Worker + fs RPC prototype | -| `p7-compile.ts` | Goal 7 — cold latency + `deno compile` | -| `p8-fidelity.ts` | Defects 2, 4, 5 pinned to their layer | diff --git a/spikes/351-worker-backends/evidence/probes/slice2-javascript.md b/spikes/351-worker-backends/evidence/probes/slice2-javascript.md deleted file mode 100644 index c04c50af..00000000 --- a/spikes/351-worker-backends/evidence/probes/slice2-javascript.md +++ /dev/null @@ -1,318 +0,0 @@ -# Issue #351 slice 2 — Worker JavaScript in a Deno Worker isolate - -Deno 2.9.1 (aarch64-apple-darwin), macOS 25.5.0. All artefacts in this directory. - -## Verdict: **LIMIT** — portable, with two named gaps - -A compiled Deno host **can** run Cloudflare's Worker JavaScript execution model -natively — no workerd, no Wrangler, no Docker, no Node-as-runtime. The full -contract (in-memory 2-module graph, DOFS capability, framed stdout/stderr, -result, error, timeout, cancellation) is proven end-to-end in `deno run` **and** -in a `deno compile`d binary, with identical output. - -Two things do not port faithfully, both named precisely below: - -1. **`Worker.terminate()` cannot reclaim a CPU-spinning worker.** The timeout - *result* is correct but the *thread is leaked* and process exit is blocked. - Missing Deno-side primitive: **a CPU/wall budget on a Worker, or a - `terminate()` that interrupts running JS** — Cloudflare's `limits: {cpuMs}` - has no Deno equivalent. -2. **`compatibilityDate` / `compatibilityFlags` have no Deno equivalent**, and - **retained execution / reattach (`getExec`/`resume`) is not expressible** — - a Deno Worker's lifetime is bounded by the host call. - -Everything else — including isolation, which was the risk — holds, and holds -*better* in the compiled binary than under `deno run`. - ---- - -## 1. Module-supply determination — THE core deliverable - -**Answer: blob: URLs with host-side specifier rewriting. No disk -materialization is required.** `deno compile` does not break it. - -### 1a. blob: URLs — `exp1a-blob.ts` - -``` -$ deno run -A exp1a-blob.ts -error: Uncaught (in worker "") invalid URL: relative URL with a cannot-be-a-base base - at blob:null/cee6b145-74cb-4411-9bf1-49aa9a7a86a9:1:21 -[1a-relative] ERROR: invalid URL: relative URL with a cannot-be-a-base base -[1a-absolute-blob] SUCCESS message: { dep: "dep-loaded" } -``` - -- A relative `import "./dep.js"` from a blob module **fails** — a `blob:` URL is - a cannot-be-a-base URL, so there is no base to resolve against. -- Rewriting the specifier to the dependency's **absolute `blob:` URL before - minting the entry blob** loads the 2-module graph and the worker runs. ✅ - -This is the mechanism used. It requires the loader to resolve and topologically -order the module map itself (`buildGraph` / `topoSort` in `loader.mjs`) — which -is exactly what Cloudflare's Worker Loader does internally for its `modules` -map, so the responsibility is not new, only relocated into our code. - -### 1b. data: URLs — `exp1b-data.ts` - -``` -[1b-relative] ERROR: invalid URL: relative URL with a cannot-be-a-base base -[1b-absolute-data] SUCCESS: { dep: "dep-from-data" } -``` - -Identical behaviour to blob:. Works, but rejected in favour of blob: — -base64-in-a-URL bloats every module and the specifier is copied into every -importer, whereas a blob URL is a short opaque handle that can be -`URL.revokeObjectURL`'d for deterministic cleanup. - -### 1c. Import maps — `exp1c-importmap.ts` + `importmap.json` - -``` -$ deno run -A --import-map=importmap.json exp1c-importmap.ts -[1c] SUCCESS: { dep: "dep-via-importmap" } -``` - -A **static** import map does let a blob-loaded worker resolve a *bare* -specifier to a `data:` URL — and it is inherited by the Worker. But there is -**no per-Worker `importMap` option** in Deno's `WorkerOptions`, and the process -import map is fixed at startup. So an import map cannot carry a *per-run*, -runtime-generated module map. Useful only if the set of bare specifiers is known -at build time. Not usable as the general mechanism. - -### 1d. Temp-dir materialization — **not needed, not used** - -Skipped deliberately: 1a succeeds, so the coherence/cleanup/security costs the -issue wanted measured (user code landing on the host filesystem outside DOFS, -crash-leaked temp files, two sources of truth) are **avoided entirely**. Blob -URLs live in the host isolate's object-URL table and are revoked on run -completion (`loader.mjs`, `finish()`). Nothing user-supplied touches disk. - ---- - -## 2. Isolation — measured - -Deno's `deno: { permissions: ... }` on a Worker requires -`--unstable-worker-options`; without it: - -``` -error: Unstable API 'Worker.deno.permissions'. The `--unstable-worker-options` flag must be provided. -``` - -### Under `deno run` — the ops layer holds, the module loader does not - -`exp2-isolation.ts`, `exp2b-import-escape.ts`, `exp2d-trusted-host.ts`, -`exp2e-cache-bypass.ts`, `exp5-node-builtins.ts`, with `permissions: "none"`: - -| Probe | Result | -|---|---| -| `Deno.env.get` | DENIED — `NotCapable: Requires env access` | -| `Deno.readTextFile("/etc/hosts")` | DENIED — `Requires read access` | -| `Deno.writeTextFile` | DENIED — `Requires write access` | -| `fetch()` | DENIED — `Requires net access` | -| `new Deno.Command("id")` | DENIED — `Requires run access` | -| `Deno.dlopen` | DENIED — `Requires ffi access` | -| `import("file:///…host-secret.mjs")` | DENIED — `Requires read access` | -| `import("http://127.0.0.1:…")` | DENIED — `Requires import access` | -| `import("https://esm.sh/…")` | DENIED — `Requires import access` | -| **`import("jsr:@std/uuid@1.0.4")`** | **ALLOWED — network fetch performed** | -| **`import("npm:is-odd@3.0.1")`** | **ALLOWED — network fetch performed** | -| **`import("https://…")` already in DENO_DIR** | **ALLOWED (`exp2e`, cold *and* warm)** | -| `import("node:fs")` then `readFileSync` | module ALLOWED, the *call* DENIED | -| `node:sqlite` open the DOFS db file directly | DENIED — `Requires read access` | -| `node:process.env.HOME` | DENIED — `Requires env access` | -| `node:os.userInfo()` | DENIED — `Requires sys access` | -| `node:net.connect` | DENIED — `getaddrinfo EPERM` | -| `node:vm.runInNewContext` | ALLOWED (no capability — harmless) | -| `node:worker_threads` | ALLOWED (no capability — harmless) | -| spawning a nested `Worker` | ALLOWED | - -**Finding: `permissions` in `WorkerOptions` governs Deno *ops*, not the *module -loader*.** `jsr:` and `npm:` specifiers resolve and fetch from inside a -`permissions: "none"` worker — that is arbitrary third-party code execution and -unmetered network egress that `net: false` does not stop. - -**`import: false` in the worker permissions object is inert** — `exp2d` ran the -same probe with `{read:false,write:false,net:false,env:false,run:false,ffi:false,import:false}` -and got byte-identical results to `"none"`. - -*Measurement note:* an earlier pass reported these imports as DENIED. That was a -false negative — the probe did `String(moduleNamespace)`, which throws -`TypeError: Cannot convert object to primitive value` on a null-prototype module -namespace. `exp2b` re-probes correctly. (Per house guidance: make every sweep -fail once before believing it — this one silently inverted.) - -### Under `deno compile` — the hole closes - -`exp4-compiled-isolation.ts`, compiled to `isobin`: - -``` -{ - "jsr": "DENIED: TypeError: Module not found: jsr:@std/uuid@1.0.4", - "npm": "DENIED: TypeError: Could not find a matching package for 'npm:is-odd@3.0.1' in the node_modules directory…", - "https": "DENIED: TypeError: Module not found: https://deno.land/std@0.224.0/version.ts", - "nodeFs": "ALLOWED: function", - "env": "DENIED: NotCapable: Requires env access to \"HOME\"…", - "read": "DENIED: NotCapable: Requires read access to \"/etc/hosts\"…", - "net": "DENIED: NotCapable: Requires net access to \"example.com:443\"…" -} -``` - -`deno compile` freezes the module graph, so the loader has nowhere to fetch -from. The only reachable surface left is `node:` builtins, and every one of -those that carries a capability is ops-gated (table above). - -**Conclusion: isolation is adequate *only in the compiled binary*.** Since the -shipping target is a compiled Deno host, this is acceptable — but it must be -stated as a hard constraint, because a dev-mode `deno run` host is *not* a -sandbox. Compared with Cloudflare, whose isolate has no module loader at all -after `load()`, Deno's guarantee is weaker in kind (it depends on graph freezing -rather than on the isolate having no I/O) even where it is equivalent in effect. - ---- - -## 3. Capability bridge + full-contract proof - -`loader.mjs` implements `createLoader({fsHandlers}).load({mainModule, modules}) -.getEntrypoint().run({input, timeoutMs, signal})` — the same shape as -`WorkspaceRuntimeLoader`. Inside the worker the generated runner mirrors -`runtimeWorkerModule()`: fake `globalThis.process`, `console.log/info` → stdout -frame, `console.warn/error` → stderr frame, `globalThis.workspace` = the fs -capability, `user.default(input)` → `result` frame → `exit` frame. - -The bridge is a postMessage request/response keyed by a sequence number -(`{kind:"fs-call", id, op, args}` → `{kind:"fs-reply", id, value|error}`). It -exists because the host owns the DOFS `DatabaseSync` handle, which is native and -**cannot be structured-cloned** into a worker. - -Host wiring is the #349 setup verbatim: -`FileSQLiteStorage(dbPath)` → `new Database(storage)` → `initializeSchema(db, Date.now)` -→ `new WorkspaceFilesystem(db)` (`file-storage.mjs`, `proof.mjs`). - -`proof.mjs` output (`proof-output.txt`; the compiled `proofbin` produces the -same, `compiled-output.txt`): - -| # | Contract point | Result | -|---|---|---| -| i | entry imports a second in-memory module | `"entry sees dependency: dep-module-executed"` | -| ii | DOFS write + read-back through the bridge | `read back from DOFS: written-through-the-bridge:deno`; `readdir /work: ["note.txt"]` | -| iii | stdout **and** stderr | both captured, separately framed | -| iv | returned value → result frame | `{ok:true, roundTripped:…, dep:…}`, `exitCode: 0` | -| v | thrown error | `outcome:"exit"`, `exitCode: 1`, `stderr: "Error: user blew up"` | -| vi | `while(true)` vs 1500 ms timeout | `outcome:"timeout"` at 1507 ms; pre-loop stdout retained | -| vii | cancel mid-flight | `outcome:"cancelled"`; pre-cancel DOFS write survives — `/cancel: ["before.txt"]`, post-cancel write absent | - -**Restart coherence** (`verify-restart.mjs`, separate process, fresh `Database` -over the same file): - -``` -/work: [ "note.txt" ] -/work/note.txt: written-through-the-bridge:deno -/cancel: [ "before.txt" ] -``` - -Writes made through the postMessage bridge are genuinely committed, and a -cancelled run leaves the workspace coherent because the host — not the worker — -owns the database. - ---- - -## 4. Latency, streaming, size - -- **Cold worker spawn → exit:** 13 ms (`deno run`), 15 ms (compiled). -- **fs-bridge round trip:** **0.065 ms/call** (`deno run`), 0.069 ms (compiled), - over 200 sequential `readFile`s. Negligible. -- **Streaming is live, not end-of-run.** Frame arrival offsets from run start - for a module logging every 400 ms: `stdout@6, stdout@410, stdout@812, - result@1214, exit@1214`. postMessage frames arrive as produced, so - Cloudflare's framed-stream behaviour is preserved. -- **Binary size:** baseline `deno compile` hello-world **68.4 MB**; the loader + - vendored DOFS + its node_modules **107.7 MB** — a **+39 MB** delta, essentially - all of it DOFS's `node_modules` (38.03 MB embedded). Compiling with - `--include ./dofs/dist` was required; without the graph the binary cannot - resolve the package. - -## 5. The timeout leak — the smallest failing proof - -`exp3-terminate-spin.ts`: - -``` -worker confirmed spinning; calling terminate() -terminate() returned after 0 ms -host event loop still responsive; elapsed 504 ms -- host exiting -EXITCODE=124 <-- `timeout 30` had to kill it -``` - -`exp3b-spin-leak.ts` confirms the host reaches the end of its main module and -**never exits**; only an explicit `Deno.exit(0)` frees it. `proof.mjs` carries -that `Deno.exit(0)` with a comment for exactly this reason. - -So: `terminate()` returns immediately, the host stays responsive and reports the -correct `timeout` outcome, but the worker's OS thread is still executing user JS -— burning a core and pinning the process. Under Cloudflare, `limits.cpuMs` -terminates the isolate and reclaims it. - -**Missing primitive (Deno-side): a CPU/wall-time budget on `new Worker(...)`, or -a `Worker.terminate()` that interrupts running JavaScript rather than only -detaching the handle.** There is no workaround inside the process; the only -faithful mitigation is running the worker in a separate OS process that can be -SIGKILLed, which reintroduces process-spawn cost per run and defeats the -13 ms cold-start number above. - ---- - -## 6. Comparison with Cloudflare's `worker-javascript.ts` - -| Aspect | Cloudflare Worker Loader | Deno port | Faithful? | -|---|---|---|---| -| Module access | native in-memory `modules` map, loader resolves | in-memory map → host-side specifier rewrite → blob: URLs | ✅ same idea, no disk | -| Relative specifiers | resolved by the loader | must be pre-resolved by us (blob = cannot-be-a-base) | ✅ equivalent, more code | -| Capability transfer | RPC stub on `host` object | postMessage request/response | ✅ 0.065 ms/call | -| Output | framed stream (`stdout`/`stderr`/`result`/`exit`) | same four frames over postMessage | ✅ live-streamed | -| Entry convention | `module.default` fn or value, awaited | identical | ✅ | -| Error | captured, exit 1, message on stderr | identical | ✅ | -| Cancellation | dispose/terminate + reject a race | `worker.terminate()` + reject a race | ✅ for I/O-bound; ❌ for CPU-bound (§5) | -| Timeout | host `setTimeout` race **+ `limits.cpuMs`** | host `setTimeout` race only | ⚠️ correct outcome, leaked thread | -| Isolation | isolate with no ambient I/O and no loader | ops permissions + frozen graph (compiled only) | ⚠️ compiled-only | -| `compatibilityDate` / `Flags` | first-class | **no equivalent** | ❌ not expressible | -| Retained execution / reattach (`getExec`/`resume`) | supported | worker lifetime is bounded by the host call | ❌ not expressible | -| Cold start | isolate spawn (sub-ms class) | 13–15 ms | ⚠️ ~10× slower, still fine | - ---- - -## What a committed spike must replicate - -1. `loader.mjs`'s `buildGraph` — topological sort of the module map plus - specifier rewriting to absolute `blob:` URLs, and `revokeObjectURL` on run - completion. This is the load-bearing mechanism. -2. The postMessage fs bridge and the host-side DOFS handler table, including the - cancellation-coherence assertion (pre-cancel write survives, post-cancel - write absent, verified from a **separate process**). -3. `--unstable-worker-options` must be on both the run and the compile command. -4. A test that the shipped binary denies `jsr:`/`npm:`/`https:` imports from the - worker — this is the isolation guarantee and it exists **only** after - compilation. A `deno run` dev host must not be presented as a sandbox. -5. An explicit acknowledgement of the CPU-spin thread leak: either a documented - limitation, or a `Deno.exit` on host shutdown, or the out-of-process design. - -## Commands run - -``` -deno run -A exp1a-blob.ts -deno run -A exp1b-data.ts -deno run -A --import-map=importmap.json exp1c-importmap.ts -deno run -A --unstable-worker-options exp2-isolation.ts -deno run -A --unstable-worker-options exp2b-import-escape.ts -deno run -A --unstable-worker-options exp2c-exfil.ts -deno run -A --unstable-worker-options exp2d-trusted-host.ts -deno run -A --unstable-worker-options exp2e-cache-bypass.ts -deno run -A --unstable-worker-options exp3-terminate-spin.ts -deno run -A --unstable-worker-options exp3b-spin-leak.ts -deno run -A --unstable-worker-options exp5-node-builtins.ts -deno run -A smoke-dofs.mjs -deno run -A --unstable-worker-options smoke-loader.mjs -deno run -A --unstable-worker-options proof.mjs ./proof.db -deno run -A verify-restart.mjs ./proof.db -deno compile -A --unstable-worker-options --include ./dofs/dist --output ./proofbin proof.mjs && ./proofbin ./compiled.db -deno compile -A --unstable-worker-options --output ./isobin exp4-compiled-isolation.ts && ./isobin -``` - -DOFS build: `cp -R spikes/349-dofs/vendor/dofs ./dofs && (cd dofs && npm install && npx tsc -p tsconfig.build.json)`. -`deno.json` maps `@cloudflare/dofs` → `./dofs/dist/index.js`. diff --git a/spikes/351-worker-backends/host/dofs-types.ts b/spikes/351-worker-backends/host/dofs-types.ts deleted file mode 100644 index 0b7c8ebd..00000000 --- a/spikes/351-worker-backends/host/dofs-types.ts +++ /dev/null @@ -1,20 +0,0 @@ -// Type-only resolution for the vendored Cloudflare adapter. -// -// vendor/worker-shell/adapter.ts must stay byte-identical to upstream, and it -// imports its option types from "@cloudflare/dofs". Deno does not pair a -// file:-resolved npm package with its declarations, so deno.json scopes that -// specifier — inside vendor/ only — to this module. Every import the adapter -// makes from it is `import type`, erased before runtime. - -export type { - GrepOptions, - MkdirOptions, - ReadFileOptions, - RmOptions, - WorkspaceDirentResult, - WorkspaceFoundEntry, - WorkspaceGrepMatch, - WorkspaceStatResult, - WriteFileContent, - WriteFileOptions, -} from "./types/dofs.d.ts"; diff --git a/spikes/351-worker-backends/host/loader.ts b/spikes/351-worker-backends/host/loader.ts deleted file mode 100644 index 28d85130..00000000 --- a/spikes/351-worker-backends/host/loader.ts +++ /dev/null @@ -1,321 +0,0 @@ -// A Deno port of Cloudflare's WorkspaceRuntimeLoader. -// -// Upstream's contract (packages/computer/src/runtime/types.ts:70) is -// `load({ mainModule, modules, ... }) => { getEntrypoint() }`, where `modules` -// is an in-memory specifier→source map. Cloudflare implements it with the -// Workers Dynamic Worker Loader; this port implements it with a Deno Worker. -// -// Two mechanics carry the port: -// -// 1. The module graph is supplied as blob: URLs, with each module's import -// specifiers rewritten to the absolute blob: URL of the module they name. -// A relative specifier does not resolve from a blob: base — Deno reports -// "invalid URL: relative URL with a cannot-be-a-base base" — so the -// loader resolves the graph itself, in topological order, before minting -// each blob. Nothing is materialized to disk. -// -// 2. The filesystem capability crosses as an async postMessage RPC. The DOFS -// handle is a node:sqlite DatabaseSync that cannot be structured-cloned, -// so the host keeps it and services calls; this mirrors upstream, where -// the same capability crosses as a Workers RPC stub. -// -// Isolation is the Deno Worker's `permissions: "none"`, which governs Deno ops -// but NOT the module loader: under `deno run` a locked worker can still import -// jsr:/npm: specifiers. A compiled artifact's frozen module graph closes that -// (see evidence/EVIDENCE.md). Ship compiled; never present `deno run` as a -// sandbox. - -export type ModuleSource = string | { js?: string }; - -export interface LoaderCode { - mainModule: string; - modules: Record; -} - -export interface RunOptions { - input?: unknown; - timeoutMs?: number; - signal?: AbortSignal; -} - -export type RunOutcome = "exit" | "timeout" | "cancelled" | "worker-error"; - -export interface RunResult { - outcome: RunOutcome; - stdout: string; - stderr: string; - result: unknown; - exitCode: number | undefined; -} - -// Handlers receive the argument list the worker sent; each host installs the -// operations it is willing to expose, which is the capability surface. -export type FsHandlers = Record< - string, - // deno-lint-ignore no-explicit-any - (...args: any[]) => Promise | unknown ->; - -export interface Loader { - load(code: LoaderCode): { - getEntrypoint(): { run(options?: RunOptions): Promise }; - }; -} - -const DEFAULT_TIMEOUT_MS = 5_000; - -export function createLoader(options: { fsHandlers: FsHandlers }): Loader { - return { - load(code: LoaderCode) { - return { - getEntrypoint() { - return { - run(runOptions: RunOptions = {}) { - return runOnce(code, runOptions, options.fsHandlers); - }, - }; - }, - }; - }, - }; -} - -function sourceOf(module: ModuleSource): string { - return typeof module === "string" ? module : module.js ?? ""; -} - -function resolveSpecifier( - specifier: string, - modules: Record, -): string | undefined { - const candidates = [ - specifier, - specifier.replace(/^\.\//, ""), - `${specifier}.js`, - `${specifier.replace(/^\.\//, "")}.js`, - ]; - return candidates.find((candidate) => candidate in modules); -} - -const SPECIFIER_PATTERN = /(?:from|import\s*\(?)\s*["']([^"']+)["']/g; -const REWRITE_PATTERN = /(\bfrom\s*|\bimport\s*\(?\s*)(["'])([^"']+)\2/g; - -function topologicalOrder( - modules: Record, - entry: string, -): string[] { - const seen = new Set(); - const order: string[] = []; - const visit = (name: string) => { - if (seen.has(name)) { - return; - } - seen.add(name); - for (const match of sourceOf(modules[name]).matchAll(SPECIFIER_PATTERN)) { - const target = resolveSpecifier(match[1], modules); - if (target !== undefined && target !== name) { - visit(target); - } - } - order.push(name); - }; - visit(entry); - return order; -} - -function buildGraph( - modules: Record, - mainModule: string, -): { entryUrl: string; urls: string[] } { - const urls = new Map(); - const created: string[] = []; - for (const name of topologicalOrder(modules, mainModule)) { - const rewritten = sourceOf(modules[name]).replace( - REWRITE_PATTERN, - (whole, lead: string, quote: string, specifier: string) => { - const target = resolveSpecifier(specifier, modules); - if (target === undefined) { - return whole; - } - return `${lead}${quote}${urls.get(target)}${quote}`; - }, - ); - const url = URL.createObjectURL( - new Blob([rewritten], { type: "application/javascript" }), - ); - urls.set(name, url); - created.push(url); - } - const entryUrl = urls.get(mainModule); - if (entryUrl === undefined) { - throw new Error(`main module ${mainModule} is not in the module map`); - } - return { entryUrl, urls: created }; -} - -// The runner mirrors upstream's runtimeWorkerModule(): it installs the -// capability bridge and a process/console shim, imports the user entry, calls -// its default export, and frames stdout/stderr/result/exit. -function runnerSource(entryUrl: string, input: unknown): string { - return ` -import * as user from ${JSON.stringify(entryUrl)}; - -let seq = 0; -const pending = new Map(); -self.addEventListener("message", (event) => { - const message = event.data; - if (message && message.kind === "fs-reply") { - const slot = pending.get(message.id); - if (slot === undefined) return; - pending.delete(message.id); - if (message.error) slot.reject(new Error(message.error)); - else slot.resolve(message.value); - } -}); -function call(op, args) { - const id = ++seq; - return new Promise((resolve, reject) => { - pending.set(id, { resolve, reject }); - self.postMessage({ kind: "fs-call", id, op, args }); - }); -} -globalThis.workspace = { - readFile: (path, encoding) => call("readFile", [path, encoding ?? "utf8"]), - writeFile: (path, body) => call("writeFile", [path, body]), - mkdir: (path, options) => call("mkdir", [path, options ?? { recursive: true }]), - readdir: (path) => call("readdir", [path]), - stat: (path) => call("stat", [path]), - rm: (path, options) => call("rm", [path, options ?? {}]), - symlink: (target, path) => call("symlink", [target, path]), -}; - -function frame(name, text) { - self.postMessage({ kind: "frame", name, text: String(text) }); -} -const format = (args) => - args - .map((value) => { - if (typeof value === "string") return value; - try { - return JSON.stringify(value); - } catch { - return String(value); - } - }) - .join(" ") + "\\n"; -console.log = console.info = (...args) => frame("stdout", format(args)); -console.warn = console.error = (...args) => frame("stderr", format(args)); -globalThis.process = { - env: {}, - argv: ["workspace", "entry"], - cwd: () => "/workspace", - stdout: { write: (chunk) => frame("stdout", chunk) }, - stderr: { write: (chunk) => frame("stderr", chunk) }, -}; - -const input = ${JSON.stringify(input ?? null)}; -try { - const value = typeof user.default === "function" - ? await user.default(input) - : user.default ?? null; - self.postMessage({ kind: "frame", name: "result", value: value ?? null }); - self.postMessage({ kind: "frame", name: "exit", code: 0 }); -} catch (error) { - frame("stderr", String(error && error.stack ? error.stack : error) + "\\n"); - self.postMessage({ kind: "frame", name: "exit", code: 1 }); -} -`; -} - -function runOnce( - code: LoaderCode, - options: RunOptions, - fsHandlers: FsHandlers, -): Promise { - const graph = buildGraph(code.modules, code.mainModule); - const runnerUrl = URL.createObjectURL( - new Blob([runnerSource(graph.entryUrl, options.input)], { - type: "application/javascript", - }), - ); - - const stdout: string[] = []; - const stderr: string[] = []; - let result: unknown; - let exitCode: number | undefined; - - const worker = new Worker(runnerUrl, { - type: "module", - deno: { permissions: "none" }, - }); - - return new Promise((resolve) => { - let settled = false; - const finish = (outcome: RunOutcome) => { - if (settled) { - return; - } - settled = true; - clearTimeout(timer); - // terminate() reclaims the worker unless user code is spinning on the - // CPU, which Deno cannot preempt — see EVIDENCE.md gap 1. - worker.terminate(); - for (const url of [...graph.urls, runnerUrl]) { - URL.revokeObjectURL(url); - } - resolve({ - outcome, - stdout: stdout.join(""), - stderr: stderr.join(""), - result, - exitCode, - }); - }; - - const timer = setTimeout( - () => finish("timeout"), - options.timeoutMs ?? DEFAULT_TIMEOUT_MS, - ); - options.signal?.addEventListener("abort", () => finish("cancelled")); - - worker.onmessage = async (event: MessageEvent) => { - const message = event.data; - if (message.kind === "frame") { - if (message.name === "stdout") { - stdout.push(message.text); - } else if (message.name === "stderr") { - stderr.push(message.text); - } else if (message.name === "result") { - result = message.value; - } else if (message.name === "exit") { - exitCode = message.code; - finish("exit"); - } - return; - } - if (message.kind === "fs-call") { - const handler = fsHandlers[message.op]; - try { - if (handler === undefined) { - throw new Error(`unsupported filesystem op: ${message.op}`); - } - const value = await handler(...message.args); - worker.postMessage({ kind: "fs-reply", id: message.id, value }); - } catch (error) { - worker.postMessage({ - kind: "fs-reply", - id: message.id, - error: error instanceof Error ? error.message : String(error), - }); - } - } - }; - - worker.onerror = (event: ErrorEvent) => { - event.preventDefault(); - stderr.push(`${event.message}\n`); - exitCode = 1; - finish("worker-error"); - }; - }); -} diff --git a/spikes/351-worker-backends/host/main.ts b/spikes/351-worker-backends/host/main.ts deleted file mode 100644 index e9cd5197..00000000 --- a/spikes/351-worker-backends/host/main.ts +++ /dev/null @@ -1,120 +0,0 @@ -import { main, until } from "effection"; -import { openWorkspace } from "./workspace.ts"; -import { execShell } from "./shell.ts"; -import { createLoader, type ModuleSource } from "./loader.ts"; - -function usage(): never { - console.error( - [ - "usage: proof [args...]", - "", - "workspace ops (host capability, the declarative core's substrate):", - " fs-write | fs-read | fs-mkdir | fs-ls ", - "", - "imperative backends:", - " shell [--cwd ] [--env K=V] [--timeout ]", - " js [--dep =] [--timeout ]", - ].join("\n"), - ); - Deno.exit(2); -} - -function parseFlag(args: string[], flag: string): string | undefined { - const index = args.indexOf(flag); - if (index === -1) { - return undefined; - } - return args[index + 1]; -} - -main(function* () { - const started = performance.now(); - const [dbPath, command, ...rest] = Deno.args; - if (dbPath === undefined || command === undefined) { - usage(); - } - - const workspace = openWorkspace(dbPath); - let payload: Record; - - switch (command) { - case "fs-write": - yield* until(workspace.fs.writeFile(rest[0], rest[1])); - payload = { ok: true }; - break; - case "fs-read": - payload = { body: yield* until(workspace.fs.readFile(rest[0], "utf8")) }; - break; - case "fs-mkdir": - yield* until(workspace.fs.mkdir(rest[0], { recursive: true })); - payload = { ok: true }; - break; - case "fs-ls": { - const entries = yield* until(workspace.fs.readdir(rest[0])); - payload = { entries: entries.map((entry) => entry.name).sort() }; - break; - } - case "shell": { - const script = rest[0]; - if (script === undefined) { - usage(); - } - const envPairs = rest - .filter((_, index) => rest[index - 1] === "--env") - .map((pair) => pair.split("=")); - const timeout = parseFlag(rest, "--timeout"); - const result = yield* execShell(workspace.fs, script, { - cwd: parseFlag(rest, "--cwd"), - env: Object.fromEntries(envPairs), - timeoutMs: timeout === undefined ? undefined : Number(timeout), - }); - payload = { ...result }; - break; - } - case "js": { - const source = rest[0]; - if (source === undefined) { - usage(); - } - const modules: Record = { "entry.js": source }; - for (let index = 0; index < rest.length; index += 1) { - if (rest[index] !== "--dep") { - continue; - } - const spec = rest[index + 1] ?? ""; - const separator = spec.indexOf("="); - modules[spec.slice(0, separator)] = spec.slice(separator + 1); - } - const timeout = parseFlag(rest, "--timeout"); - const loader = createLoader({ - fsHandlers: { - readFile: (path: string, encoding: "utf8") => - workspace.fs.readFile(path, encoding), - writeFile: (path: string, body: string) => - workspace.fs.writeFile(path, body), - mkdir: (path: string, options: { recursive?: boolean }) => - workspace.fs.mkdir(path, options), - readdir: async (path: string) => - (await workspace.fs.readdir(path)).map((entry) => entry.name), - stat: (path: string) => workspace.fs.stat(path), - rm: (path: string, options: { recursive?: boolean }) => - workspace.fs.rm(path, options), - }, - }); - const run = loader.load({ mainModule: "entry.js", modules }) - .getEntrypoint(); - payload = { - ...(yield* until( - run.run({ timeoutMs: timeout === undefined ? 5000 : Number(timeout) }), - )), - }; - break; - } - default: - usage(); - } - - workspace.storage.close(); - const ms = Math.round((performance.now() - started) * 100) / 100; - console.log(JSON.stringify({ command, ...payload, ms })); -}); diff --git a/spikes/351-worker-backends/host/shell.ts b/spikes/351-worker-backends/host/shell.ts deleted file mode 100644 index 1dac20d0..00000000 --- a/spikes/351-worker-backends/host/shell.ts +++ /dev/null @@ -1,83 +0,0 @@ -// Worker Shell over the in-process DOFS workspace. -// -// This is upstream's execution model with the Cloudflare transport removed. -// Upstream runs the same interpreter inside a Dynamic Worker that reaches the -// host Durable Object over Workers RPC for every filesystem call -// (entrypoint.ts). Locally the filesystem is in-process, so `WorkerEntrypoint`, -// `env.HOST.getWorkspace()`, and the loader glue all disappear; what remains is -// the vendored adapter plus just-bash. -// -// The exit-code contract is upstream's: timeout 124, cancellation 130, -// interpreter failure 1. - -import { type Operation, until } from "effection"; -import { Bash } from "just-bash"; -import { WorkspaceFsAdapter } from "../vendor/worker-shell/adapter.ts"; -import type { WorkspaceFsShim } from "./workspace-fs.ts"; - -const MAX_OUTPUT_BYTES = 1024 * 1024; - -export interface ShellResult { - stdout: string; - stderr: string; - exitCode: number; -} - -export interface ShellOptions { - cwd?: string; - env?: Record; - signal?: AbortSignal; - timeoutMs?: number; -} - -export function* execShell( - fs: WorkspaceFsShim, - command: string, - options: ShellOptions = {}, -): Operation { - const cwd = options.cwd ?? "/workspace"; - const controller = new AbortController(); - options.signal?.addEventListener("abort", () => { - controller.abort(new Error("Execution cancelled")); - }); - - let timedOut = false; - const timer = options.timeoutMs === undefined ? undefined : setTimeout(() => { - timedOut = true; - controller.abort(new Error("Execution timed out")); - }, options.timeoutMs); - - const bash = new Bash({ - fs: new WorkspaceFsAdapter(fs) as unknown as NonNullable< - ConstructorParameters[0] - >["fs"], - cwd, - // just-bash's own DefenseInDepthBox registers ESM loader hooks through - // node:module. Upstream disables it because workerd throws on that call and - // the Dynamic Worker is the real boundary; the spike's evidence records - // what it does under Deno, where there is no Dynamic Worker. - defenseInDepth: { enabled: false }, - executionLimits: { maxOutputSize: MAX_OUTPUT_BYTES }, - }); - - try { - return yield* until( - bash.exec(command, { - cwd, - env: options.env, - signal: controller.signal, - }), - ); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - return { - stdout: "", - stderr: `${timedOut ? "Execution timed out" : message}\n`, - exitCode: timedOut ? 124 : controller.signal.aborted ? 130 : 1, - }; - } finally { - if (timer !== undefined) { - clearTimeout(timer); - } - } -} diff --git a/spikes/351-worker-backends/host/types/dofs.d.ts b/spikes/351-worker-backends/host/types/dofs.d.ts deleted file mode 100644 index 9d185e31..00000000 --- a/spikes/351-worker-backends/host/types/dofs.d.ts +++ /dev/null @@ -1,108 +0,0 @@ -// Typed facade for the surface this spike consumes from the vendored -// @cloudflare/dofs (shared with the #349 spike, which vendors and builds it). -// Deno pairs .js imports with sibling .d.ts files for registry npm packages -// but not for file:-resolved ones, so the package's own declarations cannot -// be used directly. This mirrors vendor/dofs/dist/{types,storage,schema/index, -// fs/filesystem}.d.ts for exactly the members the host uses. - -export interface SQLCursorLike> { - toArray(): Row[]; -} - -export interface SQLStorageLike { - exec>( - query: string, - ...bindings: unknown[] - ): SQLCursorLike; -} - -export interface DurableObjectStorageLike { - sql: SQLStorageLike; - transaction?(closure: () => T | Promise): T | Promise; - transactionSync?(closure: () => T): T; -} - -export declare class Database { - constructor(storage: DurableObjectStorageLike); - all(query: string, ...bindings: unknown[]): Row[]; -} - -export declare function initializeSchema( - db: Database, - now: () => number, -): void; - -export declare const SCHEMA_VERSION: number; - -export interface WorkspaceStatResult { - size: number; - mode: number; - mtime: number; - isFile: boolean; - isDirectory: boolean; - isSymbolicLink: boolean; -} - -export interface WorkspaceDirentResult { - name: string; - isFile: boolean; - isDirectory: boolean; - isSymbolicLink?: boolean; -} - -export interface WorkspaceFoundEntry { - path: string; - type: "file" | "dir"; -} - -export interface WorkspaceGrepMatch { - path: string; - line: number; - text: string; -} - -export interface GrepOptions { - ignoreCase?: boolean; -} - -// Option types the vendored worker-shell adapter imports by name. -export interface ReadFileOptions { - encoding?: "utf8"; -} - -export interface WriteFileOptions { - exclusive?: boolean; -} - -export type WriteFileContent = string | Uint8Array; - -export interface MkdirOptions { - recursive?: boolean; -} - -export interface RmOptions { - recursive?: boolean; - force?: boolean; -} - -export declare class WorkspaceFilesystem { - constructor(db: Database, options?: { now?: () => number }); - readFile(path: string): Promise>; - readFile(path: string, encoding: "utf8"): Promise; - stat(path: string): Promise; - lstat(path: string): Promise; - readlink(path: string): Promise; - readdir(path: string): Promise; - find(directory: string, pattern?: string): Promise; - ls(prefix: string): Promise; - grep( - pattern: string, - path: string, - options?: GrepOptions, - ): Promise; - writeFile(path: string, content: string | Uint8Array): Promise; - mkdir(path: string, options?: { recursive?: boolean }): Promise; - rm(path: string, options?: { recursive?: boolean }): Promise; - chmod(path: string, mode: number): Promise; - symlink(target: string, path: string): Promise; -} diff --git a/spikes/351-worker-backends/host/workspace-fs.ts b/spikes/351-worker-backends/host/workspace-fs.ts deleted file mode 100644 index 426a0921..00000000 --- a/spikes/351-worker-backends/host/workspace-fs.ts +++ /dev/null @@ -1,111 +0,0 @@ -// WorkspaceFs over the in-process DOFS filesystem. -// -// Cloudflare's vendored adapter (vendor/worker-shell/adapter.ts) consumes a -// structural `WorkspaceFs` — the subset of their WorkspaceFilesystemStub the -// shell needs. Three of its methods (exists, statOrNull, lstatOrNull) live on -// the RPC stub rather than the base filesystem, because on Cloudflare a miss -// thrown across Workers RPC surfaces as an uncaught exception even when -// just-bash is deliberately probing. In-process there is no RPC, so they are -// a try/catch over the DOFS filesystem. - -// @ts-types="./types/dofs.d.ts" -import { Database, WorkspaceFilesystem } from "@cloudflare/dofs"; -import type { - GrepOptions, - WorkspaceDirentResult, - WorkspaceFoundEntry, - WorkspaceGrepMatch, - WorkspaceStatResult, -} from "./types/dofs.d.ts"; - -export class WorkspaceFsShim { - #fs: WorkspaceFilesystem; - - constructor(db: Database) { - this.#fs = new WorkspaceFilesystem(db); - } - - readFile(path: string): Promise>; - readFile(path: string, encoding: "utf8"): Promise; - readFile( - path: string, - encoding?: "utf8", - ): Promise> { - if (encoding === "utf8") { - return this.#fs.readFile(path, "utf8"); - } - return this.#fs.readFile(path); - } - - async exists(path: string): Promise { - return (await this.statOrNull(path)) !== null; - } - - stat(path: string): Promise { - return this.#fs.stat(path); - } - - async statOrNull(path: string): Promise { - try { - return await this.#fs.stat(path); - } catch { - return null; - } - } - - lstat(path: string): Promise { - return this.#fs.lstat(path); - } - - async lstatOrNull(path: string): Promise { - try { - return await this.#fs.lstat(path); - } catch { - return null; - } - } - - readdir(path: string): Promise { - return this.#fs.readdir(path); - } - - find(directory: string, pattern?: string): Promise { - return this.#fs.find(directory, pattern); - } - - ls(prefix: string): Promise { - return this.#fs.ls(prefix); - } - - grep( - pattern: string, - path: string, - options?: GrepOptions, - ): Promise { - return this.#fs.grep(pattern, path, options); - } - - readlink(path: string): Promise { - return this.#fs.readlink(path); - } - - writeFile(path: string, content: string | Uint8Array): Promise { - return this.#fs.writeFile(path, content); - } - - mkdir(path: string, options?: { recursive?: boolean }): Promise { - return this.#fs.mkdir(path, options); - } - - rm(path: string, options?: { recursive?: boolean }): Promise { - return this.#fs.rm(path, options); - } - - chmod(path: string, mode: number): Promise { - return this.#fs.chmod(path, mode); - } - - symlink(target: string, path: string): Promise { - return this.#fs.symlink(target, path); - } -} diff --git a/spikes/351-worker-backends/host/workspace.ts b/spikes/351-worker-backends/host/workspace.ts deleted file mode 100644 index 49c6da53..00000000 --- a/spikes/351-worker-backends/host/workspace.ts +++ /dev/null @@ -1,107 +0,0 @@ -// Opens the DOFS workspace this spike executes against. The database file is -// the workspace identity and the only filesystem source of truth; #349 proves -// its persistence and isolation properties. - -import { DatabaseSync, type StatementSync } from "node:sqlite"; -// @ts-types="./types/dofs.d.ts" -import { Database, initializeSchema } from "@cloudflare/dofs"; -import type { - DurableObjectStorageLike, - SQLCursorLike, -} from "./types/dofs.d.ts"; -import { WorkspaceFsShim } from "./workspace-fs.ts"; - -class Cursor implements SQLCursorLike { - #rows: Row[]; - constructor(rows: Row[]) { - this.#rows = rows; - } - toArray(): Row[] { - return this.#rows; - } -} - -export class FileSQLiteStorage implements DurableObjectStorageLike { - #db: DatabaseSync; - #cache = new Map(); - readonly sql: { - exec: ( - query: string, - ...bindings: unknown[] - ) => SQLCursorLike; - }; - - constructor(path: string) { - this.#db = new DatabaseSync(path); - this.#db.exec("PRAGMA journal_mode = WAL"); - this.#db.exec("PRAGMA foreign_keys = ON"); - this.#db.exec("PRAGMA synchronous = NORMAL"); - this.sql = { - exec: (query: string, ...bindings: unknown[]) => { - let statement = this.#cache.get(query); - if (statement === undefined) { - statement = this.#db.prepare(query); - this.#cache.set(query, statement); - } - const rows: Row[] = []; - for (const row of statement.all(...bindings.map(toSQLiteValue))) { - if (typeof row === "object" && row !== null) { - rows.push(row as Row); - } - } - return new Cursor(rows); - }, - }; - } - - transactionSync(closure: () => T): T { - this.#db.exec("BEGIN"); - try { - const value = closure(); - this.#db.exec("COMMIT"); - return value; - } catch (error) { - this.#db.exec("ROLLBACK"); - throw error; - } - } - - close(): void { - this.#cache.clear(); - this.#db.close(); - } -} - -type SQLiteValue = null | number | bigint | string | Uint8Array; - -function toSQLiteValue(value: unknown): SQLiteValue { - if (value === undefined || value === null) { - return null; - } - if (typeof value === "boolean") { - return value ? 1 : 0; - } - if (value instanceof Uint8Array) { - return value; - } - if ( - typeof value === "string" || typeof value === "number" || - typeof value === "bigint" - ) { - return value; - } - throw new TypeError(`cannot bind value of type ${typeof value}`); -} - -export interface OpenWorkspace { - fs: WorkspaceFsShim; - storage: FileSQLiteStorage; - db: Database; -} - -export function openWorkspace(dbPath: string): OpenWorkspace { - const storage = new FileSQLiteStorage(dbPath); - const db = new Database(storage); - initializeSchema(db, Date.now); - return { fs: new WorkspaceFsShim(db), storage, db }; -} diff --git a/spikes/351-worker-backends/package-lock.json b/spikes/351-worker-backends/package-lock.json deleted file mode 100644 index a8a9e45b..00000000 --- a/spikes/351-worker-backends/package-lock.json +++ /dev/null @@ -1,1188 +0,0 @@ -{ - "name": "spike-351-worker-backends", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "spike-351-worker-backends", - "dependencies": { - "@cloudflare/dofs": "file:../349-dofs/vendor/dofs", - "@effectionx/process": "0.8.1", - "effection": "4.1.0", - "just-bash": "3.0.1" - }, - "devDependencies": { - "@types/node": "^24" - } - }, - "node_modules/@borewit/text-codec": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/@borewit/text-codec/-/text-codec-0.2.2.tgz", - "integrity": "sha512-DDaRehssg1aNrH4+2hnj1B7vnUGEjU6OIlyRdkMd0aUdIUvKXrJfXsy8LVtXAy7DRvYVluWbMspsRhz2lcW0mQ==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/Borewit" - } - }, - "node_modules/@cloudflare/dofs": { - "version": "0.0.0", - "resolved": "file:../349-dofs/vendor/dofs" - }, - "node_modules/@effectionx/context-api": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/@effectionx/context-api/-/context-api-0.6.0.tgz", - "integrity": "sha512-t004qvlkJDMB6EhHP1lOQ97PeIn90m7cv4+wsRPnx4YBem+pJzTL+Sm1KWbKMjMeFJz4oqllUWuBJZsCi+nuTw==", - "license": "MIT", - "dependencies": { - "@effectionx/middleware": "0.1.1" - }, - "peerDependencies": { - "effection": "^3 || ^4" - } - }, - "node_modules/@effectionx/middleware": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/@effectionx/middleware/-/middleware-0.1.1.tgz", - "integrity": "sha512-ss/bZRkt/xzJNE59r8NR1+0K/xQcIyCm0y9n8FYC8jKdFn51SPe3m3t7EfPcK8zkdjCoTOU7k1UpIXRl26asYA==", - "license": "MIT" - }, - "node_modules/@effectionx/node": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/@effectionx/node/-/node-0.2.4.tgz", - "integrity": "sha512-cPnp3fvfBKjGWekmBHdhZr5ScAr3Mg+x5IXpO8uKFe7AZ8EPAT9Di6skuB4kuGFJtRtS0Z1e5G4+2eJyapKhYA==", - "license": "MIT", - "peerDependencies": { - "effection": "^3 || ^4" - } - }, - "node_modules/@effectionx/process": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/@effectionx/process/-/process-0.8.1.tgz", - "integrity": "sha512-xyXlFja0Ill80lQ3IYfksXtJkqVmWuUOogRn/qlHWCAGlZj+MGGF8gOFbyzk/3Kx4pj14riVGgF/cyT5XCzqDw==", - "license": "MIT", - "dependencies": { - "@effectionx/context-api": "0.6.0", - "@effectionx/node": "0.2.4", - "@effectionx/scope-eval": "0.1.3", - "cross-spawn": "^7", - "ctrlc-windows": "^2", - "shellwords-ts": "^3.0.1" - }, - "peerDependencies": { - "effection": "^3 || ^4" - } - }, - "node_modules/@effectionx/scope-eval": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/@effectionx/scope-eval/-/scope-eval-0.1.3.tgz", - "integrity": "sha512-Acn45lb3H94WYhNVHXYtXOZYzjpBGDPPlsyW1Talb/vYjQzCzus5lkxxOlPyphzvi7d+7mGNXiIVt4JLSZmLnQ==", - "license": "MIT", - "peerDependencies": { - "effection": "^3 || ^4" - } - }, - "node_modules/@jitl/quickjs-ffi-types": { - "version": "0.32.0", - "resolved": "https://registry.npmjs.org/@jitl/quickjs-ffi-types/-/quickjs-ffi-types-0.32.0.tgz", - "integrity": "sha512-v9T+GQpmk43VDJ7d72sf0Nexhk+ArvtUihW27dy7lqAl0zBObFKtSBBIm5RBjwIhE8VwsPPm9PNuvPvNqLWUEg==", - "license": "MIT" - }, - "node_modules/@jitl/quickjs-wasmfile-debug-asyncify": { - "version": "0.32.0", - "resolved": "https://registry.npmjs.org/@jitl/quickjs-wasmfile-debug-asyncify/-/quickjs-wasmfile-debug-asyncify-0.32.0.tgz", - "integrity": "sha512-EX8zbXwGqCgAE764M+qvkHtyXDi/FUoMBea0JnES7vCM3P7a2+EOZOjGv85wtZ2sJhI1oJ+nekmqpOODFDY+hw==", - "license": "MIT", - "dependencies": { - "@jitl/quickjs-ffi-types": "0.32.0" - } - }, - "node_modules/@jitl/quickjs-wasmfile-debug-sync": { - "version": "0.32.0", - "resolved": "https://registry.npmjs.org/@jitl/quickjs-wasmfile-debug-sync/-/quickjs-wasmfile-debug-sync-0.32.0.tgz", - "integrity": "sha512-LeYWrPGC1uNCTBWvibo3ZLJj0CSVNYUXvJpXMCmuQ5Sap2cCACc3uvGvYV4homHHBAzfw5akoTqMMS4YFRtw+Q==", - "license": "MIT", - "dependencies": { - "@jitl/quickjs-ffi-types": "0.32.0" - } - }, - "node_modules/@jitl/quickjs-wasmfile-release-asyncify": { - "version": "0.32.0", - "resolved": "https://registry.npmjs.org/@jitl/quickjs-wasmfile-release-asyncify/-/quickjs-wasmfile-release-asyncify-0.32.0.tgz", - "integrity": "sha512-3oSwPfja12ICz4aIblB58cuY8JlEq5Txt8Cut4VLo+LH47QN+mzCnSgnbB03hWzg1LBcc+VyyI9UOag7a1NF+Q==", - "license": "MIT", - "dependencies": { - "@jitl/quickjs-ffi-types": "0.32.0" - } - }, - "node_modules/@jitl/quickjs-wasmfile-release-sync": { - "version": "0.32.0", - "resolved": "https://registry.npmjs.org/@jitl/quickjs-wasmfile-release-sync/-/quickjs-wasmfile-release-sync-0.32.0.tgz", - "integrity": "sha512-BKNDI/TPBfGlLNGYpLrhcDGXmIk4xHm4MRAisOBnOzpXVn9HZWsfmMAc9WMBrAHjvvds6HOikKeaOBKdPdpVrg==", - "license": "MIT", - "dependencies": { - "@jitl/quickjs-ffi-types": "0.32.0" - } - }, - "node_modules/@mixmark-io/domino": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz", - "integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==", - "license": "BSD-2-Clause" - }, - "node_modules/@mongodb-js/zstd": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/@mongodb-js/zstd/-/zstd-7.0.0.tgz", - "integrity": "sha512-mQ2s0pYYiav+tzCDR05Zptem8Ey2v8s11lri5RKGhTtL4COVCvVCk5vtyRYNT+9L8qSfyOqqefF9UtnW8mC5jA==", - "hasInstallScript": true, - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "node-addon-api": "^8.5.0", - "prebuild-install": "^7.1.3" - }, - "engines": { - "node": ">= 20.19.0" - } - }, - "node_modules/@nodable/entities": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-3.0.0.tgz", - "integrity": "sha512-8L9xFeTYKhm49xfIypoe2W5wV1m/3Z58kT+7kR9A8OyFxcPduI4VmxaUMQyKYrRjUoLLSXv6EKKID5Tvj9cUVw==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/nodable" - } - ], - "license": "MIT" - }, - "node_modules/@tokenizer/inflate": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/@tokenizer/inflate/-/inflate-0.4.1.tgz", - "integrity": "sha512-2mAv+8pkG6GIZiF1kNg1jAjh27IDxEPKwdGul3snfztFerfPGI1LjDezZp3i7BElXompqEtPmoPx6c2wgtWsOA==", - "license": "MIT", - "dependencies": { - "debug": "^4.4.3", - "token-types": "^6.1.1" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/Borewit" - } - }, - "node_modules/@tokenizer/token": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", - "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==", - "license": "MIT" - }, - "node_modules/@types/node": { - "version": "24.13.3", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz", - "integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~7.18.0" - } - }, - "node_modules/anynum": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/anynum/-/anynum-1.0.1.tgz", - "integrity": "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT" - }, - "node_modules/balanced-match": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", - "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "license": "MIT", - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "optional": true - }, - "node_modules/bl": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", - "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "license": "MIT", - "optional": true, - "dependencies": { - "buffer": "^5.5.0", - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } - }, - "node_modules/brace-expansion": { - "version": "5.0.9", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", - "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", - "license": "MIT", - "dependencies": { - "balanced-match": "^4.0.2" - }, - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "optional": true, - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - }, - "node_modules/chownr": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", - "license": "ISC", - "optional": true - }, - "node_modules/commander": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-6.2.1.tgz", - "integrity": "sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA==", - "license": "MIT", - "engines": { - "node": ">= 6" - } - }, - "node_modules/cross-spawn": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", - "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", - "license": "MIT", - "dependencies": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/ctrlc-windows": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/ctrlc-windows/-/ctrlc-windows-2.2.0.tgz", - "integrity": "sha512-t9y568r+T8FUuBaqKK60YGFJdj3b3ktdJW9WXIT3CuBdQhAOYdSZu75jFUN0Ay4Yz5HHicVQqAYCwcnqhOn23g==", - "license": "MIT" - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/decompress-response": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", - "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", - "license": "MIT", - "optional": true, - "dependencies": { - "mimic-response": "^3.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/deep-extend": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", - "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", - "license": "MIT", - "optional": true, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/detect-libc": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", - "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "license": "Apache-2.0", - "optional": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/diff": { - "version": "8.0.4", - "resolved": "https://registry.npmjs.org/diff/-/diff-8.0.4.tgz", - "integrity": "sha512-DPi0FmjiSU5EvQV0++GFDOJ9ASQUVFh5kD+OzOnYdi7n3Wpm9hWWGfB/O2blfHcMVTL5WkQXSnRiK9makhrcnw==", - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.3.1" - } - }, - "node_modules/effection": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/effection/-/effection-4.1.0.tgz", - "integrity": "sha512-/BJuaYhzDrvC/nrIW054z1itmCNjVNFtyv/J26xrS4pP2MLfm5TG8VFe5f4zl1oGcAoF1fq1GfDO+v0qx6URPw==", - "license": "MIT", - "engines": { - "node": ">= 16" - } - }, - "node_modules/end-of-stream": { - "version": "1.4.5", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", - "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", - "license": "MIT", - "optional": true, - "dependencies": { - "once": "^1.4.0" - } - }, - "node_modules/expand-template": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", - "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", - "license": "(MIT OR WTFPL)", - "optional": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/fast-xml-builder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.3.0.tgz", - "integrity": "sha512-F74cZEdCvuw9P41GAC3rod4X04jjWGM1JPEv/GWSqFTWLsdyMSBMBMlm9Hk3GLBgLBbdBNY8yee0pQh2RBVESQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT", - "dependencies": { - "path-expression-matcher": "^1.6.2", - "xml-naming": "^0.3.0" - } - }, - "node_modules/fast-xml-parser": { - "version": "5.10.1", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.10.1.tgz", - "integrity": "sha512-IEMIf7298kXuZSRFoGfMYrl7is8LpavODgbNz1cwIudv7KwVFnuU+UsMporfq6PD6aXSlawZlARiA3UywCTfMw==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT", - "dependencies": { - "@nodable/entities": "^3.0.0", - "fast-xml-builder": "^1.2.0", - "is-unsafe": "^2.0.0", - "path-expression-matcher": "^1.6.2", - "strnum": "^2.4.1", - "xml-naming": "^0.3.0" - }, - "bin": { - "fxparser": "src/cli/cli.js" - } - }, - "node_modules/file-type": { - "version": "21.3.4", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-21.3.4.tgz", - "integrity": "sha512-Ievi/yy8DS3ygGvT47PjSfdFoX+2isQueoYP1cntFW1JLYAuS4GD7NUPGg4zv2iZfV52uDyk5w5Z0TdpRS6Q1g==", - "license": "MIT", - "dependencies": { - "@tokenizer/inflate": "^0.4.1", - "strtok3": "^10.3.4", - "token-types": "^6.1.1", - "uint8array-extras": "^1.4.0" - }, - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/sindresorhus/file-type?sponsor=1" - } - }, - "node_modules/fs-constants": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", - "license": "MIT", - "optional": true - }, - "node_modules/github-from-package": { - "version": "0.0.0", - "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", - "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", - "license": "MIT", - "optional": true - }, - "node_modules/ieee754": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "BSD-3-Clause" - }, - "node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "license": "ISC", - "optional": true - }, - "node_modules/ini": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/ini/-/ini-6.0.0.tgz", - "integrity": "sha512-IBTdIkzZNOpqm7q3dRqJvMaldXjDHWkEDfrwGEQTs5eaQMWV+djAhR+wahyNNMAa+qpbDUhBMVt4ZKNwpPm7xQ==", - "license": "ISC", - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/is-unsafe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-unsafe/-/is-unsafe-2.0.0.tgz", - "integrity": "sha512-2LdV822R+wmI86unXA93WCFpL6g+av8ynWk0nrHyJqGop5VoocYsSLFgN8jrfalT6iGeLNM4KXuVSsULP53kEA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT" - }, - "node_modules/isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "license": "ISC" - }, - "node_modules/just-bash": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/just-bash/-/just-bash-3.0.1.tgz", - "integrity": "sha512-YVyzCN08fKarUnwqy7rKOAcX+2MLYLnYInuowmUXn3mqhrtd4ieZNBuzdQG+qYV9DqnIWuv9Whiph0WRIWsBtw==", - "license": "Apache-2.0", - "dependencies": { - "diff": "^8.0.2", - "fast-xml-parser": "^5.7.3", - "file-type": "^21.2.0", - "ini": "^6.0.0", - "minimatch": "^10.1.1", - "modern-tar": "^0.7.3", - "papaparse": "^5.5.3", - "quickjs-emscripten": "^0.32.0", - "re2js": "^1.2.1", - "seek-bzip": "^2.0.0", - "smol-toml": "^1.6.0", - "sprintf-js": "^1.1.3", - "sql.js": "^1.13.0", - "turndown": "^7.2.2", - "yaml": "^2.8.2" - }, - "bin": { - "just-bash": "dist/bin/just-bash.js", - "just-bash-shell": "dist/bin/shell/shell.js" - }, - "optionalDependencies": { - "@mongodb-js/zstd": "^7.0.0", - "node-liblzma": "^2.0.3" - } - }, - "node_modules/mimic-response": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", - "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", - "license": "MIT", - "optional": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/minimatch": { - "version": "10.2.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", - "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", - "license": "BlueOak-1.0.0", - "dependencies": { - "brace-expansion": "^5.0.8" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/minimist": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", - "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "license": "MIT", - "optional": true, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/mkdirp-classic": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", - "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", - "license": "MIT", - "optional": true - }, - "node_modules/modern-tar": { - "version": "0.7.7", - "resolved": "https://registry.npmjs.org/modern-tar/-/modern-tar-0.7.7.tgz", - "integrity": "sha512-t9VmxaqrmANnEOBhpSDI6HD192Ge48k8vmWqQQL7hSFEqHEYwZbbsu49+aKLWZeRvFs3j1pMhXOqqF4kPlvjkQ==", - "license": "MIT", - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/napi-build-utils": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", - "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", - "license": "MIT", - "optional": true - }, - "node_modules/node-abi": { - "version": "3.94.0", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.94.0.tgz", - "integrity": "sha512-W5ZNO5KRPB5TkYmGVD9F6YqhsglXJzE6etpbmT+f6EQElhiX/UTG551cnsRGvLG3fyZEg9HwaDmNmj5nwJ4z9g==", - "license": "MIT", - "optional": true, - "dependencies": { - "semver": "^7.3.5" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/node-addon-api": { - "version": "8.9.1", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.9.1.tgz", - "integrity": "sha512-4eUQWVPCUUUiBjLnHS3cXWeC6ryoPUc0U3rP7IuzapoGbzMqd/r6KKO0clr0b+snQhsrueFEhCZDdK+LK7hxKg==", - "license": "MIT", - "optional": true, - "engines": { - "node": "^18 || ^20 || >= 21" - } - }, - "node_modules/node-gyp-build": { - "version": "4.8.4", - "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", - "integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==", - "license": "MIT", - "optional": true, - "bin": { - "node-gyp-build": "bin.js", - "node-gyp-build-optional": "optional.js", - "node-gyp-build-test": "build-test.js" - } - }, - "node_modules/node-liblzma": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/node-liblzma/-/node-liblzma-2.2.0.tgz", - "integrity": "sha512-s0KzNOWwOJJgPG6wxg6cKohnAl9Wk/oW1KrQaVzJBjQwVcUGPQCzpR46Ximygjqj/3KhOrtJXnYMp/xYAXp75g==", - "hasInstallScript": true, - "license": "LGPL-3.0", - "optional": true, - "dependencies": { - "node-addon-api": "^8.5.0", - "node-gyp-build": "^4.8.4" - }, - "bin": { - "nxz": "lib/cli/nxz.js" - }, - "engines": { - "node": ">=16.0.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/oorabona" - } - }, - "node_modules/once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "license": "ISC", - "optional": true, - "dependencies": { - "wrappy": "1" - } - }, - "node_modules/papaparse": { - "version": "5.5.4", - "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.5.4.tgz", - "integrity": "sha512-SwzWD9gl/ElwYLCI0nUja1mFJzjq2D8ziShfNBa7zCHzkOozeOGDwHWQ+tvCzEZcewecWZ5U7kUopDnG+DFYEQ==", - "license": "MIT" - }, - "node_modules/path-expression-matcher": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.6.2.tgz", - "integrity": "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT", - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/prebuild-install": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", - "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", - "deprecated": "No longer maintained. Please contact the author of the relevant native addon; alternatives are available.", - "license": "MIT", - "optional": true, - "dependencies": { - "detect-libc": "^2.0.0", - "expand-template": "^2.0.3", - "github-from-package": "0.0.0", - "minimist": "^1.2.3", - "mkdirp-classic": "^0.5.3", - "napi-build-utils": "^2.0.0", - "node-abi": "^3.3.0", - "pump": "^3.0.0", - "rc": "^1.2.7", - "simple-get": "^4.0.0", - "tar-fs": "^2.0.0", - "tunnel-agent": "^0.6.0" - }, - "bin": { - "prebuild-install": "bin.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/pump": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", - "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", - "license": "MIT", - "optional": true, - "dependencies": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - }, - "node_modules/quickjs-emscripten": { - "version": "0.32.0", - "resolved": "https://registry.npmjs.org/quickjs-emscripten/-/quickjs-emscripten-0.32.0.tgz", - "integrity": "sha512-So0Sqw869y/S2oE3Nuc0uT3Dhqgvsj8FSrwBdsuTosVsG8ME5/OcudU1GxsrIFdFABgy17GHnTVO9TYV/bLQcA==", - "license": "MIT", - "dependencies": { - "@jitl/quickjs-wasmfile-debug-asyncify": "0.32.0", - "@jitl/quickjs-wasmfile-debug-sync": "0.32.0", - "@jitl/quickjs-wasmfile-release-asyncify": "0.32.0", - "@jitl/quickjs-wasmfile-release-sync": "0.32.0", - "quickjs-emscripten-core": "0.32.0" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/quickjs-emscripten-core": { - "version": "0.32.0", - "resolved": "https://registry.npmjs.org/quickjs-emscripten-core/-/quickjs-emscripten-core-0.32.0.tgz", - "integrity": "sha512-QFnPfjFey8EqknSrSxe1hZrf1/8z7/6s1QzGOmKo6++02r7QRRX7ZoyNaZh7JuVjWsVW87KnQrbZqnHkOAzUyg==", - "license": "MIT", - "dependencies": { - "@jitl/quickjs-ffi-types": "0.32.0" - } - }, - "node_modules/rc": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", - "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", - "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", - "optional": true, - "dependencies": { - "deep-extend": "^0.6.0", - "ini": "~1.3.0", - "minimist": "^1.2.0", - "strip-json-comments": "~2.0.1" - }, - "bin": { - "rc": "cli.js" - } - }, - "node_modules/rc/node_modules/ini": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", - "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", - "license": "ISC", - "optional": true - }, - "node_modules/re2js": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/re2js/-/re2js-1.3.3.tgz", - "integrity": "sha512-s/I5zEAo79SUK0Qw4dpZKpiMwbQ6Gz0KU2NRr7eaO4x/p2g7Vvmn3hdeXDg8VsaUjfj/ora+e9oi27LX/C9+mw==", - "license": "MIT" - }, - "node_modules/readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "license": "MIT", - "optional": true, - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "optional": true - }, - "node_modules/seek-bzip": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/seek-bzip/-/seek-bzip-2.0.0.tgz", - "integrity": "sha512-SMguiTnYrhpLdk3PwfzHeotrcwi8bNV4iemL9tx9poR/yeaMYwB9VzR1w7b57DuWpuqR8n6oZboi0hj3AxZxQg==", - "license": "MIT", - "dependencies": { - "commander": "^6.0.0" - }, - "bin": { - "seek-bunzip": "bin/seek-bunzip", - "seek-table": "bin/seek-bzip-table" - } - }, - "node_modules/semver": { - "version": "7.8.5", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", - "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", - "license": "ISC", - "optional": true, - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "license": "MIT", - "dependencies": { - "shebang-regex": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/shellwords-ts": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/shellwords-ts/-/shellwords-ts-3.0.1.tgz", - "integrity": "sha512-GabK4ApLMqHFRGlpgNqg8dmtHTnYHt0WUUJkIeMd3QaDrUUBEDXHSSNi3I0PzMimg8W+I0EN4TshQxsnHv1cwg==", - "license": "MIT" - }, - "node_modules/simple-concat": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", - "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "optional": true - }, - "node_modules/simple-get": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", - "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "optional": true, - "dependencies": { - "decompress-response": "^6.0.0", - "once": "^1.3.1", - "simple-concat": "^1.0.0" - } - }, - "node_modules/smol-toml": { - "version": "1.7.1", - "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.7.1.tgz", - "integrity": "sha512-PPlsspAZ4jbMBu5DMFhfUGDQLu/vrL4SyBROVS37x8ynnVmFIs1VPBz1Co8Xks3TvpIaZXmU85y4DrQ+UyVFoQ==", - "license": "BSD-3-Clause", - "engines": { - "node": ">= 18" - }, - "funding": { - "url": "https://github.com/sponsors/cyyynthia" - } - }, - "node_modules/sprintf-js": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", - "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", - "license": "BSD-3-Clause" - }, - "node_modules/sql.js": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/sql.js/-/sql.js-1.14.1.tgz", - "integrity": "sha512-gcj8zBWU5cFsi9WUP+4bFNXAyF1iRpA3LLyS/DP5xlrNzGmPIizUeBggKa8DbDwdqaKwUcTEnChtd2grWo/x/A==", - "license": "MIT" - }, - "node_modules/string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "license": "MIT", - "optional": true, - "dependencies": { - "safe-buffer": "~5.2.0" - } - }, - "node_modules/strip-json-comments": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", - "license": "MIT", - "optional": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/strnum": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.4.1.tgz", - "integrity": "sha512-M9eUSMT2dCB2cTNPG7UYj6KuK7RJR2SN2+yCV/fTW3xzTCS6EaGZ5pSMgDIjB7r8zSfTGk+dvvn9rTjpVS9Mwg==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT", - "dependencies": { - "anynum": "^1.0.1" - } - }, - "node_modules/strtok3": { - "version": "10.3.5", - "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-10.3.5.tgz", - "integrity": "sha512-ki4hZQfh5rX0QDLLkOCj+h+CVNkqmp/CMf8v8kZpkNVK6jGQooMytqzLZYUVYIZcFZ6yDB70EfD8POcFXiF5oA==", - "license": "MIT", - "dependencies": { - "@tokenizer/token": "^0.3.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/Borewit" - } - }, - "node_modules/tar-fs": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.5.tgz", - "integrity": "sha512-OboTd8mmMhZDNPV+UjQcK9yKAatXu2aJ+r1w4im1Otd4M4fl2hwvdoXUxIYHFTHWK/3y3FarBP70v3vwmGlOxw==", - "license": "MIT", - "optional": true, - "dependencies": { - "chownr": "^1.1.1", - "mkdirp-classic": "^0.5.2", - "pump": "^3.0.0", - "tar-stream": "^2.1.4" - } - }, - "node_modules/tar-stream": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", - "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", - "license": "MIT", - "optional": true, - "dependencies": { - "bl": "^4.0.3", - "end-of-stream": "^1.4.1", - "fs-constants": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/token-types": { - "version": "6.1.2", - "resolved": "https://registry.npmjs.org/token-types/-/token-types-6.1.2.tgz", - "integrity": "sha512-dRXchy+C0IgK8WPC6xvCHFRIWYUbqqdEIKPaKo/AcTUNzwLTK6AH7RjdLWsEZcAN/TBdtfUw3PYEgPr5VPr6ww==", - "license": "MIT", - "dependencies": { - "@borewit/text-codec": "^0.2.1", - "@tokenizer/token": "^0.3.0", - "ieee754": "^1.2.1" - }, - "engines": { - "node": ">=14.16" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/Borewit" - } - }, - "node_modules/tunnel-agent": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "safe-buffer": "^5.0.1" - }, - "engines": { - "node": "*" - } - }, - "node_modules/turndown": { - "version": "7.2.4", - "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.2.4.tgz", - "integrity": "sha512-I8yFsfRzmzK0WV1pNNOA4A7y4RDfFxPRxb3t+e3ui14qSGOxGtiSP6GjeX+Y6CHb7HYaFj7ECUD7VE5kQMZWGQ==", - "license": "MIT", - "dependencies": { - "@mixmark-io/domino": "^2.2.0" - }, - "engines": { - "node": ">=18", - "npm": ">=9" - } - }, - "node_modules/uint8array-extras": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/uint8array-extras/-/uint8array-extras-1.5.0.tgz", - "integrity": "sha512-rvKSBiC5zqCCiDZ9kAOszZcDvdAHwwIKJG33Ykj43OKcWsnmcBRL09YTU4nOeHZ8Y2a7l1MgTd08SBe9A8Qj6A==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/undici-types": { - "version": "7.18.2", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", - "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", - "dev": true, - "license": "MIT" - }, - "node_modules/util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "license": "MIT", - "optional": true - }, - "node_modules/which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "license": "ISC", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "license": "ISC", - "optional": true - }, - "node_modules/xml-naming": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.3.0.tgz", - "integrity": "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT", - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/yaml": { - "version": "2.9.0", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", - "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", - "license": "ISC", - "bin": { - "yaml": "bin.mjs" - }, - "engines": { - "node": ">= 14.6" - }, - "funding": { - "url": "https://github.com/sponsors/eemeli" - } - } - } -} diff --git a/spikes/351-worker-backends/package.json b/spikes/351-worker-backends/package.json deleted file mode 100644 index 5e306a4c..00000000 --- a/spikes/351-worker-backends/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "spike-351-worker-backends", - "private": true, - "type": "module", - "dependencies": { - "@cloudflare/dofs": "file:../349-dofs/vendor/dofs", - "just-bash": "3.0.1", - "effection": "4.1.0", - "@effectionx/process": "0.8.1" - }, - "devDependencies": { - "@types/node": "^24" - } -} \ No newline at end of file diff --git a/spikes/351-worker-backends/tests/javascript.test.ts b/spikes/351-worker-backends/tests/javascript.test.ts deleted file mode 100644 index ee92cb5a..00000000 --- a/spikes/351-worker-backends/tests/javascript.test.ts +++ /dev/null @@ -1,160 +0,0 @@ -import { type Operation, run } from "effection"; -import { exec } from "@effectionx/process"; - -const proof = new URL("../dist/proof", import.meta.url).pathname; - -interface Outcome { - code: number | undefined; - payload: Record; -} - -function* proofRun( - db: string, - command: string, - ...args: string[] -): Operation { - const result = yield* exec(proof, { arguments: [db, command, ...args] }) - .join(); - let payload: Record = {}; - if (result.code === 0) { - const parsed: unknown = JSON.parse( - result.stdout.trim().split("\n").at(-1) ?? "{}", - ); - if (typeof parsed === "object" && parsed !== null) { - payload = Object.fromEntries(Object.entries(parsed)); - } - } - return { code: result.code, payload }; -} - -function assertEquals(actual: unknown, expected: unknown, detail: string) { - const left = JSON.stringify(actual); - const right = JSON.stringify(expected); - if (left !== right) { - throw new Error(`${detail}: expected ${right}, got ${left}`); - } -} - -function workspace(prefix: string): string { - const dir = Deno.makeTempDirSync({ prefix }); - return `${dir}/ws.db`; -} - -const ENTRY = [ - 'import { greet } from "./dep.js";', - "export default async () => {", - ' await workspace.writeFile("/workspace/js.txt", greet("dofs"));', - ' console.log("stdout line");', - ' console.error("stderr line");', - ' return { read: await workspace.readFile("/workspace/js.txt") };', - "};", -].join("\n"); - -const DEPENDENCY = "dep.js=export const greet = (name) => `hello ${name}`;"; - -Deno.test("a module graph runs in an isolate, reaches the workspace, and reports stdout, stderr, and a result", () => - run(function* () { - const db = workspace("spike351-js-"); - yield* proofRun(db, "fs-mkdir", "/workspace"); - - const outcome = yield* proofRun(db, "js", ENTRY, "--dep", DEPENDENCY); - assertEquals(outcome.payload.outcome, "exit", "run completes"); - assertEquals(outcome.payload.exitCode, 0, "exit code"); - assertEquals(outcome.payload.stdout, "stdout line\n", "stdout"); - assertEquals(outcome.payload.stderr, "stderr line\n", "stderr"); - assertEquals( - outcome.payload.result, - { read: "hello dofs" }, - "the entry's dependency ran and the workspace round-tripped", - ); - - const committed = yield* proofRun(db, "fs-read", "/workspace/js.txt"); - assertEquals( - committed.payload.body, - "hello dofs", - "the isolate's write is committed to the database, readable by a later process", - ); - })); - -Deno.test("a thrown error is reported as stderr and a nonzero exit, not a host failure", () => - run(function* () { - const db = workspace("spike351-js-throw-"); - const outcome = yield* proofRun( - db, - "js", - 'export default () => { throw new Error("boom"); };', - ); - assertEquals(outcome.code, 0, "the host process itself succeeds"); - assertEquals(outcome.payload.exitCode, 1, "user code exit code"); - if (!String(outcome.payload.stderr).includes("boom")) { - throw new Error( - `stderr does not carry the thrown message: ${outcome.payload.stderr}`, - ); - } - })); - -// The isolate receives exactly one capability: the workspace bridge. Everything -// the host process itself could do stays unreachable. Note this holds for the -// compiled artifact, whose module graph is frozen; a `deno run` host does not -// restrict `jsr:`/`npm:` imports (see evidence/EVIDENCE.md). -Deno.test("user code cannot reach host files or the environment", () => - run(function* () { - const db = workspace("spike351-js-isolation-"); - - const file = yield* proofRun( - db, - "js", - 'export default async () => await Deno.readTextFile("/etc/passwd");', - ); - assertEquals(file.payload.exitCode, 1, "host file read is refused"); - if (!String(file.payload.stderr).includes("NotCapable")) { - throw new Error(`unexpected failure: ${file.payload.stderr}`); - } - - const environment = yield* proofRun( - db, - "js", - 'export default () => Deno.env.get("HOME");', - ); - assertEquals(environment.payload.exitCode, 1, "env read is refused"); - if (!String(environment.payload.stderr).includes("NotCapable")) { - throw new Error(`unexpected failure: ${environment.payload.stderr}`); - } - })); - -// An operation the host does not offer must fail where it is called. Nothing -// falls back to host execution — the isolate has no path to one. -Deno.test("an unsupported workspace operation is refused explicitly, at both refusal points", () => - run(function* () { - const db = workspace("spike351-js-unsupported-"); - - // Exposed by the runner, deliberately not installed by this host. - const uninstalled = yield* proofRun( - db, - "js", - 'export default async () => await workspace.symlink("/a", "/b");', - ); - assertEquals(uninstalled.payload.exitCode, 1, "uninstalled op fails"); - if ( - !String(uninstalled.payload.stderr).includes( - "unsupported filesystem op: symlink", - ) - ) { - throw new Error( - `refusal does not name the operation: ${uninstalled.payload.stderr}`, - ); - } - - // Not part of the capability surface at all. - const unbound = yield* proofRun( - db, - "js", - 'export default async () => await workspace.chmod("/workspace", 493);', - ); - assertEquals(unbound.payload.exitCode, 1, "unbound op fails"); - if (!String(unbound.payload.stderr).includes("not a function")) { - throw new Error( - `unbound op did not fail at the call: ${unbound.payload.stderr}`, - ); - } - })); diff --git a/spikes/351-worker-backends/tests/shell.test.ts b/spikes/351-worker-backends/tests/shell.test.ts deleted file mode 100644 index 8f35f7eb..00000000 --- a/spikes/351-worker-backends/tests/shell.test.ts +++ /dev/null @@ -1,145 +0,0 @@ -import { type Operation, run } from "effection"; -import { exec } from "@effectionx/process"; - -const proof = new URL("../dist/proof", import.meta.url).pathname; - -interface Outcome { - code: number | undefined; - payload: Record; -} - -// Every invocation is a separate process against the same database, so any -// state a later command observes has survived a full host restart. -function* proofRun( - db: string, - command: string, - ...args: string[] -): Operation { - const result = yield* exec(proof, { arguments: [db, command, ...args] }) - .join(); - let payload: Record = {}; - if (result.code === 0) { - const parsed: unknown = JSON.parse( - result.stdout.trim().split("\n").at(-1) ?? "{}", - ); - if (typeof parsed === "object" && parsed !== null) { - payload = Object.fromEntries(Object.entries(parsed)); - } - } - return { code: result.code, payload }; -} - -function assertEquals(actual: unknown, expected: unknown, detail: string) { - const left = JSON.stringify(actual); - const right = JSON.stringify(expected); - if (left !== right) { - throw new Error(`${detail}: expected ${right}, got ${left}`); - } -} - -function workspace(prefix: string): string { - const dir = Deno.makeTempDirSync({ prefix }); - return `${dir}/ws.db`; -} - -Deno.test("a host filesystem write is visible to the shell, and a shell write is visible to the host", () => - run(function* () { - const db = workspace("spike351-coherence-"); - yield* proofRun(db, "fs-mkdir", "/workspace"); - yield* proofRun(db, "fs-write", "/workspace/from-host.txt", "alpha"); - - const read = yield* proofRun(db, "shell", "cat from-host.txt"); - assertEquals( - read.payload.stdout, - "alpha", - "the shell reads a file the host wrote in an earlier process", - ); - - yield* proofRun(db, "shell", "echo from-shell > from-shell.txt"); - const back = yield* proofRun(db, "fs-read", "/workspace/from-shell.txt"); - assertEquals( - back.payload.body, - "from-shell\n", - "the host reads a file the shell wrote in an earlier process", - ); - })); - -Deno.test("shell semantics: pipelines, redirection, exit status, environment, cwd", () => - run(function* () { - const db = workspace("spike351-semantics-"); - yield* proofRun(db, "fs-mkdir", "/workspace"); - - const pipeline = yield* proofRun(db, "shell", "echo hi | tr a-z A-Z"); - assertEquals(pipeline.payload.stdout, "HI\n", "pipeline"); - - yield* proofRun(db, "shell", "echo one > f.txt; echo two >> f.txt"); - const appended = yield* proofRun(db, "shell", "cat f.txt"); - assertEquals(appended.payload.stdout, "one\ntwo\n", "redirection"); - - const status = yield* proofRun(db, "shell", "false; echo $?"); - assertEquals(status.payload.stdout, "1\n", "exit status of a failed command"); - - const failing = yield* proofRun(db, "shell", "exit 3"); - assertEquals(failing.payload.exitCode, 3, "explicit exit code"); - - const environment = yield* proofRun( - db, - "shell", - "echo $GREETING", - "--env", - "GREETING=hello", - ); - assertEquals(environment.payload.stdout, "hello\n", "environment variable"); - - const cwd = yield* proofRun(db, "shell", "pwd"); - assertEquals(cwd.payload.stdout, "/workspace\n", "default cwd"); - - const missing = yield* proofRun(db, "shell", "definitely-not-a-command"); - if (missing.payload.exitCode === 0) { - throw new Error("an unknown command reported success"); - } - if (String(missing.payload.stderr).length === 0) { - throw new Error("an unknown command produced no stderr"); - } - })); - -Deno.test("separate database paths are separate workspaces", () => - run(function* () { - const first = workspace("spike351-identity-a-"); - const second = workspace("spike351-identity-b-"); - yield* proofRun(first, "fs-mkdir", "/workspace"); - yield* proofRun(first, "shell", "echo only-here > marker.txt"); - yield* proofRun(second, "fs-mkdir", "/workspace"); - - const other = yield* proofRun(second, "shell", "ls"); - assertEquals( - other.payload.stdout, - "", - "a second workspace does not see the first workspace's files", - ); - })); - -// just-bash is an interpreter, not a process launcher: there is no PATH lookup -// and no native executable can run. This is the boundary between what the shell -// backend offers and arbitrary native command execution. -Deno.test("the shell cannot launch a native executable", () => - run(function* () { - const db = workspace("spike351-native-"); - yield* proofRun(db, "fs-mkdir", "/workspace"); - const attempt = yield* proofRun(db, "shell", "/bin/echo hello"); - if (attempt.payload.exitCode === 0) { - throw new Error( - "a native executable ran; the shell is not an interpreter-only boundary", - ); - } - })); - -Deno.test("the shell cannot read a host file outside the workspace", () => - run(function* () { - const db = workspace("spike351-hostfs-"); - yield* proofRun(db, "fs-mkdir", "/workspace"); - const attempt = yield* proofRun(db, "shell", "cat /etc/passwd"); - if (attempt.payload.exitCode === 0) { - throw new Error("the shell read a host file outside the workspace"); - } - })); diff --git a/spikes/351-worker-backends/vendor.ts b/spikes/351-worker-backends/vendor.ts deleted file mode 100644 index 7f9fbf42..00000000 --- a/spikes/351-worker-backends/vendor.ts +++ /dev/null @@ -1,35 +0,0 @@ -// Builds the vendored @cloudflare/dofs the #349 spike pins, then re-links it -// here. The dependency is declared as a file: path into that spike so a single -// vendored copy — with one provenance record — serves both. - -import { main } from "effection"; -import { exec } from "@effectionx/process"; - -const here = new URL("./", import.meta.url).pathname; -const dofsDir = new URL("../349-dofs/vendor/dofs/", import.meta.url).pathname; - -main(function* () { - yield* exec("npm", { - arguments: ["install", "--no-audit", "--no-fund"], - cwd: dofsDir, - }).expect(); - yield* exec("npx", { - arguments: ["tsc", "-p", "tsconfig.build.json"], - cwd: dofsDir, - }).expect(); - // --install-links copies the file: dependency, and npm reuses that copy on a - // later install even when the source has been rebuilt since. Drop it so the - // freshly built dist is the one that lands here. - try { - Deno.removeSync(new URL("./node_modules/@cloudflare/dofs", import.meta.url), { - recursive: true, - }); - } catch { - // nothing to drop on a first run - } - yield* exec("npm", { - arguments: ["install", "--install-links", "--no-audit", "--no-fund"], - cwd: here, - }).expect(); - console.log("vendored dofs built and linked"); -}); diff --git a/spikes/351-worker-backends/vendor/worker-shell/LICENSE b/spikes/351-worker-backends/vendor/worker-shell/LICENSE deleted file mode 100644 index 631c4d3e..00000000 --- a/spikes/351-worker-backends/vendor/worker-shell/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License Copyright (c) 2026 Cloudflare, Inc. - -Permission is hereby granted, free of -charge, to any person obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the Software without -restriction, including without limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to the -following conditions: - -The above copyright notice and this permission notice -(including the next paragraph) shall be included in all copies or substantial -portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO -EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/spikes/351-worker-backends/vendor/worker-shell/PROVENANCE.md b/spikes/351-worker-backends/vendor/worker-shell/PROVENANCE.md deleted file mode 100644 index 9eb40604..00000000 --- a/spikes/351-worker-backends/vendor/worker-shell/PROVENANCE.md +++ /dev/null @@ -1,22 +0,0 @@ -# Vendored: Worker Shell filesystem adapter - -- Source: https://github.com/cloudflare/computer, - `packages/computer/src/backends/worker-shell/adapter.ts` -- Tag: `v0.1.1`, commit `63d363632e558f7e077794988d36ed75017c2a62` -- License: MIT (Cloudflare, Inc.) — see [LICENSE](LICENSE), copied from the - repository root. -- `adapter.ts` is byte-identical to upstream - (sha1 `c9ec5356d60ee7bc9aa3c5677320bf8c9aa4058c`). It has zero - `cloudflare:*` imports — its only import is `import type` from - `@cloudflare/dofs`, erased at build. It implements just-bash's - `IFileSystem` over the structural `WorkspaceFs` interface (a subset of - Cloudflare's `WorkspaceFilesystemStub`), which is exactly what makes it - reusable outside a Worker. -- The spike supplies the `@cloudflare/dofs` types via the vendored dofs from - the #349 spike and a small `WorkspaceFs` shim (`host/workspace-fs.ts`) - that adapts the in-process DOFS `WorkspaceFilesystem` — adding the - `exists`/`statOrNull`/`lstatOrNull` methods the adapter expects, which on - Cloudflare come from the RPC stub rather than the base filesystem. -- Upgrade procedure: re-copy `adapter.ts` from the new upstream tag, confirm - the `WorkspaceFs` interface at its top still matches the shim, re-run - `deno task spike:351:test`. diff --git a/spikes/351-worker-backends/vendor/worker-shell/adapter.ts b/spikes/351-worker-backends/vendor/worker-shell/adapter.ts deleted file mode 100644 index 75e97d9e..00000000 --- a/spikes/351-worker-backends/vendor/worker-shell/adapter.ts +++ /dev/null @@ -1,355 +0,0 @@ -// WorkspaceFsAdapter — implements just-bash's IFileSystem against -// a WorkspaceFilesystemStub. -// -// The adapter is a thin façade. Operations that map one-for-one -// (writeFile, readdir, mkdir, rm, chmod, symlink, readlink, stat, -// lstat) forward directly. Operations the stub doesn't expose — -// appendFile, cp, mv, exists — synthesize from the available -// primitives. Hard links and utimes aren't supported: link throws -// ENOSYS so a script that depends on them fails loudly; utimes -// is a documented no-op because the store has no atime column. -// -// The adapter holds no state. A single instance is safe to reuse -// across exec calls; nothing prevents a fresh adapter per exec -// either. - -import type { - GrepOptions, - MkdirOptions, - ReadFileOptions, - RmOptions, - WorkspaceDirentResult, - WorkspaceFoundEntry, - WorkspaceGrepMatch, - WorkspaceStatResult, - WriteFileContent, - WriteFileOptions, -} from "@cloudflare/dofs"; - -// The subset of WorkspaceFilesystemStub the adapter consumes. -// Declared structurally so tests can swap in a fake or a real stub -// without coupling the adapter to the cloudflare:workers RpcTarget -// base class. Production callers pass `workspace.stub().fs`; the -// shape matches that stub one-for-one. -export interface WorkspaceFs { - readFile(path: string): Promise>; - readFile(path: string, encoding: "utf8"): Promise; - readFile(path: string, options: ReadFileOptions): Promise>; - exists(path: string): Promise; - stat(path: string): Promise; - statOrNull(path: string): Promise; - lstat(path: string): Promise; - lstatOrNull(path: string): Promise; - readdir(path: string): Promise; - find(directory: string, pattern?: string): Promise; - ls(prefix: string): Promise; - grep(pattern: string, path: string, options?: GrepOptions): Promise; - readlink(path: string): Promise; - writeFile(path: string, content: WriteFileContent, options?: WriteFileOptions): Promise; - mkdir(path: string, options?: MkdirOptions): Promise; - rm(path: string, options?: RmOptions): Promise; - chmod(path: string, mode: number): Promise; - symlink(target: string, path: string): Promise; -} - -// Matches the subset of just-bash's IFileSystem the adapter -// implements. Kept structural so the package compiles without -// resolving just-bash's type entry — the Worker entry point ships -// the real binding to Bash. -interface FsStat { - isFile: boolean; - isDirectory: boolean; - isSymbolicLink: boolean; - mode: number; - size: number; - mtime: Date; -} - -interface DirentEntry { - name: string; - isFile: boolean; - isDirectory: boolean; - isSymbolicLink: boolean; -} - -type AdapterReadFileOptions = { encoding?: "utf8" | null } | "utf8" | null | undefined; - -export class WorkspaceFsAdapter { - readonly #fs: WorkspaceFs; - - constructor(fs: WorkspaceFs) { - this.#fs = fs; - } - - // --- Reads ------------------------------------------------------- - - async readFile(path: string, _options?: AdapterReadFileOptions): Promise { - if (isDevNull(path)) return ""; - // just-bash's readFile is text-by-default. The stub mirrors - // that with the "utf8" overload; we always reach for it here. - return this.#fs.readFile(path, "utf8"); - } - - async readFileBuffer(path: string): Promise { - if (isDevNull(path)) return new Uint8Array(0); - const stream = await this.#fs.readFile(path); - return new Uint8Array(await new Response(stream).arrayBuffer()); - } - - async exists(path: string): Promise { - if (isVirtualDevPath(path)) return true; - // Resolve expected misses on the host side. Throwing ENOENT across Workers - // RPC makes workerd report an uncaught exception even when just-bash is - // deliberately probing PATH and catches the rejection. - return this.#fs.exists(path); - } - - async stat(path: string): Promise { - const virtual = virtualDevStat(path); - if (virtual !== undefined) return virtual; - const stat = await this.#fs.statOrNull(path); - if (stat === null) throw missingPath(path); - return mapStat(stat); - } - - async lstat(path: string): Promise { - const virtual = virtualDevStat(path); - if (virtual !== undefined) return virtual; - const stat = await this.#fs.lstatOrNull(path); - if (stat === null) throw missingPath(path); - return mapStat(stat); - } - - async readdir(path: string): Promise { - if (isDevDir(path)) return ["null"]; - const entries = await this.#fs.readdir(path); - return entries.map((e) => e.name); - } - - async readdirWithFileTypes(path: string): Promise { - if (isDevDir(path)) { - return [{ name: "null", isFile: true, isDirectory: false, isSymbolicLink: false }]; - } - const entries = await this.#fs.readdir(path); - return entries.map((e) => ({ - name: e.name, - isFile: e.isFile, - isDirectory: e.isDirectory, - isSymbolicLink: e.isSymbolicLink ?? false, - })); - } - - async readlink(path: string): Promise { - if (isVirtualDevPath(path)) { - throw createWorkspaceError("EINVAL", "not a symbolic link", path); - } - return this.#fs.readlink(path); - } - - // --- Writes ------------------------------------------------------ - - async writeFile(path: string, content: string | Uint8Array, _options?: unknown): Promise { - if (isDevNull(path)) return; - await this.#fs.writeFile(path, content); - } - - async appendFile(path: string, content: string | Uint8Array, _options?: unknown): Promise { - if (isDevNull(path)) return; - let existing: Uint8Array; - try { - const stream = await this.#fs.readFile(path); - existing = new Uint8Array(await new Response(stream).arrayBuffer()); - } catch (err) { - if ((err as { code?: string }).code !== "ENOENT") throw err; - existing = new Uint8Array(0); - } - const addition = typeof content === "string" ? new TextEncoder().encode(content) : content; - const out = new Uint8Array(existing.byteLength + addition.byteLength); - out.set(existing, 0); - out.set(addition, existing.byteLength); - await this.#fs.writeFile(path, out); - } - - async mkdir(path: string, options?: { recursive?: boolean }): Promise { - if (isDevDir(path)) return; - await this.#fs.mkdir(path, options); - } - - async rm(path: string, options?: { recursive?: boolean; force?: boolean }): Promise { - if (isVirtualDevPath(path)) return; - await this.#fs.rm(path, options); - } - - async chmod(path: string, mode: number): Promise { - if (isVirtualDevPath(path)) return; - await this.#fs.chmod(path, mode); - } - - async symlink(target: string, linkPath: string): Promise { - if (isVirtualDevPath(linkPath)) { - throw createWorkspaceError("EEXIST", "file exists", linkPath); - } - await this.#fs.symlink(target, linkPath); - } - - // --- Composites -------------------------------------------------- - - async cp(src: string, dest: string, options?: { recursive?: boolean }): Promise { - const recursive = options?.recursive === true; - const s = await this.#fs.stat(src); - if (s.isDirectory) { - if (!recursive) { - throw createWorkspaceError("EISDIR", `cp: source is a directory: ${src}`, src); - } - await this.#fs.mkdir(dest, { recursive: true }); - const entries = await this.#fs.readdir(src); - for (const entry of entries) { - await this.cp(joinPath(src, entry.name), joinPath(dest, entry.name), options); - } - return; - } - const bytes = await this.readFileBuffer(src); - await this.#fs.writeFile(dest, bytes); - } - - async mv(src: string, dest: string): Promise { - // The store doesn't have a native rename today, so model mv as - // copy+delete. POSIX mv is atomic when src and dest live on - // the same filesystem; this approach isn't, but it matches - // what just-bash's other adapters do. - await this.cp(src, dest, { recursive: true }); - await this.#fs.rm(src, { recursive: true }); - } - - // --- Pure utilities ---------------------------------------------- - - resolvePath(base: string, path: string): string { - if (path.startsWith("/")) return normalizePath(path); - return normalizePath(`${base}/${path}`); - } - - getAllPaths(): string[] { - // Documented as best-effort. just-bash uses this only for glob - // matching against in-memory roots that the adapter doesn't - // know about. Include virtual device paths so globs can see - // the small POSIX compatibility shim this adapter provides. - return ["/dev", "/dev/null"]; - } - - // --- Known gaps -------------------------------------------------- - - async link(_existingPath: string, _newPath: string): Promise { - throw createWorkspaceError("ENOSYS", "hard links are not supported by the workspace store"); - } - - async utimes(_path: string, _atime: Date, _mtime: Date): Promise { - // No atime column in the store, and chmod-without-mode-change - // is the only way to bump mtime today. Make this a no-op so - // scripts that touch utimes don't blow up. - } - - async realpath(path: string): Promise { - // The store's resolver already follows symlinks. realpath is - // "the path you'd land on if you walked symlinks all the way - // through" — for a non-symlink the answer is the canonical - // path; for a symlink the answer is the readlink target - // resolved relative to its parent. Stat the path first so a - // missing entry surfaces ENOENT. - await this.#fs.stat(path); - return normalizePath(path); - } -} - -function missingPath(path: string): Error & { code: string; path: string } { - const error = new Error(`ENOENT: no such file or directory, stat '${path}'`) as Error & { - code: string; - path: string; - }; - error.code = "ENOENT"; - error.path = path; - return error; -} - -function mapStat(s: WorkspaceStatResult): FsStat { - return { - isFile: s.isFile, - isDirectory: s.isDirectory, - isSymbolicLink: s.isSymbolicLink, - mode: s.mode, - size: s.size, - mtime: new Date(s.mtime), - }; -} - -function isDevDir(path: string): boolean { - return normalizePath(path) === "/dev"; -} - -function isDevNull(path: string): boolean { - return normalizePath(path) === "/dev/null"; -} - -function isVirtualDevPath(path: string): boolean { - return isDevDir(path) || isDevNull(path); -} - -function virtualDevStat(path: string): FsStat | undefined { - if (isDevDir(path)) { - return { - isFile: false, - isDirectory: true, - isSymbolicLink: false, - mode: 0o755, - size: 0, - mtime: new Date(0), - }; - } - if (isDevNull(path)) { - return { - isFile: true, - isDirectory: false, - isSymbolicLink: false, - mode: 0o666, - size: 0, - mtime: new Date(0), - }; - } - return undefined; -} - -function joinPath(a: string, b: string): string { - if (a.endsWith("/")) return a + b; - return `${a}/${b}`; -} - -function normalizePath(path: string): string { - const parts = path.split("/"); - const stack: string[] = []; - for (const part of parts) { - if (part === "" || part === ".") continue; - if (part === "..") { - stack.pop(); - continue; - } - stack.push(part); - } - return `/${stack.join("/")}`; -} - -function createWorkspaceError( - code: string, - message: string, - path?: string, -): Error & { - code: string; - path?: string; -} { - const error = new Error(path === undefined ? message : `${message}: ${path}`) as Error & { - code: string; - path?: string; - }; - error.name = "WorkspaceFsError"; - error.code = code; - if (path !== undefined) error.path = path; - return error; -}