From 1fee39a73aa38b33a1a567bd32599591adfc31cf Mon Sep 17 00:00:00 2001 From: Taras Mankovski <74687+taras@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:09:22 -0400 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Retain=20atomic=20workflow=20Worksp?= =?UTF-8?q?aces?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .oxfmtrc.json | 2 +- AGENTS.md | 13 +- architecture.md | 40 +- deno.json | 11 +- deno.lock | 3 +- package.json | 2 +- packages/durable-streams/effect.ts | 36 +- packages/durable-streams/live-coordinator.ts | 34 + packages/durable-streams/mod.ts | 6 + .../tests/live-coordinator.test.ts | 72 ++ packages/workflow/deno.json | 3 + packages/workflow/mod.ts | 7 + packages/workflow/package.json | 3 +- packages/workflow/src/deno/connections.ts | 144 +++ packages/workflow/src/deno/database.ts | 176 ++- packages/workflow/src/deno/journal-route.ts | 67 ++ packages/workflow/src/deno/journal.ts | 22 +- packages/workflow/src/deno/lock.ts | 40 +- packages/workflow/src/deno/provider.ts | 136 +-- packages/workflow/src/deno/savepoints.ts | 77 ++ packages/workflow/src/deno/schema.ts | 342 +++++- packages/workflow/src/deno/transaction.ts | 21 +- .../workflow/src/deno/workspace/filesystem.ts | 127 ++ packages/workflow/src/deno/workspace/root.ts | 851 +++++++++++++ packages/workflow/src/storage/api.ts | 8 +- packages/workflow/src/storage/errors.ts | 13 + packages/workflow/src/workspace/api.ts | 49 + .../tests/support/workspace-crash-child.ts | 43 + .../tests/support/workspace-restart-child.ts | 31 + .../tests/workflow-run-storage.test.ts | 34 +- .../tests/workspace-filesystem.test.ts | 820 +++++++++++++ .../vendor/cloudflare-computer-dofs/LICENSE | 21 + .../cloudflare-computer-dofs/MANIFEST.json | 528 ++++++++ .../cloudflare-computer-dofs/PROVENANCE.md | 15 + .../generated/errors.d.ts | 7 + .../generated/errors.js | 10 + .../generated/fs/blobCache.d.ts | 3 + .../generated/fs/blobCache.js | 78 ++ .../generated/fs/chmod.d.ts | 2 + .../generated/fs/chmod.js | 25 + .../generated/fs/filesystem.d.ts | 32 + .../generated/fs/filesystem.js | 91 ++ .../generated/fs/find.d.ts | 6 + .../generated/fs/find.js | 84 ++ .../generated/fs/grep.d.ts | 10 + .../generated/fs/grep.js | 74 ++ .../generated/fs/link.d.ts | 2 + .../generated/fs/link.js | 53 + .../generated/fs/ls.d.ts | 2 + .../generated/fs/ls.js | 49 + .../generated/fs/mkdir.d.ts | 6 + .../generated/fs/mkdir.js | 83 ++ .../generated/fs/mount-guard.d.ts | 5 + .../generated/fs/mount-guard.js | 76 ++ .../generated/fs/readFile.d.ts | 8 + .../generated/fs/readFile.js | 156 +++ .../generated/fs/readdir.d.ts | 13 + .../generated/fs/readdir.js | 55 + .../generated/fs/readlink.d.ts | 2 + .../generated/fs/readlink.js | 15 + .../generated/fs/rename.d.ts | 2 + .../generated/fs/rename.js | 164 +++ .../generated/fs/resolve.d.ts | 13 + .../generated/fs/resolve.js | 175 +++ .../generated/fs/resolveCache.d.ts | 12 + .../generated/fs/resolveCache.js | 113 ++ .../generated/fs/rm.d.ts | 6 + .../generated/fs/rm.js | 131 ++ .../generated/fs/stat.d.ts | 13 + .../generated/fs/stat.js | 64 + .../generated/fs/symlink.d.ts | 2 + .../generated/fs/symlink.js | 50 + .../generated/fs/unlink.d.ts | 4 + .../generated/fs/unlink.js | 21 + .../generated/fs/writeBuffer.d.ts | 23 + .../generated/fs/writeBuffer.js | 93 ++ .../generated/fs/writeFile.d.ts | 37 + .../generated/fs/writeFile.js | 743 ++++++++++++ .../generated/path.d.ts | 7 + .../generated/path.js | 36 + .../generated/rev.d.ts | 2 + .../cloudflare-computer-dofs/generated/rev.js | 19 + .../generated/schema/core.d.ts | 3 + .../generated/schema/core.js | 79 ++ .../generated/schema/index.d.ts | 3 + .../generated/schema/index.js | 47 + .../generated/schema/migrations.d.ts | 8 + .../generated/schema/migrations.js | 138 +++ .../generated/schema/sync.d.ts | 1 + .../generated/schema/sync.js | 58 + .../generated/storage.d.ts | 12 + .../generated/storage.js | 106 ++ .../generated/sync/blobs.d.ts | 2 + .../generated/sync/blobs.js | 21 + .../generated/sync/changes.d.ts | 32 + .../generated/sync/changes.js | 66 + .../generated/sync/manifests.d.ts | 8 + .../generated/sync/manifests.js | 41 + .../generated/sync/paths.d.ts | 3 + .../generated/sync/paths.js | 41 + .../generated/types.d.ts | 11 + .../generated/types.js | 1 + .../upstream/src/errors.ts | 36 + .../upstream/src/fs/blobCache.ts | 87 ++ .../upstream/src/fs/chmod.ts | 34 + .../upstream/src/fs/filesystem.ts | 128 ++ .../upstream/src/fs/find.ts | 102 ++ .../upstream/src/fs/grep.ts | 107 ++ .../upstream/src/fs/link.ts | 84 ++ .../upstream/src/fs/ls.ts | 59 + .../upstream/src/fs/mkdir.ts | 130 ++ .../upstream/src/fs/mount-guard.ts | 82 ++ .../upstream/src/fs/readFile.ts | 194 +++ .../upstream/src/fs/readdir.ts | 84 ++ .../upstream/src/fs/readlink.ts | 17 + .../upstream/src/fs/rename.ts | 230 ++++ .../upstream/src/fs/resolve.ts | 264 ++++ .../upstream/src/fs/resolveCache.ts | 129 ++ .../upstream/src/fs/rm.ts | 181 +++ .../upstream/src/fs/stat.ts | 86 ++ .../upstream/src/fs/symlink.ts | 81 ++ .../upstream/src/fs/unlink.ts | 34 + .../upstream/src/fs/writeBuffer.ts | 138 +++ .../upstream/src/fs/writeFile.ts | 1070 +++++++++++++++++ .../upstream/src/path.ts | 52 + .../upstream/src/rev.ts | 21 + .../upstream/src/schema/core.ts | 81 ++ .../upstream/src/schema/index.ts | 82 ++ .../upstream/src/schema/migrations.ts | 169 +++ .../upstream/src/schema/sync.ts | 59 + .../upstream/src/storage.ts | 115 ++ .../upstream/src/sync/blobs.ts | 32 + .../upstream/src/sync/changes.ts | 105 ++ .../upstream/src/sync/manifests.ts | 74 ++ .../upstream/src/sync/paths.ts | 46 + .../upstream/src/types.ts | 16 + pnpm-lock.yaml | 3 + scripts/lib/verify.ts | 1 + scripts/runtime-test-exclusions.ts | 12 + scripts/tests/cloudflare-dofs-vendor.test.ts | 66 + scripts/tests/verify-coordinator.test.ts | 3 +- scripts/verify-cloudflare-dofs.ts | 182 +++ specs/workflow-spec.md | 44 +- specs/workflow-workspace-spec.md | 50 +- 144 files changed, 11457 insertions(+), 268 deletions(-) create mode 100644 packages/durable-streams/live-coordinator.ts create mode 100644 packages/durable-streams/tests/live-coordinator.test.ts create mode 100644 packages/workflow/src/deno/connections.ts create mode 100644 packages/workflow/src/deno/journal-route.ts create mode 100644 packages/workflow/src/deno/savepoints.ts create mode 100644 packages/workflow/src/deno/workspace/filesystem.ts create mode 100644 packages/workflow/src/deno/workspace/root.ts create mode 100644 packages/workflow/src/workspace/api.ts create mode 100644 packages/workflow/tests/support/workspace-crash-child.ts create mode 100644 packages/workflow/tests/support/workspace-restart-child.ts create mode 100644 packages/workflow/tests/workspace-filesystem.test.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/LICENSE create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/MANIFEST.json create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/PROVENANCE.md create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/errors.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/errors.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/blobCache.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/blobCache.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/chmod.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/chmod.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/filesystem.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/filesystem.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/find.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/find.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/grep.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/grep.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/link.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/link.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/ls.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/ls.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/mkdir.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/mkdir.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/mount-guard.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/mount-guard.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/readFile.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/readFile.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/readdir.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/readdir.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/readlink.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/readlink.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/rename.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/rename.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/resolve.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/resolve.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/resolveCache.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/resolveCache.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/rm.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/rm.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/stat.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/stat.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/symlink.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/symlink.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/unlink.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/unlink.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/writeBuffer.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/writeBuffer.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/writeFile.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/writeFile.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/path.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/path.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/rev.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/rev.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/core.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/core.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/index.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/index.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/migrations.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/migrations.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/sync.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/sync.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/storage.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/storage.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/blobs.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/blobs.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/changes.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/changes.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/manifests.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/manifests.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/paths.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/paths.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/types.d.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/generated/types.js create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/errors.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/blobCache.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/chmod.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/filesystem.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/find.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/grep.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/link.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/ls.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/mkdir.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/mount-guard.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/readFile.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/readdir.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/readlink.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/rename.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/resolve.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/resolveCache.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/rm.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/stat.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/symlink.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/unlink.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/writeBuffer.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/writeFile.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/path.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/rev.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/schema/core.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/schema/index.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/schema/migrations.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/schema/sync.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/storage.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/sync/blobs.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/sync/changes.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/sync/manifests.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/sync/paths.ts create mode 100644 packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/types.ts create mode 100644 scripts/tests/cloudflare-dofs-vendor.test.ts create mode 100644 scripts/verify-cloudflare-dofs.ts diff --git a/.oxfmtrc.json b/.oxfmtrc.json index 4be966ba..55f93bc0 100644 --- a/.oxfmtrc.json +++ b/.oxfmtrc.json @@ -4,5 +4,5 @@ // the exact indentation the rule tests assert on. // `packages/*/npm` is generated dnt output, written by a test while the // battery runs; format-checking it fails on a file nobody wrote by hand. - "ignorePatterns": ["**/*.md", "scripts/tests/fixtures/**", "**/npm/**"] + "ignorePatterns": ["**/*.md", "scripts/tests/fixtures/**", "**/npm/**", "**/vendor/**"] } diff --git a/AGENTS.md b/AGENTS.md index 1c51c704..39547579 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -127,11 +127,14 @@ here: root `deno.json`. Its `exclude` list holds the paths that must stay unchecked: the deliberately-malformed `scripts/tests/fixtures`; `.xmd-eval`, where a running document writes the `.ts` files its eval blocks compile to; - and `**/npm`, the dnt build's output, which a test rewrites while the battery - runs. All three are generated and belong to whichever command is producing - them — type-checking one mid-write fails on a partial file, and fails the - whole workspace check for a file nobody committed. The same output is skipped - by `lint` and `fmt`, for the same reason. + `**/npm`, the dnt build's output, which a test rewrites while the battery + runs; ignored local `.claude/worktrees`; ignored generated spike vendor + builds; and byte-identical vendored TypeScript inputs whose deterministic + JavaScript and declaration output is checked instead. Generated paths belong + to whichever command is producing them — type-checking one mid-write fails on + a partial file, and fails the whole workspace check for a file nobody + committed. The corresponding generated output is skipped by `lint` and + `fmt`, for the same reason. - `test:node` and `test:bun` derive the same corpus through `scripts/lib/test-files.ts`, which walks `tests/` beneath each workspace member plus `scripts/tests/` — that boundary, and nothing else. A new diff --git a/architecture.md b/architecture.md index ef6a1b50..b3bd104e 100644 --- a/architecture.md +++ b/architecture.md @@ -325,6 +325,13 @@ before their parent's effect begins. Declarative Git operations, including staging, switching and committing, operate on the same transactional Workspace rather than invoking an untracked native Git side effect. +The retained-filesystem foundation supplies this boundary to provider-level +Workspace effects. A live durable-effect coordinator lets the provider run the +mutation in an operation savepoint, publish or reuse an immutable root, and +invoke the existing filtered-event continuation while the caller-owned outer +transaction remains active. The default coordinator keeps ordinary durable +effects unchanged, and replay does not invoke either coordinator. + An external provider cannot join that transaction. Prompt, Git push and pull request effects derive a stable identity from the run and expansion, ask the provider to perform or reconcile that identity, then append one local result @@ -371,6 +378,34 @@ host serializes its Workspace-local effect transactions. A second long-lived DOFS connection is not a coherent reader because provider caches may retain negative entries across another connection's commit. +Each provider registry entry owns the physical connection, the single DOFS +database wrapper, the Workspace filesystem, the cooperative turn queue and the +savepoint allocator. Scope-owned database handles are leases over that entry. +DOFS synchronous transactions are nested savepoints on the caller-owned outer +transaction; they never open a second connection or a top-level transaction. + +Schema version 1 is the complete pre-release schema. Its exact structural +manifest freezes the workflow tables together with the pinned DOFS +schema-version-5 objects and the Workspace root tables. Initialization happens +only for an empty database, creates the canonical empty root and current-root +pointer in the same immediate transaction, and never repairs an existing file. +The earlier metadata-only version-1 shape is unsupported and is refused +unchanged. + +Workspace root format 1 is fixed-key-order canonical UTF-8 JSON over the root +directory and every reachable canonical absolute POSIX path. UTF-8 byte order +defines path ordering without Unicode normalization. Entries retain topology, +kind, mode, observable mtime, symlink target, file size and immutable DOFS +manifest identity; deterministic path-order groups represent hardlinks without +hashing mutable inode identity. The root ID is lowercase SHA-256 over the +domain-separated canonical bytes. + +Root rows retain exact normalized references to their transitive DOFS manifests +and blobs. Foreign keys keep that content alive, and this foundation neither +exposes nor invokes DOFS garbage collection. An adapter-private materializer +rebuilds a complete live DOFS frontier from a retained root inside the caller's +transaction and verifies that resnapshotting produces the same root ID. + The initial topology requires neither writable FUSE nor native subprocess access and does not bundle `workerd`. A Cloudflare-hosted or workerd-backed provider may install the same contextual contract without changing documents or @@ -638,11 +673,12 @@ Status is measured against main. | workflow run storage | creates or compatibly finds one run by public run ID, and retains its identity, state, document executions and filtered journal | built on main | | caller-owned storage transaction | publishes several changes, including journal events, in one transaction nothing else enlists in | built on main | | `xmd workflow start` / `xmd workflow resume` | starts or resumes a workflow run from the CLI | defined in `specs/workflow-workspace-spec.md`, unbuilt; the lookup it resumes through is built | -| implicit workflow Workspace | retains provider-neutral filesystem, repository and attachment state by run ID | defined in `specs/workflow-workspace-spec.md`, unbuilt (#218) | +| retained Workspace filesystem foundation | retains immutable, restorable filesystem roots and publishes provider-level mutations with filtered journal results | built by #365; no public workflow or `` surface | +| implicit workflow Workspace | retains provider-neutral repository, process and attachment state by run ID | filesystem foundation built by #365; public composition remains unbuilt (#218) | | Repository / Worktree / transactional Git effects | compose named checkouts and publish local mutations with their journal result | defined in `specs/workflow-workspace-spec.md`, unbuilt | | workflow inspection and history fork | reads status/history without advancing a run and creates a new run from a checkpoint | defined in `specs/workflow-workspace-spec.md`, unbuilt | | read-only workflow Agent / generated XMD | lets an Agent inspect a derived view and propose constrained executable changes | defined in `specs/workflow-workspace-spec.md`, unbuilt | -| Deno-local DOFS provider | stores the authoritative local Workspace in SQLite | persistence POC complete; effect-transaction integration unbuilt | +| Deno-local DOFS provider | stores the authoritative local Workspace in SQLite | retained filesystem and atomic transaction foundation built by #365 | | scoped Worker Shell | executes `just-bash` through the Workspace adapter inside a Deno Worker | containment and effect-transaction POCs complete (#351, #357); production integration unbuilt | | `` | retry a region until it completes | defined, unbuilt | | suspension effect | suspend durably | defined, unbuilt | diff --git a/deno.json b/deno.json index b7a651cb..c157a957 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,14 @@ { "workspace": ["packages/*", "site"], - "exclude": ["scripts/tests/fixtures", ".xmd-eval", "**/npm"], + "exclude": [ + "scripts/tests/fixtures", + ".xmd-eval", + ".claude/worktrees", + "**/npm", + "spikes/*/vendor/**/dist", + "**/vendor/**/upstream", + "**/vendor/**/generated/**/*.d.ts" + ], "nodeModulesDir": "auto", "lock": { "frozen": true @@ -52,6 +60,7 @@ "bump": "deno run -A scripts/bump-version.ts", "test": "deno test --allow-all --frozen", "verify": "deno run --allow-all --node-modules-dir=none --cached-only --frozen scripts/preflight.ts scripts/verify.ts", + "vendor:verify": "deno run --allow-read --allow-write=/tmp --allow-env --allow-run --cached-only --frozen scripts/verify-cloudflare-dofs.ts", "check": "deno check --frozen", "check:jsr": "deno publish --dry-run --allow-dirty", "review": "deno run --allow-all packages/cli/src/deno.ts run .reviews/ReviewPR.md --component-dir .reviews/components --component-dir .reviews/policies --component-dir packages/core/components -j .reviews/journal.jsonl", diff --git a/deno.lock b/deno.lock index 6fe90cc6..61989156 100644 --- a/deno.lock +++ b/deno.lock @@ -4191,7 +4191,8 @@ "dependencies": [ "npm:@effectionx/context-api@0.6.0", "npm:@effectionx/fs@0.3.0", - "npm:effection@4.1.0" + "npm:effection@4.1.0", + "npm:zod@^4.3.6" ] } }, diff --git a/package.json b/package.json index 1e1081eb..be2ed32e 100644 --- a/package.json +++ b/package.json @@ -71,7 +71,7 @@ "test:node": "tsx scripts/runtime-tests.ts node", "test:bun": "bun scripts/runtime-tests.ts bun", "test:deno": "deno task test", - "lint": "oxlint -c .oxlintrc.json --ignore-pattern 'scripts/tests/fixtures/**' --ignore-pattern '**/npm/**' packages scripts && oxfmt --check packages scripts", + "lint": "oxlint -c .oxlintrc.json --ignore-pattern 'scripts/tests/fixtures/**' --ignore-pattern '**/npm/**' --ignore-pattern '**/vendor/**' packages scripts && oxfmt --check packages scripts", "fmt": "oxfmt --write packages scripts" }, "workspaces": [ diff --git a/packages/durable-streams/effect.ts b/packages/durable-streams/effect.ts index 2455251d..c4f6d671 100644 --- a/packages/durable-streams/effect.ts +++ b/packages/durable-streams/effect.ts @@ -24,6 +24,10 @@ import type { Operation } from "effection"; import { type DurableContext, DurableCtx } from "./context.ts"; import { Divergence } from "./divergence.ts"; import { StaleInputError } from "./errors.ts"; +import { + coordinateLiveDurableEffect, + type LiveDurableEffectCoordinate, +} from "./live-coordinator.ts"; import { ReplayGuard } from "./replay-guard.ts"; import { protocolToEffection, serializeError } from "./serialize.ts"; import type { @@ -289,7 +293,7 @@ export function createDurableEffect( * concurrency — if the scope tears down, the operation is cancelled. * * Use this for durableCall and any effect where the work is expressed - * as an Operation (or can be wrapped as one via Effection's call()). + * as an Operation (or can be wrapped as one via Effection's until()). * * @param desc Structured description for the journal and divergence detection * @param execute Returns an Operation to run during live execution @@ -297,6 +301,7 @@ export function createDurableEffect( export function createDurableOperation( desc: EffectDescription, execute: () => Operation, + coordinate: LiveDurableEffectCoordinate = coordinateLiveDurableEffect, ): DurableEffect { return { description: `${desc.type}(${desc.name})`, @@ -316,24 +321,19 @@ export function createDurableOperation( // Run the entire execute → capture → persist → resolve sequence // as a structured operation in the routine's scope. routine.scope.run(function* () { - let result: Result; try { - const value = yield* execute(); - result = { status: "ok", value: value as Json }; - } catch (e) { - const error = e instanceof Error ? e : new Error(String(e)); - result = { status: "err", error: serializeError(error) }; - } - - const event: Yield = { - type: "yield", - coroutineId: ctx.coroutineId, - description: desc, - result, - }; - - try { - yield* ctx.stream.append(event); + const result = yield* coordinate({ + execute, + *publish(result): Operation { + const event: Yield = { + type: "yield", + coroutineId: ctx.coroutineId, + description: desc, + result, + }; + yield* ctx.stream.append(event); + }, + }); resolve(protocolToEffection(result)); } catch (err) { resolve({ diff --git a/packages/durable-streams/live-coordinator.ts b/packages/durable-streams/live-coordinator.ts new file mode 100644 index 00000000..7e2bafd6 --- /dev/null +++ b/packages/durable-streams/live-coordinator.ts @@ -0,0 +1,34 @@ +import { type Api, createApi } from "@effectionx/context-api"; +import type { Operation } from "effection"; +import { serializeError } from "./serialize.ts"; +import type { Json, Result } from "./types.ts"; + +export interface LiveDurableEffect { + execute(): Operation; + publish(result: Result): Operation; +} + +export interface LiveDurableEffectCoordinatorApi { + coordinate(effect: LiveDurableEffect): Operation; +} + +export type LiveDurableEffectCoordinate = ( + effect: LiveDurableEffect, +) => Operation; + +export const LiveDurableEffectCoordinator: Api = + createApi("executablemd.durable-streams.live-coordinator", { + *coordinate(effect: LiveDurableEffect): Operation { + let result: Result; + try { + result = { status: "ok", value: yield* effect.execute() }; + } catch (error) { + const failure = error instanceof Error ? error : new Error(String(error)); + result = { status: "err", error: serializeError(failure) }; + } + yield* effect.publish(result); + return result; + }, + }); + +export const coordinateLiveDurableEffect = LiveDurableEffectCoordinator.operations.coordinate; diff --git a/packages/durable-streams/mod.ts b/packages/durable-streams/mod.ts index 49d3765a..39b629b4 100644 --- a/packages/durable-streams/mod.ts +++ b/packages/durable-streams/mod.ts @@ -75,6 +75,12 @@ export { parseDurableEvent } from "./parse.ts"; // Core effect factories export { createDurableEffect, createDurableOperation } from "./effect.ts"; export type { Executor } from "./effect.ts"; +export { LiveDurableEffectCoordinator } from "./live-coordinator.ts"; +export type { + LiveDurableEffect, + LiveDurableEffectCoordinate, + LiveDurableEffectCoordinatorApi, +} from "./live-coordinator.ts"; // Workflow-enabled effects export { durableAction, durableCall, durableSleep, versionCheck } from "./operations.ts"; diff --git a/packages/durable-streams/tests/live-coordinator.test.ts b/packages/durable-streams/tests/live-coordinator.test.ts new file mode 100644 index 00000000..7397d0ca --- /dev/null +++ b/packages/durable-streams/tests/live-coordinator.test.ts @@ -0,0 +1,72 @@ +import { describe, it } from "@executablemd/test-support/bdd"; +import { expect } from "@executablemd/test-support/expect"; +import type { Operation } from "effection"; +import { + createDurableOperation, + durableRun, + InMemoryStream, + type Json, + type LiveDurableEffect, + type Result, + type Workflow, +} from "../mod.ts"; + +describe("live durable effect coordinator", () => { + it("preserves the default live publication and replay behavior", function* () { + const stream = new InMemoryStream(); + let executions = 0; + + function* workflow(): Workflow { + yield createDurableOperation( + { type: "coordinator", name: "default" }, + function* (): Operation { + executions += 1; + return "live"; + }, + ); + return "live"; + } + + expect(yield* durableRun(workflow, { stream })).toBe("live"); + expect(executions).toBe(1); + expect(stream.snapshot().filter((event) => event.type === "yield")).toHaveLength(1); + + expect( + yield* durableRun(workflow, { + stream: new InMemoryStream(stream.snapshot()), + }), + ).toBe("live"); + expect(executions).toBe(1); + }); + + it("lets a provider coordinate execution and publication without changing the protocol", function* () { + const stream = new InMemoryStream(); + const order: string[] = []; + + function* coordinate(effect: LiveDurableEffect): Operation { + order.push("begin"); + const value = yield* effect.execute(); + const result: Result = { status: "ok", value }; + order.push("publish"); + yield* effect.publish(result); + order.push("commit"); + return result; + } + + function* workflow(): Workflow { + yield createDurableOperation( + { type: "coordinator", name: "provider" }, + function* (): Operation { + order.push("execute"); + return { retained: true }; + }, + coordinate, + ); + return { retained: true }; + } + + expect(yield* durableRun(workflow, { stream })).toEqual({ retained: true }); + expect(order).toEqual(["begin", "execute", "publish", "commit"]); + expect(stream.snapshot().filter((event) => event.type === "yield")).toHaveLength(1); + }); +}); diff --git a/packages/workflow/deno.json b/packages/workflow/deno.json index 2a0c5041..eaeb2962 100644 --- a/packages/workflow/deno.json +++ b/packages/workflow/deno.json @@ -4,5 +4,8 @@ "exports": { ".": "./mod.ts", "./deno": "./deno.ts" + }, + "publish": { + "exclude": ["!vendor/cloudflare-computer-dofs/generated/**/*.d.ts"] } } diff --git a/packages/workflow/mod.ts b/packages/workflow/mod.ts index 74aa9390..715d7a81 100644 --- a/packages/workflow/mod.ts +++ b/packages/workflow/mod.ts @@ -37,6 +37,12 @@ export type { WorkflowRunStorageApi, WorkflowRunTransaction, } from "./src/storage/api.ts"; +export type { + WorkflowWorkspace, + WorkspaceDirectoryEntry, + WorkspaceFilesystem, + WorkspaceStat, +} from "./src/workspace/api.ts"; export { definitionToJson, parseWorkflowDefinition } from "./src/storage/definition.ts"; export type { GitWorkflowDefinitionV1, WorkflowDefinition } from "./src/storage/definition.ts"; @@ -64,6 +70,7 @@ export { WorkflowDatabaseClosedError, WorkflowDatabaseCorruptError, WorkflowDatabaseFormatError, + WorkflowIncompleteVersionOneError, WorkflowDefinitionError, WorkflowDocumentExecutionError, WorkflowRecordMalformedError, diff --git a/packages/workflow/package.json b/packages/workflow/package.json index e34ec611..766d60dd 100644 --- a/packages/workflow/package.json +++ b/packages/workflow/package.json @@ -13,6 +13,7 @@ "@executablemd/core": "workspace:*", "@executablemd/durable-streams": "workspace:*", "@executablemd/runtime": "workspace:*", - "effection": "4.1.0" + "effection": "4.1.0", + "zod": "^4.3.6" } } diff --git a/packages/workflow/src/deno/connections.ts b/packages/workflow/src/deno/connections.ts new file mode 100644 index 00000000..0599b302 --- /dev/null +++ b/packages/workflow/src/deno/connections.ts @@ -0,0 +1,144 @@ +import { randomUUID } from "node:crypto"; +import { DatabaseSync } from "node:sqlite"; +import { Database as CloudflareDatabase } from "../../vendor/cloudflare-computer-dofs/generated/storage.js"; +import { WorkspaceFilesystem } from "../../vendor/cloudflare-computer-dofs/generated/fs/filesystem.js"; +import type { + DurableObjectStorageLike, + SQLCursorLike, + SQLStorageLike, +} from "../../vendor/cloudflare-computer-dofs/generated/types.d.ts"; +import { createConnectionLock, type ConnectionLock } from "./lock.ts"; +import { createSavepointManager, type SavepointManager } from "./savepoints.ts"; + +export interface RunConnection { + readonly path: string; + readonly generation: string; + readonly database: DatabaseSync; + readonly dofs: CloudflareDatabase; + readonly filesystem: WorkspaceFilesystem; + readonly lock: ConnectionLock; + readonly savepoints: SavepointManager; + transactionOpen: boolean; + activeTransactionId: string | undefined; + close(): void; +} + +export interface WorkflowRunConnections { + at(path: string): RunConnection; + close(): void; +} + +class SqliteStorage implements SQLStorageLike { + readonly database: DatabaseSync; + readonly savepoints: () => SavepointManager; + + constructor(database: DatabaseSync, savepoints: () => SavepointManager) { + this.database = database; + this.savepoints = savepoints; + } + + exec>( + query: string, + ...bindings: unknown[] + ): SQLCursorLike { + const statement = this.database.prepare(query); + const rows = Reflect.apply(statement.all, statement, bindings); + return { + toArray(): Row[] { + return rows; + }, + }; + } +} + +function createConnection(path: string): RunConnection { + const database = new DatabaseSync(path); + database.exec("PRAGMA foreign_keys = ON"); + database.exec("PRAGMA busy_timeout = 5000"); + + let open = true; + const connection: { + savepoints: SavepointManager | undefined; + transactionOpen: boolean; + activeTransactionId: string | undefined; + } = { savepoints: undefined, transactionOpen: false, activeTransactionId: undefined }; + const storage = new SqliteStorage(database, () => { + const savepoints = connection.savepoints; + if (savepoints === undefined) { + throw new WorkflowConnectionStateError("the savepoint manager is not installed"); + } + return savepoints; + }); + const durableStorage: DurableObjectStorageLike = { + sql: storage, + transactionSync(closure: () => T): T { + return storage.savepoints().synchronous(closure); + }, + }; + const dofs = new CloudflareDatabase(durableStorage); + const savepoints = createSavepointManager(database, () => connection.transactionOpen); + connection.savepoints = savepoints; + + return { + path, + generation: randomUUID(), + database, + dofs, + filesystem: new WorkspaceFilesystem(dofs), + lock: createConnectionLock(), + savepoints, + get transactionOpen() { + return connection.transactionOpen; + }, + set transactionOpen(value: boolean) { + connection.transactionOpen = value; + }, + get activeTransactionId() { + return connection.activeTransactionId; + }, + set activeTransactionId(value: string | undefined) { + connection.activeTransactionId = value; + }, + close() { + if (open) { + open = false; + database.close(); + } + }, + }; +} + +export class WorkflowConnectionStateError extends Error { + override name = "WorkflowConnectionStateError"; +} + +export function createWorkflowRunConnections(): WorkflowRunConnections { + const entries = new Map(); + let open = true; + + return { + at(path: string): RunConnection { + if (!open) { + throw new WorkflowConnectionStateError("the workflow storage provider has closed"); + } + const existing = entries.get(path); + if (existing !== undefined) { + return existing; + } + const created = createConnection(path); + entries.set(path, created); + return created; + }, + + close(): void { + if (!open) { + return; + } + open = false; + for (const entry of entries.values()) { + entry.close(); + } + entries.clear(); + }, + }; +} diff --git a/packages/workflow/src/deno/database.ts b/packages/workflow/src/deno/database.ts index 95bdc06c..fb57ed93 100644 --- a/packages/workflow/src/deno/database.ts +++ b/packages/workflow/src/deno/database.ts @@ -25,15 +25,22 @@ * * ## Lifetime * - * The handle belongs to the scope that opened it. When that scope ends the - * connection closes, and every later call answers with a closed-handle failure - * rather than reopening the file behind the caller's back. + * The handle belongs to the scope that opened it. When that scope ends its + * lease closes, and every later call answers with a closed-handle failure. The + * provider owns the authoritative physical connection for its own scope. */ import { randomUUID } from "node:crypto"; import type { DatabaseSync, StatementSync } from "node:sqlite"; import { ensure, Err, Ok, type Operation, resource, type Result, scoped } from "effection"; -import type { DurableEvent, DurableStream, Json } from "@executablemd/durable-streams"; +import { + createDurableOperation, + type DurableEvent, + type DurableStream, + type Json, + serializeError, +} from "@executablemd/durable-streams"; +import type { LiveDurableEffect, Result as DurableResult } from "@executablemd/durable-streams"; import type { JournalEntry, WorkflowRunDatabase, WorkflowRunTransaction } from "../storage/api.ts"; import { WorkflowDatabaseClosedError, @@ -54,7 +61,12 @@ import { type WorkflowRunRecord, } from "../storage/record.ts"; import { insertJournalEvent, readJournalEntries } from "./journal.ts"; -import type { ConnectionLock } from "./lock.ts"; +import type { RunConnection } from "./connections.ts"; +import { + routeJournalAppend, + type TransactionIdentity, + useJournalDestination, +} from "./journal-route.ts"; import { ActiveTransaction, enclosing, @@ -63,6 +75,18 @@ import { } from "./transaction.ts"; import { readDocumentExecution, readRetrieval, readRunRecord, stopReasonColumns } from "./rows.ts"; import { translateSqliteError } from "./schema.ts"; +import type { WorkflowWorkspace } from "../workspace/api.ts"; +import { + clearWorkspaceCaches, + createWorkspaceFilesystem, + isJournalableWorkspaceError, +} from "./workspace/filesystem.ts"; +import { + currentWorkspaceRoot, + retainWorkspaceRoot, + setCurrentWorkspaceRoot, + snapshotWorkspace, +} from "./workspace/root.ts"; const SELECT_RUN = "SELECT * FROM workflow_run WHERE id = 1"; const UPDATE_RUN_STATE = `UPDATE workflow_run @@ -86,19 +110,12 @@ const SELECT_EXECUTIONS = "SELECT * FROM document_executions ORDER BY sequence A /** What opening needs from whoever found the file and checked its schema. */ export interface OpenConnection { - readonly database: DatabaseSync; - readonly path: string; + readonly connection: RunConnection; readonly record: WorkflowRunRecord; - /** Shared by every handle on this file, so turns are taken per database. */ - readonly lock: ConnectionLock; } /** - * Open a run's database for the life of the calling scope. - * - * The connection closes through ordinary teardown rather than a caller - * remembering to close it, so an interrupted host leaves no connection open on - * a file another process is about to take a write lock on. + * Open a scope-owned lease on a run's provider-owned database connection. */ export function openWorkflowRunDatabase( connection: OpenConnection, @@ -118,7 +135,8 @@ interface Handle { } function createHandle(connection: OpenConnection): Handle { - const { database, path, lock } = connection; + const entry = connection.connection; + const { database, path, lock } = entry; let closed = false; let record = connection.record; @@ -171,6 +189,14 @@ function createHandle(connection: OpenConnection): Handle { function* transact( body: (transaction: WorkflowRunTransaction) => Operation, + ): Operation> { + return yield* runTransaction(function* (transaction) { + return yield* body(transaction); + }); + } + + function* runTransaction( + body: (transaction: WorkflowRunTransaction, identity: TransactionIdentity) => Operation, ): Operation> { if (closed) { return Err(new WorkflowDatabaseClosedError(record.runId)); @@ -194,13 +220,22 @@ function createHandle(connection: OpenConnection): Handle { return Err(translateSqliteError(error, path)); } - const transaction = { open: true }; + const identity: TransactionIdentity = { + id: randomUUID(), + connection: entry, + open: true, + }; + entry.transactionOpen = true; + entry.activeTransactionId = identity.id; let committed = false; // Registered after the lock, so teardown rolls back while the connection // is still ours and releases it only once that is done. yield* ensure(() => { - transaction.open = false; + identity.open = false; + entry.transactionOpen = false; + entry.activeTransactionId = undefined; + clearWorkspaceCaches(entry); if (!committed) { rollback(database); } @@ -209,7 +244,11 @@ function createHandle(connection: OpenConnection): Handle { // The chain, not just this path: a transaction on another run nested // inside this one must not hide that this one is held. yield* ActiveTransaction.set(yield* enclosing(path)); - yield* useTransactionSavepoints(database, () => transaction.open); + yield* useTransactionSavepoints(entry.savepoints, () => identity.open); + + const transaction: WorkflowRunTransaction = { + journal: enlistedJournal(database, identity, path), + }; try { // The body runs in a scope of its own, so everything it started — @@ -219,22 +258,30 @@ function createHandle(connection: OpenConnection): Handle { // would let that append autocommit on its own, published whatever the // transaction went on to decide. const value = yield* scoped(function* () { - return yield* body({ journal: enlistedJournal(database, transaction, path) }); + return yield* body(transaction, identity); }); // Closed before the commit, not after: nothing may append to a // transaction whose contents are already decided. - transaction.open = false; + identity.open = false; + entry.transactionOpen = false; + entry.activeTransactionId = undefined; database.exec("COMMIT"); committed = true; return Ok(value); } catch (error) { - transaction.open = false; + identity.open = false; + entry.transactionOpen = false; + entry.activeTransactionId = undefined; return Err(translateSqliteError(error, path)); } }); } + function* standaloneAppend(event: DurableEvent): Operation { + yield* mustSucceed(write(() => insertJournalEvent(database, event))); + } + const journal: DurableStream = { *readAll(): Operation { const entries = yield* mustSucceed(read(() => readJournalEntries(database))); @@ -242,7 +289,68 @@ function createHandle(connection: OpenConnection): Handle { }, *append(event: DurableEvent): Operation { - yield* mustSucceed(write(() => insertJournalEvent(database, event))); + yield* routeJournalAppend(entry, standaloneAppend, event); + }, + }; + + const filesystem = createWorkspaceFilesystem(entry); + + function* coordinateWorkspace( + effect: LiveDurableEffect, + ): Operation { + const coordinated = yield* runTransaction(function* (_transaction, identity) { + const previousRoot = currentWorkspaceRoot(database, path); + let result: DurableResult; + let publishedRoot = previousRoot; + try { + const value = yield* entry.savepoints.operation(function* () { + return yield* effect.execute(); + }); + const root = snapshotWorkspace(database, entry.dofs, path, true); + retainWorkspaceRoot(database, root, path); + setCurrentWorkspaceRoot(database, root.rootId, path); + publishedRoot = root.rootId; + result = { status: "ok", value }; + } catch (error) { + clearWorkspaceCaches(entry); + if (!isJournalableWorkspaceError(error)) { + throw error; + } + result = { status: "err", error: serializeError(error) }; + } + + const destination = { + path, + generation: entry.generation, + transaction: identity, + journal: enlistedJournal(database, identity, path, publishedRoot), + workspaceRootId: publishedRoot, + used: false, + }; + yield* scoped(function* () { + yield* useJournalDestination(destination); + yield* effect.publish(result); + }); + if (!destination.used) { + throw new WorkflowTransactionError( + "the Workspace effect publication did not reach this database's guarded journal router.", + ); + } + return result; + }); + if (!coordinated.ok) { + throw coordinated.error; + } + return coordinated.value; + } + + const workspace: WorkflowWorkspace = { + *currentRoot(): Operation> { + return yield* read(() => currentWorkspaceRoot(database, path)); + }, + + effect(description, mutation) { + return createDurableOperation(description, () => mutation(filesystem), coordinateWorkspace); }, }; @@ -256,6 +364,7 @@ function createHandle(connection: OpenConnection): Handle { }, journal, + workspace, transact, @@ -320,6 +429,7 @@ function createHandle(connection: OpenConnection): Handle { const stoppedAt = now(); return yield* write(() => { + requireJournalStopReason(database, columns.eventId); const changed = database .prepare(FINISH_EXECUTION) .run( @@ -353,6 +463,7 @@ function createHandle(connection: OpenConnection): Handle { const updatedAt = now(); const written = yield* write(() => { + requireJournalStopReason(database, columns.eventId); database .prepare(UPDATE_RUN_STATE) .run(state.status, columns.kind, columns.code, columns.eventId, updatedAt); @@ -370,7 +481,6 @@ function createHandle(connection: OpenConnection): Handle { database: handle, close() { closed = true; - database.close(); }, }; } @@ -383,8 +493,9 @@ function createHandle(connection: OpenConnection): Handle { */ function enlistedJournal( database: DatabaseSync, - transaction: { open: boolean }, + transaction: TransactionIdentity, path: string, + workspaceRootId?: string, ): DurableStream { return { // deno-lint-ignore require-yield @@ -401,7 +512,7 @@ function enlistedJournal( *append(event: DurableEvent): Operation { assertOpen(transaction); try { - insertJournalEvent(database, event); + insertJournalEvent(database, event, workspaceRootId); } catch (error) { throw translateSqliteError(error, path); } @@ -409,7 +520,7 @@ function enlistedJournal( }; } -function assertOpen(transaction: { open: boolean }): void { +function assertOpen(transaction: TransactionIdentity): void { if (!transaction.open) { throw new WorkflowTransactionError( "this transaction has already finished, so nothing more can be appended through it. " + @@ -464,6 +575,19 @@ function retrievalFailure(reason: string, path: string): Error { ); } +function requireJournalStopReason(database: DatabaseSync, eventId: string | null): void { + if (eventId === null) { + return; + } + const present = database.prepare("SELECT 1 FROM journal_events WHERE event_id = ?").get(eventId); + if (present === undefined) { + throw new WorkflowRequestError( + "the stop reason names a journal event this run does not hold. A journal reason " + + "points at an event that has already been appended and filtered.", + ); + } +} + /** * The run the singleton row describes, for whoever opened the file. * diff --git a/packages/workflow/src/deno/journal-route.ts b/packages/workflow/src/deno/journal-route.ts new file mode 100644 index 00000000..29ad0b70 --- /dev/null +++ b/packages/workflow/src/deno/journal-route.ts @@ -0,0 +1,67 @@ +import { createContext, type Operation } from "effection"; +import type { DurableEvent, DurableStream } from "@executablemd/durable-streams"; +import { WorkflowTransactionError } from "../storage/errors.ts"; +import type { RunConnection } from "./connections.ts"; + +export interface TransactionIdentity { + readonly id: string; + readonly connection: RunConnection; + open: boolean; +} + +export interface JournalDestination { + readonly path: string; + readonly generation: string; + readonly transaction: TransactionIdentity; + readonly journal: DurableStream; + readonly workspaceRootId: string; + used: boolean; +} + +const Destination = createContext( + "executablemd.workflow.deno.journal-destination", + undefined, +); + +export function* useJournalDestination(destination: JournalDestination): Operation { + yield* Destination.set(destination); +} + +export function* routeJournalAppend( + connection: RunConnection, + standalone: (event: DurableEvent) => Operation, + event: DurableEvent, +): Operation { + const destination = yield* Destination.get(); + if (destination === undefined) { + return yield* standalone(event); + } + validateDestination(connection, destination); + destination.used = true; + yield* destination.journal.append(event); +} + +function validateDestination(connection: RunConnection, destination: JournalDestination): void { + if (destination.used) { + refuse("this journal destination has already published its one effect result"); + } + if (destination.path !== connection.path || destination.transaction.connection !== connection) { + refuse("this journal destination belongs to a different workflow run database"); + } + if (destination.generation !== connection.generation) { + refuse("this journal destination belongs to a stale database connection generation"); + } + if (!destination.transaction.open || !connection.transactionOpen) { + refuse("this journal destination's transaction has already completed"); + } + if (connection.activeTransactionId !== destination.transaction.id) { + refuse("this journal destination does not name the active transaction identity"); + } + if (destination.workspaceRootId === "") { + refuse("this journal destination names no Workspace root"); + } +} + +function refuse(reason: string): never { + throw new WorkflowTransactionError(`${reason}. The event is not appended.`); +} diff --git a/packages/workflow/src/deno/journal.ts b/packages/workflow/src/deno/journal.ts index afddb156..707477bf 100644 --- a/packages/workflow/src/deno/journal.ts +++ b/packages/workflow/src/deno/journal.ts @@ -30,7 +30,8 @@ export interface JournalEntry { readonly event: DurableEvent; } -const INSERT = "INSERT INTO journal_events (event_id, record) VALUES (?, ?)"; +const INSERT = `INSERT INTO journal_events (event_id, record, workspace_root_id) + VALUES (?, ?, ?)`; const SELECT = "SELECT event_id, record FROM journal_events ORDER BY sequence ASC"; /** @@ -40,12 +41,27 @@ const SELECT = "SELECT event_id, record FROM journal_events ORDER BY sequence AS * one a caller opened is decided above this function, which is what lets a * standalone append and an enlisted append share one statement. */ -export function insertJournalEvent(database: DatabaseSync, event: DurableEvent): string { +export function insertJournalEvent( + database: DatabaseSync, + event: DurableEvent, + workspaceRootId = currentWorkspaceRoot(database), +): string { const eventId = randomUUID(); - database.prepare(INSERT).run(eventId, serializeDurableEvent(event)); + database.prepare(INSERT).run(eventId, serializeDurableEvent(event), workspaceRootId); return eventId; } +function currentWorkspaceRoot(database: DatabaseSync): string { + const row = database + .prepare("SELECT current_root_id FROM workspace_state WHERE singleton_id = 1") + .get(); + const rootId = row?.["current_root_id"]; + if (typeof rootId !== "string") { + throw new Error("the workflow Workspace has no current root"); + } + return rootId; +} + /** Every retained event, in the order it was appended. */ export function readJournalEntries(database: DatabaseSync): JournalEntry[] { const entries: JournalEntry[] = []; diff --git a/packages/workflow/src/deno/lock.ts b/packages/workflow/src/deno/lock.ts index 8fb9375a..dd9f3dc2 100644 --- a/packages/workflow/src/deno/lock.ts +++ b/packages/workflow/src/deno/lock.ts @@ -8,14 +8,11 @@ * serializes everything that touches one database rather than relying on * callers to take turns. * - * Turns are taken per database file, not per connection. Two handles opened for - * the same run have two connections, and the second one entering SQLite while - * the first holds a write lock does not wait politely: `node:sqlite` is - * synchronous, so it stops the host's event loop for the whole busy timeout — - * during which the first transaction cannot resume to commit, and the second - * ends up reporting the database busy. Waiting here instead leaves the host - * running and lets the first transaction finish. SQLite's own locking remains - * responsible for contention between processes. + * Turns are taken by the provider-owned entry for one database file. Every + * handle for that run leases the same physical connection, so a second + * operation must wait here before it can issue statements inside the first + * operation's transaction. Waiting cooperatively leaves the host running; + * SQLite's own locking remains responsible for contention between processes. * * Waiting is cancellable and hand-off is synchronous. A caller torn down while * queued leaves the queue without ever running its statements, and a caller @@ -30,33 +27,6 @@ export interface ConnectionLock { hold(): Operation; } -/** - * The turns for every database one provider has opened. - * - * Owned by the provider installation rather than the module, so the - * coordination lasts exactly as long as the scope that installed the provider - * and nothing accumulates across runs. - */ -export interface ConnectionLocks { - at(path: string): ConnectionLock; -} - -export function createConnectionLocks(): ConnectionLocks { - const locks = new Map(); - - return { - at(path: string): ConnectionLock { - const existing = locks.get(path); - if (existing !== undefined) { - return existing; - } - const created = createConnectionLock(); - locks.set(path, created); - return created; - }, - }; -} - interface Turn { readonly gate: WithResolvers; granted: boolean; diff --git a/packages/workflow/src/deno/provider.ts b/packages/workflow/src/deno/provider.ts index 0cbcb850..b6372980 100644 --- a/packages/workflow/src/deno/provider.ts +++ b/packages/workflow/src/deno/provider.ts @@ -26,7 +26,6 @@ */ import { dirname, isAbsolute } from "node:path"; -import { DatabaseSync } from "node:sqlite"; import { ensureDir, exists } from "@effectionx/fs"; import { ensure, Err, Ok, type Operation, type Result, scoped } from "effection"; import { @@ -57,19 +56,14 @@ import { } from "../storage/members.ts"; import { canonicalJson, type WorkflowRunRecord } from "../storage/record.ts"; import { openWorkflowRunDatabase, readRunRow } from "./database.ts"; -import { type ConnectionLocks, createConnectionLocks } from "./lock.ts"; +import { + createWorkflowRunConnections, + type RunConnection, + type WorkflowRunConnections, +} from "./connections.ts"; import { workflowRunPath } from "./path.ts"; import { initializeSchema, isUninitialized, translateSqliteError, verifySchema } from "./schema.ts"; -/** - * How long a connection waits for another host's write lock. - * - * SQLite is reached synchronously, so this is also how long the thread can - * stop. Long enough for a transaction that is committing, short enough that a - * host holding a lock it will never release is reported rather than waited on. - */ -const BUSY_TIMEOUT_MS = 5_000; - const INSERT_RUN = `INSERT INTO workflow_run (id, run_id, definition, base, props, status, created_at, updated_at) VALUES (1, ?, ?, ?, ?, 'running', ?, ?)`; @@ -100,15 +94,18 @@ export interface WorkflowRunStorageOptions { */ export function* useWorkflowRunStorage(options: WorkflowRunStorageOptions): Operation { const root = authorizedRoot(options.root); - const locks = createConnectionLocks(); + const connections = createWorkflowRunConnections(); + yield* ensure(() => { + connections.close(); + }); yield* WorkflowRunStorage.around( { *create([request]) { - return yield* createWorkflowRun(root, locks, request); + return yield* createWorkflowRun(root, connections, request); }, *lookup([runId]) { - return yield* lookupWorkflowRun(root, locks, runId); + return yield* lookupWorkflowRun(root, connections, runId); }, }, { at: "min" }, @@ -144,7 +141,7 @@ interface CheckedRequest { function* createWorkflowRun( root: string, - locks: ConnectionLocks, + connections: WorkflowRunConnections, request: CreateWorkflowRunRequest, ): Operation> { const checked = checkRequest(request); @@ -158,15 +155,15 @@ function* createWorkflowRun( yield* ensureDir(dirname(path)); } - const lock = locks.at(path); - - return yield* withConnection(path, function* (database): Operation> { + try { + const connection = connections.at(path); + const { lock } = connection; // Held across initialization, so a second caller creating the same run // waits here rather than inside a synchronous `BEGIN IMMEDIATE` that // would stop the host while the first one is still committing. const stored = yield* scoped(function* () { yield* lock.hold(); - return establish(database, path, wanted); + return establish(connection, path, wanted); }); if (!stored.ok) { return stored; @@ -182,13 +179,15 @@ function* createWorkflowRun( return Err(new WorkflowRunConflictError(wanted.runId, differing)); } - return Ok(yield* openWorkflowRunDatabase({ database, path, record, lock })); - }); + return Ok(yield* openWorkflowRunDatabase({ connection, record })); + } catch (error) { + return refusal(error, path); + } } function* lookupWorkflowRun( root: string, - locks: ConnectionLocks, + connections: WorkflowRunConnections, runId: string, ): Operation> { const checked = checkRunId(runId); @@ -205,13 +204,13 @@ function* lookupWorkflowRun( return Err(new WorkflowRunNotFoundError(wanted)); } - const lock = locks.at(path); - - return yield* withConnection(path, function* (database): Operation> { + try { + const connection = connections.at(path); + const { database, lock } = connection; const record = yield* scoped(function* (): Operation> { yield* lock.hold(); try { - verifySchema(database, path); + verifySchema(database, path, connection.dofs); return Ok(readRunRow(database, path)); } catch (error) { return refusal(error, path); @@ -225,81 +224,10 @@ function* lookupWorkflowRun( return Err(new WorkflowRunIdMismatchError(runId, path)); } - return Ok(yield* openWorkflowRunDatabase({ database, path, record: record.value, lock })); - }); -} - -/** - * Open the file, and close it again unless a handle takes ownership. - * - * A refused database must not leave a connection open on a file the caller is - * about to be told is unusable — that connection would hold a lock nothing was - * going to release until the process ended. That includes a refusal raised on - * the way to producing the handle: reading a row while the handle is being - * built is as capable of finding an unreadable record as reading one later. - * - * Between opening the file and handing it to a handle there is checking to do, - * and a caller may be cancelled during it. The connection is therefore given - * up through ordinary teardown as well, so an interrupted open closes what it - * opened rather than leaving the file locked by a connection nobody holds. - */ -function* withConnection( - path: string, - body: (database: DatabaseSync) => Operation>, -): Operation> { - let database: DatabaseSync; - try { - database = new DatabaseSync(path); - } catch (error) { - return refusal(error, path); - } - - let adopted = false; - let released = false; - - function release(): void { - if (adopted || released) { - return; - } - released = true; - database.close(); - } - - // Registered immediately, before the connection is even configured: from - // here on there is an open file handle, and every way out of this function — - // a failing pragma, a refusal, cancellation part-way through the checking — - // has to close it. Once a handle owns the connection this is a no-op and the - // handle's own teardown closes it. - yield* ensure(release); - - try { - // Connection settings, not changes to the file. Without a busy timeout - // SQLite refuses a contended write lock immediately, so a second host - // reaching the same run would be told the database is busy rather than - // waiting the moment it takes the first one to commit. Foreign keys are - // off by default and per connection, and without them a stop reason could - // name a journal event that is not there. - database.exec(`PRAGMA busy_timeout = ${BUSY_TIMEOUT_MS}`); - database.exec("PRAGMA foreign_keys = ON"); - } catch (error) { - release(); - return refusal(error, path); - } - - let result: Result; - try { - result = yield* body(database); + return Ok(yield* openWorkflowRunDatabase({ connection, record: record.value })); } catch (error) { - release(); return refusal(error, path); } - - if (result.ok) { - adopted = true; - } else { - release(); - } - return result; } /** @@ -311,20 +239,22 @@ function* withConnection( * run in between. */ function establish( - database: DatabaseSync, + connection: RunConnection, path: string, request: CheckedRequest, ): Result { + const { database } = connection; try { if (!isUninitialized(database, path)) { - verifySchema(database, path); + verifySchema(database, path, connection.dofs); } database.exec("BEGIN IMMEDIATE"); + connection.transactionOpen = true; try { if (isUninitialized(database, path)) { const stamp = new Date().toISOString(); - initializeSchema(database); + initializeSchema(database, connection.dofs); database .prepare(INSERT_RUN) .run( @@ -336,13 +266,15 @@ function establish( stamp, ); } else { - verifySchema(database, path); + verifySchema(database, path, connection.dofs); } const record = readRunRow(database, path); + connection.transactionOpen = false; database.exec("COMMIT"); return Ok(record); } catch (error) { + connection.transactionOpen = false; database.exec("ROLLBACK"); throw error; } diff --git a/packages/workflow/src/deno/savepoints.ts b/packages/workflow/src/deno/savepoints.ts new file mode 100644 index 00000000..affdbbec --- /dev/null +++ b/packages/workflow/src/deno/savepoints.ts @@ -0,0 +1,77 @@ +import { ensure, type Operation, scoped } from "effection"; +import type { DatabaseSync } from "node:sqlite"; +import { WorkflowTransactionError } from "../storage/errors.ts"; + +export interface SavepointManager { + synchronous(body: () => T): T; + operation(body: () => Operation): Operation; +} + +export function createSavepointManager( + database: DatabaseSync, + isTransactionOpen: () => boolean, +): SavepointManager { + let next = 0; + + function allocate(): string { + const name = `xmd_savepoint_${next}`; + next += 1; + return name; + } + + function assertOpen(): void { + if (!isTransactionOpen()) { + throw new WorkflowTransactionError( + "a savepoint needs the caller-owned workflow transaction to remain open.", + ); + } + } + + function rollback(name: string): void { + database.exec(`ROLLBACK TO ${name}`); + database.exec(`RELEASE ${name}`); + } + + return { + synchronous(body: () => T): T { + assertOpen(); + const name = allocate(); + database.exec(`SAVEPOINT ${name}`); + try { + const value = body(); + database.exec(`RELEASE ${name}`); + return value; + } catch (error) { + rollback(name); + throw error; + } + }, + + *operation(body: () => Operation): Operation { + assertOpen(); + const name = allocate(); + database.exec(`SAVEPOINT ${name}`); + let finished = false; + + yield* ensure(() => { + if (!finished) { + rollback(name); + finished = true; + } + }); + + let value: T; + try { + value = yield* scoped(body); + } catch (error) { + rollback(name); + finished = true; + throw error; + } + + database.exec(`RELEASE ${name}`); + finished = true; + return value; + }, + }; +} diff --git a/packages/workflow/src/deno/schema.ts b/packages/workflow/src/deno/schema.ts index d0c8c707..afa14926 100644 --- a/packages/workflow/src/deno/schema.ts +++ b/packages/workflow/src/deno/schema.ts @@ -24,12 +24,15 @@ */ import type { DatabaseSync } from "node:sqlite"; +import type { Database as CloudflareDatabase } from "../../vendor/cloudflare-computer-dofs/generated/storage.js"; +import { initializeSchema as initializeCloudflareSchema } from "../../vendor/cloudflare-computer-dofs/generated/schema/index.js"; import { WorkflowDatabaseCorruptError, WorkflowDatabaseFormatError, - WorkflowRequestError, + WorkflowIncompleteVersionOneError, WorkflowSchemaVersionError, } from "../storage/errors.ts"; +import { emptyWorkspaceRoot, verifyWorkspace, WORKSPACE_ROOT_FORMAT } from "./workspace/root.ts"; /** * The bytes `XMD1` as a 32-bit integer, written into the SQLite header. @@ -66,23 +69,241 @@ function coherentStopReason(): string { * Kept as separate definitions so verification can compare what a file holds * with what this build writes, rather than settling for the table's name. * - * The journal is here from the first version even though metadata landed - * first: a schema that grew a table between two commits of the same release - * would owe a migration to databases that never existed. It is also created - * first, because the stop-reason references point at it. + * The complete version-1 shape includes the pinned DOFS objects, retained + * Workspace roots, journal and metadata. Dependency order is explicit: DOFS + * content precedes root references, and roots precede the journal rows that + * name them. */ -const TABLES: ReadonlyMap = new Map([ +interface DeclaredObject { + readonly type: "table" | "index"; + readonly sql: string; +} + +const OBJECTS: ReadonlyMap = new Map([ + [ + "vfs_meta", + { + type: "table", + sql: `CREATE TABLE vfs_meta ( + k TEXT PRIMARY KEY, + v INTEGER NOT NULL + )`, + }, + ], + [ + "vfs_nodes", + { + type: "table", + sql: `CREATE TABLE vfs_nodes ( + inode INTEGER PRIMARY KEY AUTOINCREMENT, + type TEXT NOT NULL CHECK(type IN ('file','dir','symlink')), + mode INTEGER NOT NULL DEFAULT 493, + mtime INTEGER NOT NULL, + rev INTEGER NOT NULL DEFAULT 0, + mount_root TEXT, + stub_size INTEGER, + manifest_hash BLOB, + link_target TEXT, + size INTEGER NOT NULL DEFAULT 0 + )`, + }, + ], + [ + "vfs_dirents", + { + type: "table", + sql: `CREATE TABLE vfs_dirents ( + parent_inode INTEGER NOT NULL, + name TEXT NOT NULL, + child_inode INTEGER NOT NULL, + PRIMARY KEY (parent_inode, name) + ) WITHOUT ROWID`, + }, + ], + [ + "vfs_dirents_by_child", + { type: "index", sql: "CREATE INDEX vfs_dirents_by_child ON vfs_dirents(child_inode)" }, + ], + ["vfs_nodes_by_rev", { type: "index", sql: "CREATE INDEX vfs_nodes_by_rev ON vfs_nodes(rev)" }], + [ + "vfs_nodes_by_manifest_hash", + { + type: "index", + sql: `CREATE INDEX vfs_nodes_by_manifest_hash + ON vfs_nodes(manifest_hash) WHERE manifest_hash IS NOT NULL`, + }, + ], + [ + "vfs_blobs", + { + type: "table", + sql: `CREATE TABLE vfs_blobs ( + hash BLOB PRIMARY KEY, + size INTEGER NOT NULL, + last_seen INTEGER NOT NULL + )`, + }, + ], + [ + "vfs_blob_bytes", + { + type: "table", + sql: `CREATE TABLE vfs_blob_bytes ( + hash BLOB PRIMARY KEY REFERENCES vfs_blobs(hash) ON DELETE CASCADE, + bytes BLOB NOT NULL + )`, + }, + ], + [ + "vfs_chunks", + { + type: "table", + sql: `CREATE TABLE vfs_chunks ( + inode INTEGER NOT NULL, + idx INTEGER NOT NULL, + hash BLOB NOT NULL, + size INTEGER NOT NULL, + PRIMARY KEY (inode, idx) + ) WITHOUT ROWID`, + }, + ], + [ + "vfs_chunks_by_hash", + { type: "index", sql: "CREATE INDEX vfs_chunks_by_hash ON vfs_chunks(hash)" }, + ], + [ + "vfs_manifests", + { + type: "table", + sql: `CREATE TABLE vfs_manifests ( + hash BLOB PRIMARY KEY, + size INTEGER NOT NULL, + encoded BLOB NOT NULL, + last_seen INTEGER NOT NULL DEFAULT 0 + )`, + }, + ], + [ + "vfs_changes", + { + type: "table", + sql: `CREATE TABLE vfs_changes ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + rev INTEGER NOT NULL, + path TEXT NOT NULL, + op TEXT NOT NULL CHECK(op IN ('delete')) + )`, + }, + ], + [ + "vfs_changes_by_rev", + { type: "index", sql: "CREATE INDEX vfs_changes_by_rev ON vfs_changes(rev)" }, + ], + [ + "vfs_changes_by_path", + { type: "index", sql: "CREATE INDEX vfs_changes_by_path ON vfs_changes(path, id DESC)" }, + ], + [ + "_vfs_watermark", + { + type: "table", + sql: `CREATE TABLE _vfs_watermark ( + k TEXT NOT NULL, + backend TEXT NOT NULL DEFAULT 'default', + v INTEGER NOT NULL, + PRIMARY KEY (k, backend) + )`, + }, + ], + [ + "_vfs_fetch_cursor", + { + type: "table", + sql: `CREATE TABLE _vfs_fetch_cursor ( + k TEXT NOT NULL CHECK(k = 'fetch'), + backend TEXT NOT NULL DEFAULT 'default', + path TEXT, + PRIMARY KEY (k, backend) + )`, + }, + ], + [ + "_vfs_mounts", + { + type: "table", + sql: `CREATE TABLE _vfs_mounts ( + root TEXT PRIMARY KEY, + kind TEXT NOT NULL, + indexed INTEGER NOT NULL DEFAULT 0, + mode TEXT NOT NULL DEFAULT 'read-only' + CHECK(mode IN ('read-only', 'read-write')) + )`, + }, + ], + [ + "workspace_roots", + { + type: "table", + sql: `CREATE TABLE workspace_roots ( + root_id TEXT PRIMARY KEY CHECK ( + length(root_id) = 64 AND root_id NOT GLOB '*[^0-9a-f]*' + ), + format_version INTEGER NOT NULL CHECK (format_version = 1), + manifest TEXT NOT NULL CHECK (json_valid(manifest)) +) STRICT`, + }, + ], + [ + "workspace_root_manifest_refs", + { + type: "table", + sql: `CREATE TABLE workspace_root_manifest_refs ( + root_id TEXT NOT NULL REFERENCES workspace_roots(root_id) ON DELETE CASCADE, + manifest_hash BLOB NOT NULL REFERENCES vfs_manifests(hash) ON DELETE RESTRICT, + PRIMARY KEY (root_id, manifest_hash) +) STRICT, WITHOUT ROWID`, + }, + ], + [ + "workspace_root_blob_refs", + { + type: "table", + sql: `CREATE TABLE workspace_root_blob_refs ( + root_id TEXT NOT NULL REFERENCES workspace_roots(root_id) ON DELETE CASCADE, + blob_hash BLOB NOT NULL, + PRIMARY KEY (root_id, blob_hash), + FOREIGN KEY (blob_hash) REFERENCES vfs_blobs(hash) ON DELETE RESTRICT, + FOREIGN KEY (blob_hash) REFERENCES vfs_blob_bytes(hash) ON DELETE RESTRICT +) STRICT, WITHOUT ROWID`, + }, + ], + [ + "workspace_state", + { + type: "table", + sql: `CREATE TABLE workspace_state ( + singleton_id INTEGER PRIMARY KEY CHECK (singleton_id = 1), + current_root_id TEXT NOT NULL REFERENCES workspace_roots(root_id) ON DELETE RESTRICT +) STRICT`, + }, + ], [ "journal_events", - `CREATE TABLE journal_events ( + { + type: "table", + sql: `CREATE TABLE journal_events ( sequence INTEGER PRIMARY KEY AUTOINCREMENT, event_id TEXT NOT NULL UNIQUE, - record TEXT NOT NULL CHECK (json_valid(record)) + record TEXT NOT NULL CHECK (json_valid(record)), + workspace_root_id TEXT NOT NULL REFERENCES workspace_roots(root_id) ON DELETE RESTRICT ) STRICT`, + }, ], [ "workflow_run", - `CREATE TABLE workflow_run ( + { + type: "table", + sql: `CREATE TABLE workflow_run ( id INTEGER PRIMARY KEY CHECK (id = 1), run_id TEXT NOT NULL, definition TEXT NOT NULL CHECK (json_valid(definition)), @@ -96,19 +317,25 @@ const TABLES: ReadonlyMap = new Map([ updated_at TEXT NOT NULL, ${coherentStopReason()} ) STRICT`, + }, ], [ "definition_retrieval", - `CREATE TABLE definition_retrieval ( + { + type: "table", + sql: `CREATE TABLE definition_retrieval ( id INTEGER PRIMARY KEY CHECK (id = 1), metadata TEXT NOT NULL CHECK (json_valid(metadata)), revision INTEGER NOT NULL CHECK (revision >= 1 AND revision <= 9007199254740991), updated_at TEXT NOT NULL ) STRICT`, + }, ], [ "document_executions", - `CREATE TABLE document_executions ( + { + type: "table", + sql: `CREATE TABLE document_executions ( sequence INTEGER PRIMARY KEY AUTOINCREMENT, execution_id TEXT NOT NULL UNIQUE, started_at TEXT NOT NULL, @@ -121,14 +348,24 @@ const TABLES: ReadonlyMap = new Map([ CHECK (stop_status IS NOT NULL OR stop_reason_kind IS NULL), ${coherentStopReason()} ) STRICT`, + }, ], ]); +/** Objects version 1 declares, including the pinned Cloudflare structure. */ +export const REQUIRED_OBJECTS: readonly string[] = Object.freeze([...OBJECTS.keys()]); + /** Tables version 1 declares. */ -export const REQUIRED_TABLES: readonly string[] = Object.freeze([...TABLES.keys()]); +export const REQUIRED_TABLES: readonly string[] = Object.freeze( + [...OBJECTS.entries()].filter(([, object]) => object.type === "table").map(([name]) => name), +); /** Version 1 in full. */ -export const SCHEMA_SQL = [...TABLES.values()].map((sql) => `${sql};`).join("\n\n"); +export const SCHEMA_SQL = [...OBJECTS.values()] + .filter((object) => object.type === "table" && !object.sql.startsWith("CREATE TABLE vfs_")) + .filter((object) => !object.sql.startsWith("CREATE TABLE _vfs_")) + .map((object) => `${object.sql};`) + .join("\n\n"); /** * Write the version-1 schema into a database that holds nothing. @@ -137,10 +374,18 @@ export const SCHEMA_SQL = [...TABLES.values()].map((sql) => `${sql};`).join("\n\ * and the tables appear together or not at all — a half-initialized file would * be indistinguishable from one this build must refuse. */ -export function initializeSchema(database: DatabaseSync): void { +export function initializeSchema(database: DatabaseSync, dofs: CloudflareDatabase): void { database.exec(`PRAGMA application_id = ${APPLICATION_ID};`); - database.exec(`PRAGMA user_version = ${SCHEMA_VERSION};`); + initializeCloudflareSchema(dofs, () => 0); database.exec(SCHEMA_SQL); + const empty = emptyWorkspaceRoot(); + database + .prepare("INSERT INTO workspace_roots (root_id, format_version, manifest) VALUES (?, ?, ?)") + .run(empty.rootId, WORKSPACE_ROOT_FORMAT, empty.manifest); + database + .prepare("INSERT INTO workspace_state (singleton_id, current_root_id) VALUES (1, ?)") + .run(empty.rootId); + database.exec(`PRAGMA user_version = ${SCHEMA_VERSION};`); } /** @@ -165,7 +410,11 @@ export function isUninitialized(database: DatabaseSync, path: string): boolean { * Structure only. Whether the rows describe the run that was asked for is a * separate question, asked after this one succeeds. */ -export function verifySchema(database: DatabaseSync, path: string): void { +export function verifySchema( + database: DatabaseSync, + path: string, + dofs?: CloudflareDatabase, +): void { checkIntegrity(database, path); const applicationId = readPragmaNumber(database, "application_id", path); @@ -183,6 +432,10 @@ export function verifySchema(database: DatabaseSync, path: string): void { verifyStructure(database, path); checkForeignKeys(database, path); + verifyDofsVersion(database, path); + if (dofs !== undefined) { + verifyWorkspace(database, dofs, path); + } } /** @@ -194,36 +447,56 @@ export function verifySchema(database: DatabaseSync, path: string): void { */ function verifyStructure(database: DatabaseSync, path: string): void { const objects = schemaObjects(database, path); + const intermediate = [ + "definition_retrieval", + "document_executions", + "journal_events", + "workflow_run", + ]; + if ( + objects.length === intermediate.length && + objects.every((object) => object.type === "table") && + objects + .map((object) => object.name) + .sort() + .join("\0") === intermediate.join("\0") + ) { + throw new WorkflowIncompleteVersionOneError(path); + } for (const object of objects) { - if (object.type !== "table") { - throw new WorkflowDatabaseCorruptError( - path, - `it declares a ${object.type} that version ${SCHEMA_VERSION} does not`, - ); - } - const expected = TABLES.get(object.name); + const expected = OBJECTS.get(object.name); if (expected === undefined) { throw new WorkflowDatabaseCorruptError( path, - `it declares a table that version ${SCHEMA_VERSION} does not`, + `it declares an object that version ${SCHEMA_VERSION} does not`, ); } - if (normalize(object.sql) !== normalize(expected)) { + if (object.type !== expected.type || normalize(object.sql) !== normalize(expected.sql)) { throw new WorkflowDatabaseCorruptError( path, - `its ${object.name} table is not shaped the way version ${SCHEMA_VERSION} declares it`, + `its ${object.name} object is not shaped the way version ${SCHEMA_VERSION} declares it`, ); } } const present = new Set(objects.map((object) => object.name)); - const missing = REQUIRED_TABLES.filter((table) => !present.has(table)); + const missing = REQUIRED_OBJECTS.filter((name) => !present.has(name)); if (missing.length > 0) { throw new WorkflowDatabaseCorruptError(path, `it is missing the table ${missing.join(", ")}`); } } +function verifyDofsVersion(database: DatabaseSync, path: string): void { + const row = query(database, "SELECT v FROM vfs_meta WHERE k = 'schema_version'", path)[0]; + if (row?.["v"] !== 5 && row?.["v"] !== 5n) { + throw new WorkflowDatabaseCorruptError( + path, + "its Workspace filesystem schema is not version 5", + ); + } +} + /** * Ask SQLite whether it can still read its own file. * @@ -243,8 +516,8 @@ export function checkIntegrity(database: DatabaseSync, path: string): void { /** * Ask SQLite whether its references still point at anything. * - * A stop reason naming a journal event is only a reason while that event - * exists; a row pointing at one that does not is damage, not a reason. + * A retained reference names an object only while that object exists; a row + * pointing at nothing is damage rather than a partial retained state. */ function checkForeignKeys(database: DatabaseSync, path: string): void { if (query(database, "PRAGMA foreign_key_check", path).length > 0) { @@ -315,9 +588,6 @@ const SQLITE_CORRUPT = 11; /** `SQLITE_NOTADB`: the bytes are not a SQLite database at all. */ const SQLITE_NOTADB = 26; -/** `SQLITE_CONSTRAINT_FOREIGNKEY`: a reference points at a row that is not there. */ -const SQLITE_CONSTRAINT_FOREIGNKEY = 787; - /** * The typed refusal a SQLite failure describes, or the failure unchanged. * @@ -331,14 +601,6 @@ export function translateSqliteError(error: unknown, path: string): unknown { return new WorkflowDatabaseFormatError(path, "SQLite does not recognize it as a database"); case SQLITE_CORRUPT: return new WorkflowDatabaseCorruptError(path, "SQLite reported a damaged image"); - case SQLITE_CONSTRAINT_FOREIGNKEY: - // The only reference version 1 declares. A stop reason may name a - // journal event, and naming one this run does not hold is a reason that - // refers to nothing. - return new WorkflowRequestError( - "the stop reason names a journal event this run does not hold. A journal reason " + - "points at an event that has already been appended and filtered.", - ); default: return error; } diff --git a/packages/workflow/src/deno/transaction.ts b/packages/workflow/src/deno/transaction.ts index 19d84e05..5c6e8bfb 100644 --- a/packages/workflow/src/deno/transaction.ts +++ b/packages/workflow/src/deno/transaction.ts @@ -17,8 +17,8 @@ import { type Api, createApi } from "@effectionx/context-api"; import { type Context, createContext, type Operation } from "effection"; -import type { DatabaseSync } from "node:sqlite"; import { WorkflowTransactionError } from "../storage/errors.ts"; +import type { SavepointManager } from "./savepoints.ts"; /** * Every database the current scope holds a transaction on. @@ -100,11 +100,9 @@ export const savepoint: TransactionApi["savepoint"] = Transaction.operations.sav /** What the open transaction installs so `savepoint()` can answer. */ export function useTransactionSavepoints( - database: DatabaseSync, + savepoints: SavepointManager, isOpen: () => boolean, ): Operation { - let depth = 0; - return Transaction.around( { // deno-lint-ignore require-yield @@ -115,20 +113,7 @@ export function useTransactionSavepoints( ); } - const name = `xmd_savepoint_${depth}`; - depth += 1; - database.exec(`SAVEPOINT ${name}`); - try { - const value = body(); - database.exec(`RELEASE ${name}`); - return value; - } catch (error) { - database.exec(`ROLLBACK TO ${name}`); - database.exec(`RELEASE ${name}`); - throw error; - } finally { - depth -= 1; - } + return savepoints.synchronous(body); }, }, { at: "min" }, diff --git a/packages/workflow/src/deno/workspace/filesystem.ts b/packages/workflow/src/deno/workspace/filesystem.ts new file mode 100644 index 00000000..06446b9d --- /dev/null +++ b/packages/workflow/src/deno/workspace/filesystem.ts @@ -0,0 +1,127 @@ +import { type Operation, until } from "effection"; +import { link as dofsLink } from "../../../vendor/cloudflare-computer-dofs/generated/fs/link.js"; +import { rename as dofsRename } from "../../../vendor/cloudflare-computer-dofs/generated/fs/rename.js"; +import { clearBlobCache } from "../../../vendor/cloudflare-computer-dofs/generated/fs/blobCache.js"; +import { clearResolveCache } from "../../../vendor/cloudflare-computer-dofs/generated/fs/resolveCache.js"; +import type { RunConnection } from "../connections.ts"; +import type { + WorkspaceDirectoryEntry, + WorkspaceFilesystem, + WorkspaceStat, +} from "../../workspace/api.ts"; + +export function createWorkspaceFilesystem(connection: RunConnection): WorkspaceFilesystem { + const { dofs, filesystem } = connection; + + function toStat(value: { + mode: number; + mtime: number; + size: number; + isFile: boolean; + isDirectory: boolean; + isSymbolicLink: boolean; + }): WorkspaceStat { + const kind = value.isFile ? "file" : value.isDirectory ? "directory" : "symlink"; + return { mode: value.mode, mtime: value.mtime, size: value.size, kind }; + } + + return { + *readFile(path): Operation { + const stream = yield* until(filesystem.readFile(path)); + return new Uint8Array(yield* until(new Response(stream).arrayBuffer())); + }, + + *readTextFile(path): Operation { + const content = yield* until(filesystem.readFile(path, "utf8")); + if (typeof content !== "string") { + throw new Error("the Workspace text read returned a byte stream"); + } + return content; + }, + + *stat(path): Operation { + return toStat(yield* until(filesystem.stat(path))); + }, + + *lstat(path): Operation { + return toStat(yield* until(filesystem.lstat(path))); + }, + + *readlink(path): Operation { + return yield* until(filesystem.readlink(path)); + }, + + *readdir(path): Operation { + const entries = yield* until(filesystem.readdir(path)); + const result: WorkspaceDirectoryEntry[] = []; + for (const entry of entries) { + result.push({ + name: entry.name, + kind: entry.isFile ? "file" : entry.isDirectory ? "directory" : "symlink", + }); + } + return result; + }, + + *writeFile(path, content, mode): Operation { + yield* until(filesystem.writeFile(path, content, mode === undefined ? {} : { mode })); + }, + + *mkdir(path, options = {}): Operation { + yield* until(filesystem.mkdir(path, options)); + }, + + *remove(path, options = {}): Operation { + yield* until(filesystem.rm(path, options)); + }, + + // deno-lint-ignore require-yield + *rename(from, to): Operation { + dofsRename(dofs, from, to); + }, + + *chmod(path, mode): Operation { + yield* until(filesystem.chmod(path, mode)); + }, + + *symlink(target, path): Operation { + yield* until(filesystem.symlink(target, path)); + }, + + // deno-lint-ignore require-yield + *link(existingPath, newPath): Operation { + dofsLink(dofs, existingPath, newPath); + }, + }; +} + +export function clearWorkspaceCaches(connection: RunConnection): void { + clearResolveCache(connection.dofs); + clearBlobCache(connection.dofs); +} + +const WORKSPACE_ERROR_CODES = new Set([ + "ENOENT", + "ENOTEMPTY", + "ENOTDIR", + "EISDIR", + "EEXIST", + "EINVAL", + "EACCES", + "EPERM", + "EROFS", + "ENOSYS", + "EBADF", + "ELOOP", + "EUNKNOWN_HASH", +]); + +export function isJournalableWorkspaceError(error: unknown): error is Error { + return ( + error instanceof Error && + error.name === "WorkspaceFsError" && + "code" in error && + typeof error.code === "string" && + WORKSPACE_ERROR_CODES.has(error.code) + ); +} diff --git a/packages/workflow/src/deno/workspace/root.ts b/packages/workflow/src/deno/workspace/root.ts new file mode 100644 index 00000000..3c2ee657 --- /dev/null +++ b/packages/workflow/src/deno/workspace/root.ts @@ -0,0 +1,851 @@ +import { createHash } from "node:crypto"; +import type { DatabaseSync } from "node:sqlite"; +import { z } from "zod"; +import type { Database as CloudflareDatabase } from "../../../vendor/cloudflare-computer-dofs/generated/storage.js"; +import { buildManifest } from "../../../vendor/cloudflare-computer-dofs/generated/sync/manifests.js"; +import { WorkflowDatabaseCorruptError } from "../../storage/errors.ts"; +import type { SavepointManager } from "../savepoints.ts"; + +export const WORKSPACE_ROOT_FORMAT = 1; +const DOMAIN = new TextEncoder().encode("xmd-workspace-root\0v1\0"); +const decoder = new TextDecoder("utf-8", { fatal: true }); +const encoder = new TextEncoder(); +const SHA256 = /^[0-9a-f]{64}$/; + +const directoryEntrySchema = z.object({ + path: z.string(), + kind: z.literal("directory"), + mode: z.number().int().min(0).max(0o7777), + mtime: z.number().int().safe(), +}); +const fileEntrySchema = z.object({ + path: z.string(), + kind: z.literal("file"), + mode: z.number().int().min(0).max(0o7777), + mtime: z.number().int().safe(), + size: z.number().int().safe().nonnegative(), + manifest: z.string().regex(SHA256), + hardlink: z + .string() + .regex(/^h[0-9]+$/) + .nullable(), +}); +const symlinkEntrySchema = z.object({ + path: z.string(), + kind: z.literal("symlink"), + mode: z.number().int().min(0).max(0o7777), + mtime: z.number().int().safe(), + target: z.string(), +}); +const rootManifestSchema = z.object({ + format: z.literal(WORKSPACE_ROOT_FORMAT), + entries: z.array( + z.discriminatedUnion("kind", [directoryEntrySchema, fileEntrySchema, symlinkEntrySchema]), + ), +}); +const dofsManifestSchema = z.object({ + version: z.literal(1), + chunks: z.array( + z.object({ hash: z.string().regex(SHA256), size: z.number().int().safe().positive() }), + ), +}); + +export type WorkspaceRootEntry = z.infer["entries"][number]; +export type WorkspaceRootManifest = z.infer; + +export interface StoredWorkspaceRoot { + readonly rootId: string; + readonly manifest: string; + readonly manifestHashes: readonly string[]; + readonly blobHashes: readonly string[]; +} + +interface NodeRow { + readonly inode: number; + readonly type: "file" | "dir" | "symlink"; + readonly mode: number; + readonly mtime: number; + readonly manifestHash: Uint8Array | null; + readonly linkTarget: string | null; + readonly size: number; +} + +interface Chunk { + readonly hash: Uint8Array; + readonly size: number; +} + +export function emptyWorkspaceRoot(): StoredWorkspaceRoot { + return workspaceRoot( + '{"format":1,"entries":[{"path":"/","kind":"directory","mode":493,"mtime":0}]}', + [], + [], + ); +} + +export function workspaceRoot( + manifest: string, + manifestHashes: readonly string[], + blobHashes: readonly string[], +): StoredWorkspaceRoot { + const bytes = encoder.encode(manifest); + const hash = createHash("sha256"); + hash.update(DOMAIN); + hash.update(bytes); + return Object.freeze({ + rootId: hash.digest("hex"), + manifest, + manifestHashes: Object.freeze([...manifestHashes]), + blobHashes: Object.freeze([...blobHashes]), + }); +} + +export function snapshotWorkspace( + database: DatabaseSync, + dofs: CloudflareDatabase, + path: string, + repairMissingManifest: boolean, +): StoredWorkspaceRoot { + const entries: Array<{ entry: WorkspaceRootEntry; inode: number }> = []; + const visiting = new Set(); + const reachable = new Set(); + const filePaths = new Map(); + const manifestHashes = new Set(); + const blobHashes = new Set(); + + function visit(inode: number, canonicalPath: string): void { + const node = readNode(database, inode, path); + if (node.type === "dir") { + if (visiting.has(inode) || reachable.has(inode)) { + corrupt(path, "its live Workspace contains a directory cycle or directory hardlink"); + } + visiting.add(inode); + reachable.add(inode); + entries.push({ + inode, + entry: { path: canonicalPath, kind: "directory", mode: node.mode, mtime: node.mtime }, + }); + for (const child of readDirents(database, inode, path)) { + validateName(child.name, path); + const childPath = + canonicalPath === "/" ? `/${child.name}` : `${canonicalPath}/${child.name}`; + visit(child.inode, childPath); + } + visiting.delete(inode); + return; + } + + reachable.add(inode); + if (node.type === "symlink") { + const target = node.linkTarget; + if (target === null || target.includes("\0") || hasUnpairedSurrogate(target)) { + corrupt(path, "its live Workspace contains an invalid symbolic-link target"); + } + entries.push({ + inode, + entry: { + path: canonicalPath, + kind: "symlink", + mode: node.mode, + mtime: node.mtime, + target, + }, + }); + return; + } + + const paths = filePaths.get(inode) ?? []; + paths.push(canonicalPath); + filePaths.set(inode, paths); + const content = validateFile(database, dofs, node, path, repairMissingManifest); + manifestHashes.add(content.manifest); + for (const hash of content.blobs) { + blobHashes.add(hash); + } + entries.push({ + inode, + entry: { + path: canonicalPath, + kind: "file", + mode: node.mode, + mtime: node.mtime, + size: node.size, + manifest: content.manifest, + hardlink: null, + }, + }); + } + + visit(1, "/"); + if (count(database, "vfs_nodes") !== reachable.size) { + corrupt(path, "its live Workspace contains unreachable filesystem nodes"); + } + if (count(database, "vfs_dirents") !== entries.length - 1) { + corrupt(path, "its live Workspace contains unreachable directory entries"); + } + + entries.sort((left, right) => compareUtf8(left.entry.path, right.entry.path)); + let hardlink = 0; + const groups = [...filePaths.values()] + .filter((paths) => paths.length > 1) + .map((paths) => [...paths].sort(compareUtf8)) + .sort((left, right) => compareUtf8(left[0] ?? "", right[0] ?? "")); + for (const paths of groups) { + const group = `h${hardlink}`; + hardlink += 1; + for (const item of entries) { + if (item.entry.kind === "file" && paths.includes(item.entry.path)) { + item.entry.hardlink = group; + } + } + } + + const logical = entries.map((item) => item.entry); + validateEntryOrder(logical, path); + const manifest = JSON.stringify({ format: WORKSPACE_ROOT_FORMAT, entries: logical }); + return workspaceRoot(manifest, [...manifestHashes].sort(), [...blobHashes].sort()); +} + +export function retainWorkspaceRoot( + database: DatabaseSync, + root: StoredWorkspaceRoot, + path: string, +): void { + const existing = database + .prepare("SELECT manifest FROM workspace_roots WHERE root_id = ?") + .get(root.rootId); + if (existing === undefined) { + database + .prepare("INSERT INTO workspace_roots (root_id, format_version, manifest) VALUES (?, 1, ?)") + .run(root.rootId, root.manifest); + for (const hash of root.manifestHashes) { + database + .prepare("INSERT INTO workspace_root_manifest_refs (root_id, manifest_hash) VALUES (?, ?)") + .run(root.rootId, fromHex(hash)); + } + for (const hash of root.blobHashes) { + database + .prepare("INSERT INTO workspace_root_blob_refs (root_id, blob_hash) VALUES (?, ?)") + .run(root.rootId, fromHex(hash)); + } + return; + } + if (existing["manifest"] !== root.manifest) { + corrupt(path, "a retained Workspace root identity has different stored bytes"); + } + requireReferenceSet( + database, + root.rootId, + "workspace_root_manifest_refs", + "manifest_hash", + root.manifestHashes, + path, + ); + requireReferenceSet( + database, + root.rootId, + "workspace_root_blob_refs", + "blob_hash", + root.blobHashes, + path, + ); +} + +export function setCurrentWorkspaceRoot( + database: DatabaseSync, + rootId: string, + path: string, +): void { + const changed = database + .prepare("UPDATE workspace_state SET current_root_id = ? WHERE singleton_id = 1") + .run(rootId); + if (changed.changes !== 1) { + corrupt(path, "its Workspace current-root pointer is missing"); + } +} + +export function currentWorkspaceRoot(database: DatabaseSync, path: string): string { + const row = database + .prepare("SELECT current_root_id FROM workspace_state WHERE singleton_id = 1") + .get(); + const value = row?.["current_root_id"]; + if (typeof value !== "string" || !SHA256.test(value)) { + corrupt(path, "its Workspace current-root pointer is malformed"); + } + return value; +} + +export function verifyWorkspace( + database: DatabaseSync, + dofs: CloudflareDatabase, + path: string, +): void { + const stateCount = count(database, "workspace_state"); + if (stateCount !== 1) { + corrupt(path, "it does not hold exactly one Workspace current-root pointer"); + } + for (const row of database + .prepare("SELECT root_id, format_version, manifest FROM workspace_roots") + .all()) { + const root = parseStoredRoot(database, row, path); + requireReferenceSet( + database, + root.rootId, + "workspace_root_manifest_refs", + "manifest_hash", + root.manifestHashes, + path, + ); + requireReferenceSet( + database, + root.rootId, + "workspace_root_blob_refs", + "blob_hash", + root.blobHashes, + path, + ); + validateRetainedContent(database, root, path); + } + const current = currentWorkspaceRoot(database, path); + const live = snapshotWorkspace(database, dofs, path, false); + if (live.rootId !== current) { + corrupt(path, "its live Workspace frontier does not equal its current root"); + } +} + +export function materializeWorkspaceRoot( + database: DatabaseSync, + dofs: CloudflareDatabase, + savepoints: SavepointManager, + path: string, + rootId: string, +): void { + savepoints.synchronous(() => { + const row = database + .prepare("SELECT root_id, format_version, manifest FROM workspace_roots WHERE root_id = ?") + .get(rootId); + if (row === undefined) { + throw new Error(`no retained Workspace root exists under ${rootId}`); + } + const root = parseStoredRoot(database, row, path); + validateRetainedContent(database, root, path); + rebuild(database, root, path); + const restored = snapshotWorkspace(database, dofs, path, false); + if (restored.rootId !== root.rootId) { + corrupt(path, "a retained Workspace root did not materialize to its own identity"); + } + setCurrentWorkspaceRoot(database, root.rootId, path); + }); +} + +function rebuild(database: DatabaseSync, root: StoredWorkspaceRoot, path: string): void { + const parsed = parseManifest(root.manifest, path); + database.exec("DELETE FROM vfs_dirents"); + database.exec("DELETE FROM vfs_chunks"); + database.exec("DELETE FROM vfs_changes"); + database.exec("DELETE FROM vfs_nodes"); + + const currentRev = scalarInteger(database, "SELECT v FROM vfs_meta WHERE k = 'rev'", path); + const rev = currentRev + 1; + database.prepare("UPDATE vfs_meta SET v = ? WHERE k = 'rev'").run(rev); + const rootEntry = parsed.entries[0]; + if (rootEntry === undefined || rootEntry.kind !== "directory" || rootEntry.path !== "/") { + corrupt(path, "a retained Workspace root has no root directory"); + } + database + .prepare( + "INSERT INTO vfs_nodes (inode, type, mode, mtime, rev, size) VALUES (1, 'dir', ?, ?, ?, 0)", + ) + .run(rootEntry.mode, rootEntry.mtime, rev); + + const inodes = new Map([["/", 1]]); + const hardlinks = new Map(); + for (const entry of parsed.entries.slice(1).sort(parentFirst)) { + const parent = parentPath(entry.path); + const parentInode = inodes.get(parent); + if (parentInode === undefined) { + corrupt(path, "a retained Workspace root names a child before a valid parent"); + } + const name = entry.path.slice(parent === "/" ? 1 : parent.length + 1); + let inode: number; + if (entry.kind === "file" && entry.hardlink !== null && hardlinks.has(entry.hardlink)) { + const linked = hardlinks.get(entry.hardlink); + if (linked === undefined) { + corrupt(path, "a retained Workspace root has an invalid hardlink group"); + } + inode = linked; + } else { + inode = insertNode(database, entry, rev, path); + if (entry.kind === "file" && entry.hardlink !== null) { + hardlinks.set(entry.hardlink, inode); + } + } + database + .prepare("INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)") + .run(parentInode, name, inode); + inodes.set(entry.path, inode); + } +} + +function insertNode( + database: DatabaseSync, + entry: WorkspaceRootEntry, + rev: number, + path: string, +): number { + if (entry.kind === "directory") { + const result = database + .prepare("INSERT INTO vfs_nodes (type, mode, mtime, rev, size) VALUES ('dir', ?, ?, ?, 0)") + .run(entry.mode, entry.mtime, rev); + return Number(result.lastInsertRowid); + } + if (entry.kind === "symlink") { + const result = database + .prepare( + "INSERT INTO vfs_nodes (type, mode, mtime, rev, link_target, size) VALUES ('symlink', ?, ?, ?, ?, 0)", + ) + .run(entry.mode, entry.mtime, rev, entry.target); + return Number(result.lastInsertRowid); + } + const manifest = readDofsManifest(database, entry.manifest, path); + const result = database + .prepare( + "INSERT INTO vfs_nodes (type, mode, mtime, rev, manifest_hash, size) VALUES ('file', ?, ?, ?, ?, ?)", + ) + .run(entry.mode, entry.mtime, rev, fromHex(entry.manifest), entry.size); + const inode = Number(result.lastInsertRowid); + for (const [index, chunk] of manifest.chunks.entries()) { + database + .prepare("INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)") + .run(inode, index, fromHex(chunk.hash), chunk.size); + } + return inode; +} + +function parseStoredRoot( + database: DatabaseSync, + row: Record, + path: string, +): StoredWorkspaceRoot { + const rootId = row["root_id"]; + const format = integer(row["format_version"], path, "Workspace root format"); + const manifest = row["manifest"]; + if ( + typeof rootId !== "string" || + !SHA256.test(rootId) || + format !== WORKSPACE_ROOT_FORMAT || + typeof manifest !== "string" + ) { + corrupt(path, "one of its retained Workspace roots is malformed"); + } + const parsed = parseManifest(manifest, path); + const canonical = JSON.stringify(parsed); + if (canonical !== manifest) { + corrupt(path, "one of its retained Workspace roots is not canonically encoded"); + } + const manifests = new Set(); + for (const entry of parsed.entries) { + if (entry.kind === "file") { + manifests.add(entry.manifest); + } + } + const blobs = new Set(); + for (const hash of manifests) { + for (const chunk of readDofsManifest(database, hash, path).chunks) { + blobs.add(chunk.hash); + } + } + const root = workspaceRoot(manifest, [...manifests].sort(), [...blobs].sort()); + if (root.rootId !== rootId) { + corrupt(path, "one of its retained Workspace root identities does not match its bytes"); + } + return root; +} + +function parseManifest(manifest: string, path: string): WorkspaceRootManifest { + let offered: unknown; + try { + offered = JSON.parse(manifest); + } catch { + corrupt(path, "one of its retained Workspace roots is not JSON"); + } + const parsed = rootManifestSchema.safeParse(offered); + if (!parsed.success) { + corrupt(path, "one of its retained Workspace roots has an invalid manifest"); + } + validateEntryOrder(parsed.data.entries, path); + return parsed.data; +} + +function validateEntryOrder(entries: WorkspaceRootEntry[], path: string): void { + if (entries.length === 0 || entries[0]?.path !== "/" || entries[0]?.kind !== "directory") { + corrupt(path, "a Workspace root does not begin with its root directory"); + } + let previous: string | undefined; + const hardlinks = new Map(); + const hardlinkEntries = new Map(); + const directories = new Set(); + let nextHardlink = 0; + for (const entry of entries) { + validateCanonicalPath(entry.path, path); + if (previous !== undefined && compareUtf8(previous, entry.path) >= 0) { + corrupt(path, "a Workspace root's paths are duplicated or out of canonical order"); + } + previous = entry.path; + if (entry.path !== "/" && !directories.has(parentPath(entry.path))) { + corrupt(path, "a Workspace root contains an entry without a retained parent directory"); + } + if (entry.kind === "directory") { + directories.add(entry.path); + } + if ( + entry.kind === "symlink" && + (entry.target.includes("\0") || hasUnpairedSurrogate(entry.target)) + ) { + corrupt(path, "a Workspace root contains an invalid symbolic-link target"); + } + if (entry.kind === "file" && entry.hardlink !== null) { + const first = hardlinkEntries.get(entry.hardlink); + if (first === undefined) { + if (entry.hardlink !== `h${nextHardlink}`) { + corrupt(path, "a Workspace root's hardlink groups are not canonically numbered"); + } + nextHardlink += 1; + hardlinkEntries.set(entry.hardlink, entry); + } else if ( + first.mode !== entry.mode || + first.mtime !== entry.mtime || + first.size !== entry.size || + first.manifest !== entry.manifest + ) { + corrupt(path, "a Workspace root's hardlink group describes different inode properties"); + } + hardlinks.set(entry.hardlink, (hardlinks.get(entry.hardlink) ?? 0) + 1); + } + } + for (const count of hardlinks.values()) { + if (count < 2) { + corrupt(path, "a Workspace root contains a one-member hardlink group"); + } + } +} + +function validateFile( + database: DatabaseSync, + dofs: CloudflareDatabase, + node: NodeRow, + path: string, + repair: boolean, +): { manifest: string; blobs: string[] } { + const chunks = readChunks(database, node.inode, path); + const total = chunks.reduce((sum, chunk) => sum + chunk.size, 0); + if (total !== node.size) { + corrupt(path, "a Workspace file size does not equal its ordered chunks"); + } + const blobs = chunks.map((chunk) => validateBlob(database, chunk, path)); + let manifestHash = node.manifestHash; + if (manifestHash === null) { + if (!repair) { + corrupt(path, "a Workspace file has no retained DOFS manifest"); + } + manifestHash = buildManifest(dofs, chunks, 0); + database + .prepare("UPDATE vfs_nodes SET manifest_hash = ? WHERE inode = ?") + .run(manifestHash, node.inode); + } + const manifest = toHex(manifestHash); + const encoded = readDofsManifest(database, manifest, path); + if ( + encoded.size !== node.size || + JSON.stringify(encoded.chunks) !== + JSON.stringify(chunks.map((chunk) => ({ hash: toHex(chunk.hash), size: chunk.size }))) + ) { + corrupt(path, "a Workspace file's DOFS manifest does not equal its chunks"); + } + return { manifest, blobs }; +} + +function readDofsManifest( + database: DatabaseSync, + hash: string, + path: string, +): { size: number; chunks: Array<{ hash: string; size: number }> } { + const row = database + .prepare("SELECT size, encoded FROM vfs_manifests WHERE hash = ?") + .get(fromHex(hash)); + if (row === undefined) { + corrupt(path, "a retained Workspace root names a missing DOFS manifest"); + } + const size = integer(row["size"], path, "DOFS manifest size"); + const encoded = bytes(row["encoded"], path, "DOFS manifest encoding"); + if (toHex(sha256(encoded)) !== hash) { + corrupt(path, "a DOFS manifest hash does not match its bytes"); + } + let offered: unknown; + try { + offered = JSON.parse(decoder.decode(encoded)); + } catch { + corrupt(path, "a DOFS manifest is not canonical UTF-8 JSON"); + } + const parsed = dofsManifestSchema.safeParse(offered); + if (!parsed.success || JSON.stringify(parsed.data) !== decoder.decode(encoded)) { + corrupt(path, "a DOFS manifest is not canonically encoded"); + } + const total = parsed.data.chunks.reduce((sum, chunk) => sum + chunk.size, 0); + if (total !== size) { + corrupt(path, "a DOFS manifest size does not equal its chunks"); + } + return { size, chunks: parsed.data.chunks }; +} + +function validateRetainedContent( + database: DatabaseSync, + root: StoredWorkspaceRoot, + path: string, +): void { + for (const manifest of root.manifestHashes) { + const parsed = readDofsManifest(database, manifest, path); + for (const chunk of parsed.chunks) { + validateBlob(database, { hash: fromHex(chunk.hash), size: chunk.size }, path); + } + } +} + +function validateBlob(database: DatabaseSync, chunk: Chunk, path: string): string { + const hash = toHex(chunk.hash); + const row = database + .prepare( + "SELECT b.size, x.bytes FROM vfs_blobs b JOIN vfs_blob_bytes x ON x.hash = b.hash WHERE b.hash = ?", + ) + .get(chunk.hash); + if (row === undefined) { + corrupt(path, "a Workspace file names missing DOFS blob bytes"); + } + const size = integer(row["size"], path, "DOFS blob size"); + const content = bytes(row["bytes"], path, "DOFS blob bytes"); + if (size !== chunk.size || content.byteLength !== chunk.size || toHex(sha256(content)) !== hash) { + corrupt(path, "a DOFS blob's hash or size does not match its bytes"); + } + return hash; +} + +function readNode(database: DatabaseSync, inode: number, path: string): NodeRow { + const row = database + .prepare( + "SELECT inode, type, mode, mtime, manifest_hash, link_target, size FROM vfs_nodes WHERE inode = ?", + ) + .get(inode); + if (row === undefined) { + corrupt(path, "its live Workspace contains a dangling directory entry"); + } + const type = row["type"]; + if (type !== "file" && type !== "dir" && type !== "symlink") { + corrupt(path, "its live Workspace contains an unknown node type"); + } + const manifest = row["manifest_hash"]; + if (manifest !== null && !(manifest instanceof Uint8Array)) { + corrupt(path, "its live Workspace contains an invalid manifest hash"); + } + const target = row["link_target"]; + if (target !== null && typeof target !== "string") { + corrupt(path, "its live Workspace contains an invalid link target"); + } + const result: NodeRow = { + inode: integer(row["inode"], path, "Workspace inode"), + type, + mode: mode(row["mode"], path), + mtime: integer(row["mtime"], path, "Workspace mtime"), + manifestHash: manifest, + linkTarget: target, + size: nonnegative(row["size"], path, "Workspace size"), + }; + if ( + result.type === "dir" && + (result.manifestHash !== null || result.linkTarget !== null || result.size !== 0) + ) { + corrupt(path, "a Workspace directory carries file or symbolic-link metadata"); + } + if ( + result.type === "symlink" && + (result.manifestHash !== null || result.linkTarget === null || result.size !== 0) + ) { + corrupt(path, "a Workspace symbolic link carries inconsistent metadata"); + } + if (result.type === "file" && result.linkTarget !== null) { + corrupt(path, "a Workspace file carries a symbolic-link target"); + } + return result; +} + +function readDirents( + database: DatabaseSync, + inode: number, + path: string, +): Array<{ name: string; inode: number }> { + const entries: Array<{ name: string; inode: number }> = []; + for (const row of database + .prepare("SELECT name, child_inode FROM vfs_dirents WHERE parent_inode = ?") + .all(inode)) { + const name = row["name"]; + if (typeof name !== "string") { + corrupt(path, "its live Workspace contains an invalid directory-entry name"); + } + entries.push({ name, inode: integer(row["child_inode"], path, "Workspace child inode") }); + } + return entries.sort((left, right) => compareUtf8(left.name, right.name)); +} + +function readChunks(database: DatabaseSync, inode: number, path: string): Chunk[] { + const chunks: Chunk[] = []; + for (const [expected, row] of database + .prepare("SELECT idx, hash, size FROM vfs_chunks WHERE inode = ? ORDER BY idx") + .all(inode) + .entries()) { + const index = integer(row["idx"], path, "Workspace chunk index"); + const hash = bytes(row["hash"], path, "Workspace chunk hash"); + const size = integer(row["size"], path, "Workspace chunk size"); + if (index !== expected || hash.byteLength !== 32 || size <= 0) { + corrupt(path, "a Workspace file has malformed or unordered chunks"); + } + chunks.push({ hash, size }); + } + return chunks; +} + +function requireReferenceSet( + database: DatabaseSync, + rootId: string, + table: string, + column: string, + expected: readonly string[], + path: string, +): void { + const actual = database + .prepare(`SELECT ${column} FROM ${table} WHERE root_id = ?`) + .all(rootId) + .map((row) => toHex(bytes(row[column], rootId, `${table}.${column}`))) + .sort(); + if (JSON.stringify(actual) !== JSON.stringify([...expected].sort())) { + corrupt(path, `a retained Workspace root has an inexact ${table} reference set`); + } +} + +function validateCanonicalPath(value: string, databasePath: string): void { + if (value === "/") { + return; + } + if ( + !value.startsWith("/") || + value.endsWith("/") || + value.includes("\0") || + hasUnpairedSurrogate(value) + ) { + corrupt(databasePath, "a Workspace root contains a noncanonical path"); + } + for (const part of value.slice(1).split("/")) { + if (part === "" || part === "." || part === "..") { + corrupt(databasePath, "a Workspace root contains a noncanonical path component"); + } + } +} + +function validateName(name: string, path: string): void { + if ( + name === "" || + name === "." || + name === ".." || + name.includes("/") || + name.includes("\0") || + hasUnpairedSurrogate(name) + ) { + corrupt(path, "its live Workspace contains a noncanonical name"); + } +} + +function hasUnpairedSurrogate(value: string): boolean { + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index); + if (code >= 0xd800 && code <= 0xdbff) { + const next = value.charCodeAt(index + 1); + if (next < 0xdc00 || next > 0xdfff) { + return true; + } + index += 1; + } else if (code >= 0xdc00 && code <= 0xdfff) { + return true; + } + } + return false; +} + +function compareUtf8(left: string, right: string): number { + return Buffer.compare(encoder.encode(left), encoder.encode(right)); +} + +function parentFirst(left: WorkspaceRootEntry, right: WorkspaceRootEntry): number { + const depth = left.path.split("/").length - right.path.split("/").length; + return depth === 0 ? compareUtf8(left.path, right.path) : depth; +} + +function parentPath(path: string): string { + const boundary = path.lastIndexOf("/"); + return boundary === 0 ? "/" : path.slice(0, boundary); +} + +function count(database: DatabaseSync, table: string): number { + const row = database.prepare(`SELECT COUNT(*) AS count FROM ${table}`).get(); + const value = row?.["count"]; + return typeof value === "bigint" ? Number(value) : typeof value === "number" ? value : -1; +} + +function scalarInteger(database: DatabaseSync, sql: string, path: string): number { + const row = database.prepare(sql).get(); + return integer(row?.["v"], path, "Workspace revision"); +} + +function mode(value: unknown, path: string): number { + const parsed = integer(value, path, "Workspace mode"); + if (parsed < 0 || parsed > 0o7777) { + corrupt(path, "a Workspace node has an invalid mode"); + } + return parsed; +} + +function nonnegative(value: unknown, path: string, label: string): number { + const parsed = integer(value, path, label); + if (parsed < 0) { + corrupt(path, `${label} is negative`); + } + return parsed; +} + +function integer(value: unknown, path: string, label: string): number { + const parsed = typeof value === "bigint" ? Number(value) : value; + if (typeof parsed !== "number" || !Number.isSafeInteger(parsed)) { + corrupt(path, `${label} is not a safe integer`); + } + return parsed; +} + +function bytes(value: unknown, path: string, label: string): Uint8Array { + if (!(value instanceof Uint8Array)) { + corrupt(path, `${label} is not bytes`); + } + return value; +} + +function sha256(value: Uint8Array): Uint8Array { + return new Uint8Array(createHash("sha256").update(value).digest()); +} + +function toHex(value: Uint8Array): string { + return Buffer.from(value).toString("hex"); +} + +function fromHex(value: string): Uint8Array { + return new Uint8Array(Buffer.from(value, "hex")); +} + +function corrupt(path: string, reason: string): never { + throw new WorkflowDatabaseCorruptError(path, reason); +} diff --git a/packages/workflow/src/storage/api.ts b/packages/workflow/src/storage/api.ts index f5c536b2..1e78a981 100644 --- a/packages/workflow/src/storage/api.ts +++ b/packages/workflow/src/storage/api.ts @@ -25,8 +25,8 @@ * ## Lifetime * * A handle is owned by the scope that asked for it. When that scope ends the - * connection closes, and every later call on the handle fails rather than - * reopening anything behind the caller's back. + * lease closes, and every later call on the handle fails. A provider may keep + * one authoritative physical connection for its own longer-lived scope. */ import { type Api, createApi } from "@effectionx/context-api"; @@ -42,6 +42,7 @@ import type { StoredRunState, WorkflowRunRecord, } from "./record.ts"; +import type { WorkflowWorkspace } from "../workspace/api.ts"; /** What a caller must decide before a run can exist. */ export interface CreateWorkflowRunRequest { @@ -107,6 +108,9 @@ export interface WorkflowRunDatabase { /** The run's filtered journal. An append here commits on its own. */ readonly journal: DurableStream; + /** The retained provider-level Workspace transaction foundation. */ + readonly workspace: WorkflowWorkspace; + /** Every retained event with its opaque id, in append order. */ readJournalEntries(): Operation>; diff --git a/packages/workflow/src/storage/errors.ts b/packages/workflow/src/storage/errors.ts index 7f919183..ceaabdc3 100644 --- a/packages/workflow/src/storage/errors.ts +++ b/packages/workflow/src/storage/errors.ts @@ -124,6 +124,19 @@ export class WorkflowDatabaseCorruptError extends WorkflowStorageError { } } +/** The unsupported pre-release schema that claimed the now-complete version 1. */ +export class WorkflowIncompleteVersionOneError extends WorkflowDatabaseCorruptError { + override name = "WorkflowIncompleteVersionOneError"; + + constructor(path: string) { + super(path, "it contains the incomplete pre-release version-1 structure"); + this.message = + `The workflow-run database at ${path} contains the unsupported incomplete ` + + "pre-release version-1 structure. It is not migrated or changed. Delete and recreate " + + "this pre-release database with the complete version-1 provider."; + } +} + /** A stored row does not describe what its column claims to hold. */ export class WorkflowRecordMalformedError extends WorkflowStorageError { override name = "WorkflowRecordMalformedError"; diff --git a/packages/workflow/src/workspace/api.ts b/packages/workflow/src/workspace/api.ts new file mode 100644 index 00000000..1363a081 --- /dev/null +++ b/packages/workflow/src/workspace/api.ts @@ -0,0 +1,49 @@ +import type { DurableEffect, EffectDescription, Json } from "@executablemd/durable-streams"; +import type { Operation, Result } from "effection"; + +export interface WorkspaceStat { + readonly mode: number; + readonly mtime: number; + readonly size: number; + readonly kind: "file" | "directory" | "symlink"; +} + +export interface WorkspaceDirectoryEntry { + readonly name: string; + readonly kind: "file" | "directory" | "symlink"; +} + +export interface WorkspaceFilesystem { + readFile(path: string): Operation; + readTextFile(path: string): Operation; + stat(path: string): Operation; + lstat(path: string): Operation; + readlink(path: string): Operation; + readdir(path: string): Operation; + writeFile(path: string, content: string | Uint8Array, mode?: number): Operation; + mkdir( + path: string, + options?: { readonly recursive?: boolean; readonly mode?: number }, + ): Operation; + remove(path: string, options?: { readonly recursive?: boolean }): Operation; + rename(from: string, to: string): Operation; + chmod(path: string, mode: number): Operation; + symlink(target: string, path: string): Operation; + link(existingPath: string, newPath: string): Operation; +} + +export interface WorkflowWorkspace { + /** The immutable root currently published for this run. */ + currentRoot(): Operation>; + + /** + * One provider-level durable Workspace operation. + * + * This is the retained transaction foundation used by later public + * Workspace effects. It is not a public file component or history command. + */ + effect( + description: EffectDescription, + mutation: (filesystem: WorkspaceFilesystem) => Operation, + ): DurableEffect; +} diff --git a/packages/workflow/tests/support/workspace-crash-child.ts b/packages/workflow/tests/support/workspace-crash-child.ts new file mode 100644 index 00000000..8c1d75be --- /dev/null +++ b/packages/workflow/tests/support/workspace-crash-child.ts @@ -0,0 +1,43 @@ +import process from "node:process"; +import { durableRun, type Json, type Workflow } from "@executablemd/durable-streams"; +import { main, type Operation, suspend } from "effection"; +import { WorkflowRunStorage } from "../../mod.ts"; +import { useWorkflowRunStorage } from "../../deno.ts"; + +const DEFINITION = { + version: 1, + kind: "git", + objectFormat: "sha1", + objectId: "9fceb02d0ae598e95dc970b74767f19372d61af8", + rootDocumentPath: "workflows/release.md", +} as const; + +main(function* () { + const [root] = process.argv.slice(2); + yield* useWorkflowRunStorage({ root }); + const opened = yield* WorkflowRunStorage.operations.create({ + runId: "release-1.4", + definition: DEFINITION, + base: "main", + props: { channel: "stable" }, + }); + if (!opened.ok) { + throw opened.error; + } + const database = opened.value; + + function* work(): Workflow { + yield database.workspace.effect( + { type: "workspace", name: "killed" }, + function* (filesystem): Operation { + yield* filesystem.writeFile("/uncommitted.txt", "killed before publication"); + console.log("XMD_UNCOMMITTED_WORKSPACE_WRITE"); + yield* suspend(); + return null; + }, + ); + return null; + } + + yield* durableRun(work, { stream: database.journal }); +}); diff --git a/packages/workflow/tests/support/workspace-restart-child.ts b/packages/workflow/tests/support/workspace-restart-child.ts new file mode 100644 index 00000000..08039eea --- /dev/null +++ b/packages/workflow/tests/support/workspace-restart-child.ts @@ -0,0 +1,31 @@ +import process from "node:process"; +import { main } from "effection"; +import { WorkflowRunStorage } from "../../mod.ts"; +import { useWorkflowRunStorage } from "../../deno.ts"; + +main(function* () { + const [root] = process.argv.slice(2); + yield* useWorkflowRunStorage({ root }); + const opened = yield* WorkflowRunStorage.operations.lookup("release-1.4"); + if (!opened.ok) { + throw opened.error; + } + const database = opened.value; + const events = yield* database.readJournalEntries(); + if (!events.ok) { + throw events.error; + } + const rootId = yield* database.workspace.currentRoot(); + if (!rootId.ok) { + throw rootId.error; + } + + console.log( + JSON.stringify({ + rootId: rootId.value, + events: events.value.map((entry) => + entry.event.type === "yield" ? entry.event.description.name : "close", + ), + }), + ); +}); diff --git a/packages/workflow/tests/workflow-run-storage.test.ts b/packages/workflow/tests/workflow-run-storage.test.ts index f96e13d6..38ff3a47 100644 --- a/packages/workflow/tests/workflow-run-storage.test.ts +++ b/packages/workflow/tests/workflow-run-storage.test.ts @@ -780,6 +780,35 @@ describe("Tier WS — refusing what is not this run's database", () => { } }); + it("WS22b: the intermediate metadata-only version 1 is refused byte-for-byte", function* () { + const root = yield* useStorageRoot(); + const path = runPath(root, "release-1.4"); + tamper(path, (database) => { + database.exec(` + PRAGMA application_id = ${APPLICATION_ID}; + PRAGMA user_version = 1; + CREATE TABLE journal_events ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + event_id TEXT NOT NULL UNIQUE, + record TEXT NOT NULL CHECK (json_valid(record)) + ) STRICT; + CREATE TABLE workflow_run (id INTEGER PRIMARY KEY) STRICT; + CREATE TABLE definition_retrieval (id INTEGER PRIMARY KEY) STRICT; + CREATE TABLE document_executions (sequence INTEGER PRIMARY KEY AUTOINCREMENT) STRICT; + `); + }); + const before = readFileSync(path); + + const result = yield* withStorage(root, function* () { + return yield* lookup("release-1.4"); + }); + + expect(result.ok).toBe(false); + expect(!result.ok && result.error).toBeInstanceOf(WorkflowDatabaseCorruptError); + expect(!result.ok && result.error.message).toContain("Delete and recreate"); + expect(readFileSync(path)).toEqual(before); + }); + it("WS23: a stored descriptor that describes no definition is refused", function* () { const root = yield* useStorageRoot(); const path = runPath(root, "release-1.4"); @@ -992,7 +1021,10 @@ describe("Tier WS — refusing what is not this run's database", () => { tamper(path, (database) => { for (let index = 0; index < 400; index++) { database - .prepare("INSERT INTO journal_events (event_id, record) VALUES (?, ?)") + .prepare( + `INSERT INTO journal_events (event_id, record, workspace_root_id) + SELECT ?, ?, current_root_id FROM workspace_state WHERE singleton_id = 1`, + ) .run(`e${index}`, JSON.stringify({ padding: "x".repeat(200), index })); } }); diff --git a/packages/workflow/tests/workspace-filesystem.test.ts b/packages/workflow/tests/workspace-filesystem.test.ts new file mode 100644 index 00000000..99052328 --- /dev/null +++ b/packages/workflow/tests/workspace-filesystem.test.ts @@ -0,0 +1,820 @@ +import { DatabaseSync } from "node:sqlite"; +import { readFileSync } from "node:fs"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; +import { describe, it } from "@executablemd/test-support/bdd"; +import { expect } from "@executablemd/test-support/expect"; +import { + type DurableEvent, + type DurableStream, + durableRun, + guardDurableStream, + type Json, + type Workflow, +} from "@executablemd/durable-streams"; +import { ensure, type Operation, scoped, spawn, suspend, withResolvers } from "effection"; +import { exec } from "@effectionx/process"; +import { exec as runProcess } from "@executablemd/runtime"; +import { + WorkflowDatabaseCorruptError, + WorkflowRunStorage, + WorkflowTransactionError, +} from "../mod.ts"; +import { + allowJournalInserts, + createRun, + refuseJournalInsertNamed, + runPath, + tamper, + useStorageRoot, + withStorage, +} from "./support/storage.ts"; +import { createWorkflowRunConnections } from "../src/deno/connections.ts"; +import { createWorkspaceFilesystem } from "../src/deno/workspace/filesystem.ts"; +import { materializeWorkspaceRoot } from "../src/deno/workspace/root.ts"; +import { + type JournalDestination, + routeJournalAppend, + useJournalDestination, +} from "../src/deno/journal-route.ts"; + +const REPOSITORY = fileURLToPath(new URL("../../../", import.meta.url)); +const CRASH_CHILD = fileURLToPath(new URL("./support/workspace-crash-child.ts", import.meta.url)); +const RESTART_CHILD = fileURLToPath( + new URL("./support/workspace-restart-child.ts", import.meta.url), +); + +describe("Tier WW — retained provider Workspace", () => { + it("WW1: a successful mutation publishes the filesystem, root, pointer, and result", function* () { + const storage = yield* useStorageRoot(); + + yield* withStorage(storage, function* () { + const database = yield* createRun(); + const before = yield* database.workspace.currentRoot(); + if (!before.ok) { + throw before.error; + } + + function* work(): Workflow { + yield database.workspace.effect( + { type: "workspace", name: "create" }, + function* (filesystem): Operation { + yield* filesystem.mkdir("/notes", { mode: 0o750 }); + yield* filesystem.writeFile("/notes/release.txt", "ready", 0o640); + return { written: true }; + }, + ); + return { written: true }; + } + + expect(yield* durableRun(work, { stream: database.journal })).toEqual({ written: true }); + const after = yield* database.workspace.currentRoot(); + if (!after.ok) { + throw after.error; + } + expect(after.value).not.toBe(before.value); + }); + + const sqlite = new DatabaseSync(runPath(storage, "release-1.4")); + try { + const state = sqlite.prepare("SELECT current_root_id FROM workspace_state").get(); + const event = sqlite + .prepare( + 'SELECT workspace_root_id, record FROM journal_events WHERE record LIKE \'%"name":"create"%\'', + ) + .get(); + expect(event?.["workspace_root_id"]).toBe(state?.["current_root_id"]); + expect(sqlite.prepare("SELECT COUNT(*) AS total FROM workspace_roots").get()?.["total"]).toBe( + 2, + ); + expect(sqlite.prepare("SELECT COUNT(*) AS total FROM vfs_manifests").get()?.["total"]).toBe( + 1, + ); + expect( + sqlite.prepare("SELECT COUNT(*) AS total FROM workspace_root_blob_refs").get()?.["total"], + ).toBe(1); + } finally { + sqlite.close(); + } + }); + + it("WW2: a known mutation failure retains the previous root and one failed Yield", function* () { + const storage = yield* useStorageRoot(); + + yield* withStorage(storage, function* () { + const database = yield* createRun(); + const before = yield* database.workspace.currentRoot(); + if (!before.ok) { + throw before.error; + } + + function* work(): Workflow { + yield database.workspace.effect( + { type: "workspace", name: "missing" }, + function* (filesystem): Operation { + yield* filesystem.writeFile("/missing/file.txt", "no parent"); + return null; + }, + ); + return null; + } + + let failure: unknown; + try { + yield* durableRun(work, { stream: database.journal }); + } catch (error) { + failure = error; + } + expect(failure).toBeInstanceOf(Error); + + const after = yield* database.workspace.currentRoot(); + if (!after.ok) { + throw after.error; + } + expect(after.value).toBe(before.value); + const events = yield* database.journal.readAll(); + const failed = events.filter( + (event) => event.type === "yield" && event.description.name === "missing", + ); + expect(failed).toHaveLength(1); + expect(failed[0]?.type === "yield" ? failed[0].result.status : undefined).toBe("err"); + }); + }); + + it("WW3: an older root reconstructs topology, metadata, links, and retained bytes", function* () { + const storage = yield* useStorageRoot(); + const path = runPath(storage, "release-1.4"); + let historical = ""; + + yield* withStorage(storage, function* () { + const database = yield* createRun(); + + function* changes(): Workflow { + yield database.workspace.effect( + { type: "workspace", name: "first-root" }, + function* (filesystem): Operation { + yield* filesystem.mkdir("/a"); + yield* filesystem.writeFile("/a/x.txt", "group one"); + yield* filesystem.link("/a/x.txt", "/z-one.txt"); + yield* filesystem.writeFile("/a-.txt", "group zero"); + yield* filesystem.link("/a-.txt", "/z-zero.txt"); + yield* filesystem.mkdir("/tree", { mode: 0o750 }); + yield* filesystem.writeFile("/tree/file.txt", "first", 0o640); + yield* filesystem.link("/tree/file.txt", "/tree/hardlink.txt"); + yield* filesystem.symlink("file.txt", "/tree/current.txt"); + return null; + }, + ); + yield database.workspace.effect( + { type: "workspace", name: "later-root" }, + function* (filesystem): Operation { + yield* filesystem.writeFile("/tree/file.txt", "second"); + yield* filesystem.rename("/tree/file.txt", "/renamed.txt"); + yield* filesystem.remove("/tree/current.txt"); + yield* filesystem.mkdir("/later", { mode: 0o700 }); + yield* filesystem.chmod("/tree/hardlink.txt", 0o600); + return null; + }, + ); + return null; + } + yield* durableRun(changes, { stream: database.journal }); + }); + + const roots = new DatabaseSync(path); + try { + const first = roots + .prepare( + `SELECT e.workspace_root_id, r.manifest + FROM journal_events e + JOIN workspace_roots r ON r.root_id = e.workspace_root_id + WHERE e.record LIKE '%"name":"first-root"%'`, + ) + .get(); + const current = roots + .prepare("SELECT current_root_id FROM workspace_state WHERE singleton_id = 1") + .get(); + historical = String(first?.["workspace_root_id"]); + expect(current?.["current_root_id"]).not.toBe(historical); + const manifest = JSON.parse(String(first?.["manifest"])); + const hardlinks = Object.fromEntries( + manifest.entries + .filter((entry: { hardlink?: string | null }) => entry.hardlink !== undefined) + .map((entry: { path: string; hardlink: string | null }) => [entry.path, entry.hardlink]), + ); + expect(hardlinks).toEqual({ + "/a-.txt": "h0", + "/a/x.txt": "h1", + "/tree/file.txt": "h2", + "/tree/hardlink.txt": "h2", + "/z-one.txt": "h1", + "/z-zero.txt": "h0", + }); + } finally { + roots.close(); + } + + const connections = createWorkflowRunConnections(); + try { + const connection = connections.at(path); + connection.database.exec("BEGIN IMMEDIATE"); + connection.transactionOpen = true; + try { + materializeWorkspaceRoot( + connection.database, + connection.dofs, + connection.savepoints, + path, + historical, + ); + connection.transactionOpen = false; + connection.database.exec("COMMIT"); + } catch (error) { + connection.transactionOpen = false; + connection.database.exec("ROLLBACK"); + throw error; + } + const filesystem = createWorkspaceFilesystem(connection); + expect(yield* filesystem.readTextFile("/tree/file.txt")).toBe("first"); + expect(yield* filesystem.readTextFile("/tree/hardlink.txt")).toBe("first"); + expect((yield* filesystem.lstat("/tree/file.txt")).mode).toBe(0o640); + expect(yield* filesystem.readlink("/tree/current.txt")).toBe("file.txt"); + expect((yield* filesystem.readdir("/tree")).map((entry) => entry.name).sort()).toEqual([ + "current.txt", + "file.txt", + "hardlink.txt", + ]); + } finally { + connections.close(); + } + + yield* withStorage(storage, function* () { + const found = yield* createRun(); + const root = yield* found.workspace.currentRoot(); + if (!root.ok) { + throw root.error; + } + expect(root.value).toBe(historical); + }); + + const sqlite = new DatabaseSync(path); + try { + sqlite.exec("PRAGMA foreign_keys = ON"); + let refused: unknown; + try { + sqlite.prepare("DELETE FROM vfs_manifests").run(); + } catch (error) { + refused = error; + } + expect(refused).toBeInstanceOf(Error); + let bytesRefused: unknown; + try { + sqlite.prepare("DELETE FROM vfs_blob_bytes").run(); + } catch (error) { + bytesRefused = error; + } + expect(bytesRefused).toBeInstanceOf(Error); + } finally { + sqlite.close(); + } + }); + + it("WW4: a journal insertion failure rolls back the mutation and root publication", function* () { + const storage = yield* useStorageRoot(); + const path = runPath(storage, "release-1.4"); + + yield* withStorage(storage, function* () { + const database = yield* createRun(); + const before = yield* database.workspace.currentRoot(); + if (!before.ok) { + throw before.error; + } + refuseJournalInsertNamed(path, "refused-workspace"); + + function* work(): Workflow { + yield database.workspace.effect( + { type: "workspace", name: "refused-workspace" }, + function* (filesystem): Operation { + yield* filesystem.writeFile("/must-roll-back.txt", "uncommitted"); + return null; + }, + ); + return null; + } + + let failure: unknown; + try { + yield* durableRun(work, { stream: database.journal }); + } catch (error) { + failure = error; + } finally { + allowJournalInserts(path); + } + expect(failure).toBeInstanceOf(Error); + const after = yield* database.workspace.currentRoot(); + expect(after.ok && after.value).toBe(before.value); + const raw = new DatabaseSync(path); + try { + expect(raw.prepare("SELECT COUNT(*) AS total FROM vfs_dirents").get()?.["total"]).toBe(0); + } finally { + raw.close(); + } + }); + }); + + it("WW5: gate rejection happens before routing and rolls back everything", function* () { + const storage = yield* useStorageRoot(); + const path = runPath(storage, "release-1.4"); + + yield* withStorage(storage, function* () { + const database = yield* createRun(); + const before = yield* database.workspace.currentRoot(); + if (!before.ok) { + throw before.error; + } + const guarded = guardDurableStream(database.journal, function* () { + throw new Error("rejected before storage"); + }); + + function* work(): Workflow { + yield database.workspace.effect( + { type: "workspace", name: "secret-rejected" }, + function* (filesystem): Operation { + yield* filesystem.writeFile("/secret.txt", "never retained"); + return null; + }, + ); + return null; + } + + let failure: unknown; + try { + yield* durableRun(work, { stream: guarded }); + } catch (error) { + failure = error; + } + expect(failure).toBeInstanceOf(Error); + const after = yield* database.workspace.currentRoot(); + expect(after.ok && after.value).toBe(before.value); + expect(yield* database.journal.readAll()).toEqual([]); + const raw = new DatabaseSync(path); + try { + expect(raw.prepare("SELECT COUNT(*) AS total FROM vfs_dirents").get()?.["total"]).toBe(0); + } finally { + raw.close(); + } + }); + }); + + it("WW6: cancellation during mutation teardown publishes nothing", function* () { + const storage = yield* useStorageRoot(); + const path = runPath(storage, "release-1.4"); + + yield* withStorage(storage, function* () { + const database = yield* createRun(); + const before = yield* database.workspace.currentRoot(); + if (!before.ok) { + throw before.error; + } + const tearingDown = withResolvers(); + const finishTeardown = withResolvers(); + + function* work(): Workflow { + yield database.workspace.effect( + { type: "workspace", name: "cancelled-teardown" }, + function* (filesystem): Operation { + yield* ensure(function* () { + tearingDown.resolve(); + yield* finishTeardown.operation; + }); + yield* filesystem.writeFile("/cancelled.txt", "uncommitted"); + return null; + }, + ); + return null; + } + + const running = yield* spawn(function* () { + yield* durableRun(work, { stream: database.journal }); + }); + yield* tearingDown.operation; + const halting = yield* spawn(function* () { + yield* running.halt(); + }); + finishTeardown.resolve(); + yield* halting; + + const after = yield* database.workspace.currentRoot(); + expect(after.ok && after.value).toBe(before.value); + expect(yield* database.journal.readAll()).toEqual([]); + const raw = new DatabaseSync(path); + try { + expect(raw.prepare("SELECT COUNT(*) AS total FROM vfs_dirents").get()?.["total"]).toBe(0); + } finally { + raw.close(); + } + }); + }); + + it("WW6b: cancellation before and during mutation publishes nothing", function* () { + const storage = yield* useStorageRoot(); + + yield* withStorage(storage, function* () { + const holder = yield* createRun({ runId: "cancel-before" }); + const waiting = yield* createRun({ runId: "cancel-before" }); + const held = withResolvers(); + const release = withResolvers(); + const holding = yield* spawn(function* () { + const result = yield* holder.transact(function* () { + held.resolve(); + yield* release.operation; + }); + if (!result.ok) { + throw result.error; + } + }); + yield* held.operation; + + const attempted = withResolvers(); + function* queued(): Workflow { + yield waiting.workspace.effect( + { type: "workspace", name: "cancelled-before" }, + function* (filesystem): Operation { + yield* filesystem.writeFile("/before.txt", "must not run"); + return null; + }, + ); + return null; + } + const queuedRun = yield* spawn(function* () { + attempted.resolve(); + yield* durableRun(queued, { stream: waiting.journal }); + }); + yield* attempted.operation; + yield* queuedRun.halt(); + release.resolve(); + yield* holding; + expect(yield* waiting.journal.readAll()).toEqual([]); + + const during = yield* createRun({ runId: "cancel-during" }); + const entered = withResolvers(); + function* interrupted(): Workflow { + yield during.workspace.effect( + { type: "workspace", name: "cancelled-during" }, + function* (filesystem): Operation { + yield* filesystem.writeFile("/during.txt", "uncommitted"); + entered.resolve(); + yield* suspend(); + return null; + }, + ); + return null; + } + const running = yield* spawn(function* () { + yield* durableRun(interrupted, { stream: during.journal }); + }); + yield* entered.operation; + yield* running.halt(); + expect(yield* during.journal.readAll()).toEqual([]); + }); + + for (const runId of ["cancel-before", "cancel-during"]) { + const sqlite = new DatabaseSync(runPath(storage, runId)); + try { + expect(sqlite.prepare("SELECT COUNT(*) AS total FROM vfs_dirents").get()?.["total"]).toBe( + 0, + ); + expect( + sqlite.prepare("SELECT COUNT(*) AS total FROM workspace_roots").get()?.["total"], + ).toBe(1); + } finally { + sqlite.close(); + } + } + }); + + it("WW7: SIGKILL after a real write exposes none of the open transaction", function* () { + const storage = yield* useStorageRoot(); + const path = runPath(storage, "release-1.4"); + let baseline = ""; + + yield* withStorage(storage, function* () { + const database = yield* createRun(); + const root = yield* database.workspace.currentRoot(); + if (!root.ok) { + throw root.error; + } + baseline = root.value; + }); + + const reached = withResolvers(); + let output = ""; + const child = yield* exec(process.execPath, { + arguments: ["run", "--allow-all", "--frozen", CRASH_CHILD, storage], + cwd: REPOSITORY, + }); + yield* child.around({ + *stdout([bytes], next) { + output += new TextDecoder().decode(bytes); + if (output.includes("XMD_UNCOMMITTED_WORKSPACE_WRITE")) { + reached.resolve(); + } + return yield* next(bytes); + }, + }); + yield* reached.operation; + process.kill(child.pid, "SIGKILL"); + const status = yield* child.join(); + expect(status.signal).toBe("SIGKILL"); + + yield* withStorage(storage, function* () { + const database = yield* createRun(); + const root = yield* database.workspace.currentRoot(); + if (!root.ok) { + throw root.error; + } + expect(root.value).toBe(baseline); + expect(yield* database.journal.readAll()).toEqual([]); + }); + + const sqlite = new DatabaseSync(path); + try { + expect(sqlite.prepare("SELECT COUNT(*) AS total FROM vfs_dirents").get()?.["total"]).toBe(0); + expect(sqlite.prepare("SELECT COUNT(*) AS total FROM workspace_roots").get()?.["total"]).toBe( + 1, + ); + expect(sqlite.prepare("SELECT COUNT(*) AS total FROM journal_events").get()?.["total"]).toBe( + 0, + ); + } finally { + sqlite.close(); + } + }); + + it("WW8: serialized handles share the authoritative filesystem and its cache", function* () { + const storage = yield* useStorageRoot(); + const path = runPath(storage, "release-1.4"); + let retained = ""; + + yield* withStorage(storage, function* () { + const first = yield* createRun(); + const second = yield* createRun(); + + function* sequence(): Workflow { + try { + yield first.workspace.effect( + { type: "workspace", name: "negative-cache" }, + function* (filesystem): Operation { + yield* filesystem.readTextFile("/later.txt"); + return null; + }, + ); + } catch { + // The expected failed effect does not stop the workflow from + // exercising the next serialized provider turn. + } + yield second.workspace.effect( + { type: "workspace", name: "create-later" }, + function* (filesystem): Operation { + yield* filesystem.writeFile("/later.txt", "visible"); + return null; + }, + ); + yield first.workspace.effect( + { type: "workspace", name: "observe-later" }, + function* (filesystem): Operation { + expect(yield* filesystem.readTextFile("/later.txt")).toBe("visible"); + return null; + }, + ); + return null; + } + yield* durableRun(sequence, { stream: first.journal }); + const observed = yield* first.workspace.currentRoot(); + if (!observed.ok) { + throw observed.error; + } + retained = observed.value; + const events = yield* first.journal.readAll(); + expect( + events.flatMap((event) => (event.type === "yield" ? [event.description.name] : [])), + ).toEqual(["negative-cache", "create-later", "observe-later"]); + }); + + const sqlite = new DatabaseSync(path); + try { + const roots = sqlite.prepare("SELECT COUNT(*) AS total FROM workspace_roots").get(); + expect(roots?.["total"]).toBe(2); + const references = sqlite + .prepare( + `SELECT workspace_root_id FROM journal_events + WHERE record LIKE '%"name":"create-later"%' + OR record LIKE '%"name":"observe-later"%' + ORDER BY sequence`, + ) + .all(); + expect(references.map((row) => row["workspace_root_id"])).toEqual([retained, retained]); + } finally { + sqlite.close(); + } + }); + + it("WW9: Workspace corruption is distinguished and left byte-for-byte unchanged", function* () { + const storage = yield* useStorageRoot(); + const cases: Array<{ runId: string; damage(database: DatabaseSync): void }> = [ + { + runId: "partial-root-schema", + damage(database) { + database.exec("DROP TABLE workspace_root_blob_refs"); + }, + }, + { + runId: "changed-root-constraint", + damage(database) { + database.exec(` + ALTER TABLE workspace_state RENAME TO workspace_state_original; + CREATE TABLE workspace_state ( + singleton_id INTEGER PRIMARY KEY, + current_root_id TEXT NOT NULL + ); + INSERT INTO workspace_state SELECT * FROM workspace_state_original; + DROP TABLE workspace_state_original; + `); + }, + }, + { + runId: "malformed-root", + damage(database) { + database + .prepare("UPDATE workspace_roots SET manifest = ?") + .run('{"format":1,"entries":[]}'); + }, + }, + { + runId: "live-frontier-mismatch", + damage(database) { + database.prepare("UPDATE vfs_nodes SET mode = 448 WHERE inode = 1").run(); + }, + }, + ]; + + for (const one of cases) { + yield* withStorage(storage, function* () { + yield* createRun({ runId: one.runId }); + }); + const path = runPath(storage, one.runId); + tamper(path, one.damage); + const before = readFileSync(path); + const result = yield* withStorage(storage, function* () { + return yield* WorkflowRunStorage.operations.lookup(one.runId); + }); + expect(result.ok).toBe(false); + expect(!result.ok && result.error).toBeInstanceOf(WorkflowDatabaseCorruptError); + expect(readFileSync(path)).toEqual(before); + } + + const blobRun = "corrupt-retained-blob"; + yield* withStorage(storage, function* () { + const database = yield* createRun({ runId: blobRun }); + function* write(): Workflow { + yield database.workspace.effect( + { type: "workspace", name: "blob" }, + function* (filesystem): Operation { + yield* filesystem.writeFile("/blob.txt", "retained bytes"); + return null; + }, + ); + return null; + } + yield* durableRun(write, { stream: database.journal }); + }); + const blobPath = runPath(storage, blobRun); + tamper(blobPath, (database) => { + database.prepare("UPDATE vfs_blob_bytes SET bytes = X'00'").run(); + }); + const before = readFileSync(blobPath); + const corrupted = yield* withStorage(storage, function* () { + return yield* WorkflowRunStorage.operations.lookup(blobRun); + }); + expect(corrupted.ok).toBe(false); + expect(!corrupted.ok && corrupted.error).toBeInstanceOf(WorkflowDatabaseCorruptError); + expect(readFileSync(blobPath)).toEqual(before); + }); + + it("WW10: a second process observes the retained Workspace and ordered journal", function* () { + const storage = yield* useStorageRoot(); + let rootId = ""; + + yield* withStorage(storage, function* () { + const database = yield* createRun(); + function* write(): Workflow { + yield database.workspace.effect( + { type: "workspace", name: "first-process" }, + function* (filesystem): Operation { + yield* filesystem.writeFile("/survives.txt", "across processes"); + return null; + }, + ); + return null; + } + yield* durableRun(write, { stream: database.journal }); + const root = yield* database.workspace.currentRoot(); + if (!root.ok) { + throw root.error; + } + rootId = root.value; + }); + + const child = yield* runProcess({ + command: [process.execPath, "run", "--allow-all", "--frozen", RESTART_CHILD, storage], + cwd: REPOSITORY, + }); + expect(child.exitCode).toBe(0); + const restored = JSON.parse(child.stdout.trim()); + expect(restored.rootId).toBe(rootId); + expect(restored.events).toEqual(["first-process", "close"]); + }); + + it("WW11: explicit journal destinations fence foreign, completed, and stale handles", function* () { + const storage = yield* useStorageRoot(); + const connections = createWorkflowRunConnections(); + yield* ensure(() => { + connections.close(); + }); + const connection = connections.at(runPath(storage, "route-a")); + const foreign = connections.at(runPath(storage, "route-b")); + const event: DurableEvent = { + type: "yield", + coroutineId: "root", + description: { type: "workspace", name: "routed" }, + result: { status: "ok", value: null }, + }; + let standalone = 0; + let enlisted = 0; + const standaloneAppend = function* (): Operation { + standalone += 1; + }; + const journal: DurableStream = { + *readAll(): Operation { + return []; + }, + *append(): Operation { + enlisted += 1; + }, + }; + const active = { id: "active", connection, open: true }; + connection.transactionOpen = true; + connection.activeTransactionId = active.id; + + yield* routeJournalAppend(connection, standaloneAppend, event); + expect(standalone).toBe(1); + expect(enlisted).toBe(0); + + function destination(overrides: Partial = {}): JournalDestination { + return { + path: connection.path, + generation: connection.generation, + transaction: active, + journal, + workspaceRootId: "root", + used: false, + ...overrides, + }; + } + + function* appendThrough(offered: JournalDestination): Operation { + try { + yield* scoped(function* () { + yield* useJournalDestination(offered); + yield* routeJournalAppend(connection, standaloneAppend, event); + }); + } catch (error) { + return error; + } + return undefined; + } + + const foreignTransaction = { id: "foreign", connection: foreign, open: true }; + const completed = { id: active.id, connection, open: false }; + const fabricated = { id: "fabricated", connection, open: true }; + const refusals = [ + yield* appendThrough(destination({ path: foreign.path, transaction: foreignTransaction })), + yield* appendThrough(destination({ transaction: completed })), + yield* appendThrough(destination({ generation: "stale-generation" })), + yield* appendThrough(destination({ transaction: fabricated })), + yield* appendThrough(destination({ workspaceRootId: "" })), + ]; + for (const refusal of refusals) { + expect(refusal).toBeInstanceOf(WorkflowTransactionError); + } + expect(enlisted).toBe(0); + + const valid = destination(); + expect(yield* appendThrough(valid)).toBeUndefined(); + expect(enlisted).toBe(1); + expect(yield* appendThrough(valid)).toBeInstanceOf(WorkflowTransactionError); + expect(enlisted).toBe(1); + connection.transactionOpen = false; + connection.activeTransactionId = undefined; + }); +}); diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/LICENSE b/packages/workflow/vendor/cloudflare-computer-dofs/LICENSE new file mode 100644 index 00000000..631c4d3e --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/LICENSE @@ -0,0 +1,21 @@ +MIT License Copyright (c) 2026 Cloudflare, Inc. + +Permission is hereby granted, free of +charge, to any person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice +(including the next paragraph) shall be included in all copies or substantial +portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO +EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/MANIFEST.json b/packages/workflow/vendor/cloudflare-computer-dofs/MANIFEST.json new file mode 100644 index 00000000..fee352c2 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/MANIFEST.json @@ -0,0 +1,528 @@ +{ + "format": 1, + "repository": "https://github.com/cloudflare/computer", + "commit": "63d363632e558f7e077794988d36ed75017c2a62", + "compiler": "5.9.3", + "files": [ + { + "path": "LICENSE", + "kind": "license", + "sha256": "df3e141c129d1804bea95fe087b157b156241b3bb1a4664808630aa85f2cd8fc" + }, + { + "path": "PROVENANCE.md", + "kind": "provenance", + "sha256": "1e3edf8e2b7298a498d9aca63571a833ee6726ef8101628db4af1e05dad7b04f" + }, + { + "path": "generated/errors.d.ts", + "kind": "generated", + "sha256": "2cfec0dc6c4510425fe44923e2fe1d937b7b7a96d16dcc87147a1ed04a9e03e9" + }, + { + "path": "generated/errors.js", + "kind": "generated", + "sha256": "56d0df65a7eb776fe8d21b68d39f110ac9c09a08b06eb9289c8f7053fb242739" + }, + { + "path": "generated/fs/blobCache.d.ts", + "kind": "generated", + "sha256": "69f7400dd9d4f95384759cf4decbba95e231411a1584d0f04607c81bc8c4b982" + }, + { + "path": "generated/fs/blobCache.js", + "kind": "generated", + "sha256": "972ed59cd69a3295231f9462c93613deee91ca6a7280468697323ecf651bddd6" + }, + { + "path": "generated/fs/chmod.d.ts", + "kind": "generated", + "sha256": "74fad23eb0a4191367dc6e6d26c5374db6f0cb403c15f2dc9d1363e4b611f8fe" + }, + { + "path": "generated/fs/chmod.js", + "kind": "generated", + "sha256": "87255fcc6923bcf1355bc619885a57273c3b4278afce468b34a5605bae18b598" + }, + { + "path": "generated/fs/filesystem.d.ts", + "kind": "generated", + "sha256": "059d00fce9dd2fae304f6e3a33fef7b5adae569a5421a5ff6452be0de6a2c7d9" + }, + { + "path": "generated/fs/filesystem.js", + "kind": "generated", + "sha256": "c7a99b1e2435711abb034ad0ffd8152ebb3ef64dcd4a84285ebea34c21f79507" + }, + { + "path": "generated/fs/find.d.ts", + "kind": "generated", + "sha256": "74254d5a3489194a31ba7f2da946de6af8a2d37c1c3b97de5118b744b733dd0c" + }, + { + "path": "generated/fs/find.js", + "kind": "generated", + "sha256": "e0ec39aae4e8e137fd9c1d1365caae5660f4f564b7ff3a45bccdc548e0a03831" + }, + { + "path": "generated/fs/grep.d.ts", + "kind": "generated", + "sha256": "7a5a496644e2543c7bd9c9906e95a98c5b643c279852d2c34625809346c9ec19" + }, + { + "path": "generated/fs/grep.js", + "kind": "generated", + "sha256": "92d88446365d91ab3c9143af05ef417a865ab215eb6c8aaeb7576a913cde65fc" + }, + { + "path": "generated/fs/link.d.ts", + "kind": "generated", + "sha256": "c3fb357f5cdaf67ec545671620b19001d9f70c84f0f702f7199a16c6e637245f" + }, + { + "path": "generated/fs/link.js", + "kind": "generated", + "sha256": "c689acd26192bd436729c9bb88b1e2f49a432cdc90c3949911156cf30177efb8" + }, + { + "path": "generated/fs/ls.d.ts", + "kind": "generated", + "sha256": "e22c4287406380bc6444c1e8558f5de611037576f945f681858b670626dfc60d" + }, + { + "path": "generated/fs/ls.js", + "kind": "generated", + "sha256": "3cfea818021121d63141685cd0b5cfd600bc172f7d25b035f5a73f4d37bcdf19" + }, + { + "path": "generated/fs/mkdir.d.ts", + "kind": "generated", + "sha256": "4d7842dd9a41c60917128f964ddac12787d049ff3ce80f2f13afc78bd6cba131" + }, + { + "path": "generated/fs/mkdir.js", + "kind": "generated", + "sha256": "cd8f3751cea94dd882dffdcd94833a77939a21cb0116e3e2a807af90c5ef011f" + }, + { + "path": "generated/fs/mount-guard.d.ts", + "kind": "generated", + "sha256": "6e1c8cbd49d2abe8b74790161cbbee516b0ca433a5f636524c16f80a31464c8d" + }, + { + "path": "generated/fs/mount-guard.js", + "kind": "generated", + "sha256": "6cba2523fa03251885e6dc9fa5db494b2491f01693687d1618cec1d86f558f9b" + }, + { + "path": "generated/fs/readFile.d.ts", + "kind": "generated", + "sha256": "593fa6f651437535671553cfd5a646f63b2dc22fc27ba6b3c2d9dd268d518763" + }, + { + "path": "generated/fs/readFile.js", + "kind": "generated", + "sha256": "0bc3b89cc463e8fd666cd40609a617e77f9c26d473c65c8b3fdabbb410848990" + }, + { + "path": "generated/fs/readdir.d.ts", + "kind": "generated", + "sha256": "758771b5d968fff5ca6b50e91f58cf608e63c81bd3ba037cb6f1c32758d21716" + }, + { + "path": "generated/fs/readdir.js", + "kind": "generated", + "sha256": "fb6106188f30f712e261c4b2612a284d55988466cdec75a572cc5550d2092a45" + }, + { + "path": "generated/fs/readlink.d.ts", + "kind": "generated", + "sha256": "2f0e0dac2cf4b847d7c2481a949adf7f088d0d744b7d27f748316e292b79daf8" + }, + { + "path": "generated/fs/readlink.js", + "kind": "generated", + "sha256": "89ef4a6c48507c82340b2226523170ee07e2051de27bafad43b098974db9e50e" + }, + { + "path": "generated/fs/rename.d.ts", + "kind": "generated", + "sha256": "f20599e49c218ebe145a9b75ebca382d62ccbfcfd14a2e18236cea4d451df244" + }, + { + "path": "generated/fs/rename.js", + "kind": "generated", + "sha256": "3c83fcbf26f620ce74bc4f6a29403d8c3cd45403beb28b7743084ce6d188d7f9" + }, + { + "path": "generated/fs/resolve.d.ts", + "kind": "generated", + "sha256": "45c807995e7671c9a4cae007ac292bab04346f6a0030ff098cc6e68e80109295" + }, + { + "path": "generated/fs/resolve.js", + "kind": "generated", + "sha256": "4b55e7bf7984310f4f5c41ffed81624f4bd5a1b40e336465b6d09afb1c1a6ebd" + }, + { + "path": "generated/fs/resolveCache.d.ts", + "kind": "generated", + "sha256": "17cd821e1361da956ab59ee3287f74d34723c99b581374398254a6b260a69fe8" + }, + { + "path": "generated/fs/resolveCache.js", + "kind": "generated", + "sha256": "900a95e7aba99f30aa55e8fcb568f35481528660eb64dc7c5ea2a240c8a1dd61" + }, + { + "path": "generated/fs/rm.d.ts", + "kind": "generated", + "sha256": "cdfa98a383c7270880fccb56f6919ab84115ca6116b4807d96f5cb8caa197419" + }, + { + "path": "generated/fs/rm.js", + "kind": "generated", + "sha256": "1708fb407e470d411e55f812c90498d7a7d52d57c992b4c965a27c7f1c45f276" + }, + { + "path": "generated/fs/stat.d.ts", + "kind": "generated", + "sha256": "0e9edc8702597c6bdb85941a797640279672d7b8d413ef736a1eafa5697919d2" + }, + { + "path": "generated/fs/stat.js", + "kind": "generated", + "sha256": "8c71946f7f6e16031b146a6b76113bd74916815b1d7e13f5f699c867754d585b" + }, + { + "path": "generated/fs/symlink.d.ts", + "kind": "generated", + "sha256": "f9c5a1f9eaf138c8e18254b98491699f78d356ffcb5c3ba22d65709186bf1776" + }, + { + "path": "generated/fs/symlink.js", + "kind": "generated", + "sha256": "9b1448280ff4f1c72192c2a104e52a0f841c14453e1092a42396320e78dfb540" + }, + { + "path": "generated/fs/unlink.d.ts", + "kind": "generated", + "sha256": "8e7992ecb0ae6510514776fdf2a8cf59927e4206f4415dbfa85f68cc9c527496" + }, + { + "path": "generated/fs/unlink.js", + "kind": "generated", + "sha256": "813e4c8909fd02c5d13d10619fad09ccc7aaa0547ec3652e676d3cd195a05ca6" + }, + { + "path": "generated/fs/writeBuffer.d.ts", + "kind": "generated", + "sha256": "be6358edae7024f7fa0041ab2e32cf19f31a0d57d179540434f0c371bcb39ce4" + }, + { + "path": "generated/fs/writeBuffer.js", + "kind": "generated", + "sha256": "afce99eacacd893eef00c2ea8b94ece20362fc8ee3e1c1b3c8a7df990a81c12f" + }, + { + "path": "generated/fs/writeFile.d.ts", + "kind": "generated", + "sha256": "6650261b452af9cd236e5228f37f7d0e97fe109ddf959e328b9534842e0e6e20" + }, + { + "path": "generated/fs/writeFile.js", + "kind": "generated", + "sha256": "c58460d023f3d2c81c67dd9690656058c9edcd0843f0e3883336600d286fa9c9" + }, + { + "path": "generated/path.d.ts", + "kind": "generated", + "sha256": "7d840de3f1de20786df8dbba448fa769433b8fc1cbfbf890b66949999054b40c" + }, + { + "path": "generated/path.js", + "kind": "generated", + "sha256": "373d244c231ad1bcbf6d2b63f8b08f0bd260bc99614aebaad148553b279a89f9" + }, + { + "path": "generated/rev.d.ts", + "kind": "generated", + "sha256": "0e8582965e59f98866745e756a6f814d25fb04d82028d6aa0bd18c00102f7b5a" + }, + { + "path": "generated/rev.js", + "kind": "generated", + "sha256": "c2dcbfa0515e8dcb33c3d8b85637c9cd9025b3af59cea2641cd21fa8cb97266d" + }, + { + "path": "generated/schema/core.d.ts", + "kind": "generated", + "sha256": "1be0af177c724d8cc93988bc98b3ad4e805abdadbfb3e29e66adaa01eb9a40f4" + }, + { + "path": "generated/schema/core.js", + "kind": "generated", + "sha256": "ff392297d7e9b89cc2d13eb3a5ba8674c6a86ec33f3944a7f6b506911b694a13" + }, + { + "path": "generated/schema/index.d.ts", + "kind": "generated", + "sha256": "6da044bd5fd7b45d6477a80f95020b9f9fe5c03c051a4084670a16cd324ef29e" + }, + { + "path": "generated/schema/index.js", + "kind": "generated", + "sha256": "12cf220d6346842f82661c803d4e88789e82d63d6856b14c436cace84bb00bbe" + }, + { + "path": "generated/schema/migrations.d.ts", + "kind": "generated", + "sha256": "bba9a6ee77c48b711557619d93ee606c31352507ed13b7874f8ac90595c3fe9c" + }, + { + "path": "generated/schema/migrations.js", + "kind": "generated", + "sha256": "a7cece1cd73d9f48ecd581ad84432e36d6910ee973575f016bfda9295ebb69bb" + }, + { + "path": "generated/schema/sync.d.ts", + "kind": "generated", + "sha256": "8522f9beb07a77a0aa72a04696e91275daa75f4c4cd8f4e55c2abace16cc7c63" + }, + { + "path": "generated/schema/sync.js", + "kind": "generated", + "sha256": "8443d06e882501142bc339d0352cebf0e34b4e567e6108dc2c159e7f1e01a2a7" + }, + { + "path": "generated/storage.d.ts", + "kind": "generated", + "sha256": "d390227824a063d864b0547f56c5a002fbd9ba87040b704cd1a2cb25acec86b4" + }, + { + "path": "generated/storage.js", + "kind": "generated", + "sha256": "987aef02885f9253cd7e9a960d22c301ac016f23a26822341a53c9015a8d8a7b" + }, + { + "path": "generated/sync/blobs.d.ts", + "kind": "generated", + "sha256": "a2d16de1449896c36b05c5a190a196feff273bbc7fbfc32265ba80dbee5acb74" + }, + { + "path": "generated/sync/blobs.js", + "kind": "generated", + "sha256": "c8de1530d8f350a73cebb3b250130dfb8035b2e637159c7c0f22cf72e883370b" + }, + { + "path": "generated/sync/changes.d.ts", + "kind": "generated", + "sha256": "6024e72f09a4d41adc7dda911d0e4abd600385500394708dc21e8dbd30c75a00" + }, + { + "path": "generated/sync/changes.js", + "kind": "generated", + "sha256": "9913fd8fd0dc49c44b423f72034bd3baac771b863032404ad7716a8e1f522b88" + }, + { + "path": "generated/sync/manifests.d.ts", + "kind": "generated", + "sha256": "d2c3007a5589629bc09e82246c17f815c1c43f8f42206d82943be670efaf2dc0" + }, + { + "path": "generated/sync/manifests.js", + "kind": "generated", + "sha256": "c7fda6645cb3f8c14f01534886e712f33b4c6e08c7aa2665c99e78d2223dee84" + }, + { + "path": "generated/sync/paths.d.ts", + "kind": "generated", + "sha256": "bb217c939bf2e39ef20742ca446e90ff8a6722936ec060a5ee622e1659be2142" + }, + { + "path": "generated/sync/paths.js", + "kind": "generated", + "sha256": "348e3da2113fe90f16255519332c5cfdd52b4e87e922abe5a986209cfa9b297b" + }, + { + "path": "generated/types.d.ts", + "kind": "generated", + "sha256": "04426960190ecede5463f0267926fdf8d1ffe0aa6ab51a240603d601f1520f88" + }, + { + "path": "generated/types.js", + "kind": "generated", + "sha256": "8e609bb71c20b858c77f0e9f90bb1319db8477b13f9f965f1a1e18524bf50881" + }, + { + "path": "upstream/src/errors.ts", + "kind": "upstream", + "sha256": "54d6fbe642c70855b861982eba6166eddb91008e25bb23fff4e986104eac8326" + }, + { + "path": "upstream/src/fs/blobCache.ts", + "kind": "upstream", + "sha256": "d59f08a862c6f73819a93eac109a63bba526478f2be19d4d79875e76c83fd58e" + }, + { + "path": "upstream/src/fs/chmod.ts", + "kind": "upstream", + "sha256": "1479a40bb057f4a7704f2ffb5142ba33053a66db7492a3fa3544d0595e8a040b" + }, + { + "path": "upstream/src/fs/filesystem.ts", + "kind": "upstream", + "sha256": "7ec77a71db7d4a70b0446b68a5af71775238dd5854ddd4e967d8177bb751afd8" + }, + { + "path": "upstream/src/fs/find.ts", + "kind": "upstream", + "sha256": "2f76e31226d9a4b0d77b8eb962307b118a52482343232ba5b98c358493b5b34f" + }, + { + "path": "upstream/src/fs/grep.ts", + "kind": "upstream", + "sha256": "296d25c79d79b0bb9f7732073f50c72a4611013792dfa757907bd396677c2733" + }, + { + "path": "upstream/src/fs/link.ts", + "kind": "upstream", + "sha256": "e981de8dae0a9a4578848a609946dccf8aaaa8d6ae13b7b4a50f5b2138adca12" + }, + { + "path": "upstream/src/fs/ls.ts", + "kind": "upstream", + "sha256": "bc4773a5fffb1cf61e73be13b70bff7123992f0b6fef7f0993b8796bf152dfcb" + }, + { + "path": "upstream/src/fs/mkdir.ts", + "kind": "upstream", + "sha256": "40ca9604ae3d14a30c42edefc80fabf43c9c7d686b8d4ef25c8021eeca468626" + }, + { + "path": "upstream/src/fs/mount-guard.ts", + "kind": "upstream", + "sha256": "38d6f5898f369f1f2888289e1b17bac577027e3dbd36d77404c86028f5548caf" + }, + { + "path": "upstream/src/fs/readFile.ts", + "kind": "upstream", + "sha256": "405dc0078f6703e407151f0fcb7a10dbfdc349d7413e9eb39d0267689d30ecef" + }, + { + "path": "upstream/src/fs/readdir.ts", + "kind": "upstream", + "sha256": "6dddd5e18358256e2338f19b4f91e1eea28e6d6509d4b6088c6352371aed454e" + }, + { + "path": "upstream/src/fs/readlink.ts", + "kind": "upstream", + "sha256": "2f55a466512644b00e32457d2d0bea497acb1b00db34721cf7a398bd3a5c9f42" + }, + { + "path": "upstream/src/fs/rename.ts", + "kind": "upstream", + "sha256": "93141e250557b422da0f81e8f365a15e6654b92c195f636e3248fd731034b9c2" + }, + { + "path": "upstream/src/fs/resolve.ts", + "kind": "upstream", + "sha256": "0a37b1c63967393e1f03f4d12c96a9eba4a85f63ee9c7f4afd7a3269c7ed52f9" + }, + { + "path": "upstream/src/fs/resolveCache.ts", + "kind": "upstream", + "sha256": "ad9c28a749068a1d6c2fca657b57606f1167d52bf76428a88e65a2d1414f851a" + }, + { + "path": "upstream/src/fs/rm.ts", + "kind": "upstream", + "sha256": "c724651f291ff92d5d2231320d029175d36f25d00bd6750b6c3e7feecfbf76ce" + }, + { + "path": "upstream/src/fs/stat.ts", + "kind": "upstream", + "sha256": "075bce429f4b667a78a4313f162403eda83f4c833560741b7c9071042a26912f" + }, + { + "path": "upstream/src/fs/symlink.ts", + "kind": "upstream", + "sha256": "7461150ff111663560ffc51a16f4dcff2caf9520f1771943545194c5d1d5c7f7" + }, + { + "path": "upstream/src/fs/unlink.ts", + "kind": "upstream", + "sha256": "936e86ca8c1280795406d20bd59ec030f75824c50e86a4ca4584034b3be05fbf" + }, + { + "path": "upstream/src/fs/writeBuffer.ts", + "kind": "upstream", + "sha256": "063c9059cd31b526cef0da4258ef74e082c04545f3020aeee390d7f1100898b6" + }, + { + "path": "upstream/src/fs/writeFile.ts", + "kind": "upstream", + "sha256": "819b279949a835c2efb76f25c2b1e7561939e36ffa9fcff38e9ccee7253432c9" + }, + { + "path": "upstream/src/path.ts", + "kind": "upstream", + "sha256": "b1ac0e5f9f30ca650d2537879af111d016c0e520621e4a77db23efdcc3076fab" + }, + { + "path": "upstream/src/rev.ts", + "kind": "upstream", + "sha256": "eadac77a6394e60415b33c8cd2591865b3bded21c01bdc76efa6babf3f9ffdb5" + }, + { + "path": "upstream/src/schema/core.ts", + "kind": "upstream", + "sha256": "e44335a39467e1b5106961933d98f23fc969998a69bbeb57dff67abdb9de1cb5" + }, + { + "path": "upstream/src/schema/index.ts", + "kind": "upstream", + "sha256": "8ec4fbc922106202d974a8372e6737b1e72886201c69a7246df6d3138bf4ed2f" + }, + { + "path": "upstream/src/schema/migrations.ts", + "kind": "upstream", + "sha256": "8ab5509afde31d895e5ca60343be5cb80f262a0df09f087c5a6dee2de8993309" + }, + { + "path": "upstream/src/schema/sync.ts", + "kind": "upstream", + "sha256": "424460caacdf54581aa9dd2e25b3bf85d80981adc608027aac70aafec5c29586" + }, + { + "path": "upstream/src/storage.ts", + "kind": "upstream", + "sha256": "d17b61c94d0f213f76575a0fdb93c8d2983bb7346126f1e16a3d75e49f3d66e5" + }, + { + "path": "upstream/src/sync/blobs.ts", + "kind": "upstream", + "sha256": "d97c293c3bfbeb3ed07d753444eb318ce1730b30504964084c105eac0612aba6" + }, + { + "path": "upstream/src/sync/changes.ts", + "kind": "upstream", + "sha256": "80bad97bf4cc88c355ea3de043bb1b7d19a0344472e138cb852b6810431c3ab9" + }, + { + "path": "upstream/src/sync/manifests.ts", + "kind": "upstream", + "sha256": "f0eb11968e85330d5aea4394aa927907f5ba91b8ae994083e631012803b38878" + }, + { + "path": "upstream/src/sync/paths.ts", + "kind": "upstream", + "sha256": "e3da020a811637829c6e82590821a985d1f4af428cbd43ab6c0e7cdb1ebf5b35" + }, + { + "path": "upstream/src/types.ts", + "kind": "upstream", + "sha256": "9a18a3bd91f61069f4011595856cd3bb456d9c9df742994e3be401f54e322e09" + } + ] +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/PROVENANCE.md b/packages/workflow/vendor/cloudflare-computer-dofs/PROVENANCE.md new file mode 100644 index 00000000..a6adba04 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/PROVENANCE.md @@ -0,0 +1,15 @@ +# Cloudflare Computer DOFS provenance + +This directory contains the production source closure used by Executable Markdown's local retained Workspace provider. + +- Repository: `https://github.com/cloudflare/computer` +- Commit: `63d363632e558f7e077794988d36ed75017c2a62` +- Upstream path: `packages/dofs/src` +- License: MIT; the complete repository notice is copied as `LICENSE`. +- Retrieved: 2026-08-07 + +`upstream/src` preserves the selected upstream TypeScript files byte-for-byte. The selection is the transitive closure required by the DOFS database and schema, `WorkspaceFilesystem`, the mutation operations used by the adapter, and manifest/blob validation. It excludes garbage collection, backends, workers, workerd, Containers, Worker Shell, FUSE, benchmarks, and upstream tests. + +The `.js` and `.d.ts` files in `generated` are deterministic artifacts emitted from those inputs by the repository-pinned TypeScript compiler. They exist because the upstream sources use emitted `.js` import specifiers. XMD's SQLite and transaction adapters remain outside this snapshot. + +There are no source patches. `deno task vendor:verify` performs an offline digest, inventory, and generated-output comparison. It rejects missing, extra, changed, or unreproducible files. diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/errors.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/errors.d.ts new file mode 100644 index 00000000..0604f1f1 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/errors.d.ts @@ -0,0 +1,7 @@ +export type WorkspaceErrorCode = "ENOENT" | "ENOTEMPTY" | "ENOTDIR" | "EISDIR" | "EEXIST" | "EINVAL" | "EACCES" | "EPERM" | "EROFS" | "ENOSYS" | "EBADF" | "ELOOP" | "EUNKNOWN_HASH" | "EIO"; +export interface WorkspaceFsError extends Error { + code: WorkspaceErrorCode; + path?: string; +} +export declare function createWorkspaceError(code: WorkspaceErrorCode, message: string, path?: string): WorkspaceFsError; +export declare function invalidPath(path: string, reason: string): WorkspaceFsError; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/errors.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/errors.js new file mode 100644 index 00000000..b1988fe3 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/errors.js @@ -0,0 +1,10 @@ +export function createWorkspaceError(code, message, path) { + const error = new Error(path === undefined ? message : `${message}: ${path}`); + error.name = "WorkspaceFsError"; + error.code = code; + error.path = path; + return error; +} +export function invalidPath(path, reason) { + return createWorkspaceError("EINVAL", `Invalid path (${reason})`, path); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/blobCache.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/blobCache.d.ts new file mode 100644 index 00000000..7090733f --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/blobCache.d.ts @@ -0,0 +1,3 @@ +import type { Database } from "../storage.js"; +export declare function getBlobBytes(db: Database, hash: Uint8Array): Uint8Array | undefined; +export declare function clearBlobCache(db: Database): void; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/blobCache.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/blobCache.js new file mode 100644 index 00000000..8b30d50b --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/blobCache.js @@ -0,0 +1,78 @@ +// In-process LRU cache of vfs_blob_bytes payloads, keyed by hash. +// +// FUSE reads up to 128 KiB at a time (the kernel's default max_read); +// our chunk size is 512 KiB. A sequential read of a chunk-backed file +// re-fetches the same blob 4x by default. Worse, a 64 MiB file of +// repeated content (e.g. `dd if=/dev/zero`) deduplicates to a single +// blob in vfs_blobs, and we then re-fetch that one blob 512 times +// over the lifetime of one read pass. +// +// vfs_blob_bytes is content-addressed. The normal write path +// (upsertChunkBlob) uses ON CONFLICT DO NOTHING, so a correct +// (hash, bytes) pair is never overwritten and the cache stays valid +// for it. The one exception is repair: stageBlob (the sync receiver +// path) uses ON CONFLICT DO UPDATE SET bytes to replace an incomplete +// or size-mismatched payload left by an interrupted or corrupt write, +// and clears this cache afterward so a stale payload is never served +// after a repair. +// +// The cache is bounded (CHUNK_CACHE_MAX_ENTRIES) and per-Database so +// independent test databases don't pollute each other. Eviction is +// LRU; access moves an entry to the most-recent position. +// Number of distinct blob payloads kept in memory per Database. +// At 512 KiB per blob this caps the cache at ~8 MiB, large enough +// to hold a handful of hot chunks for sequential reads of large +// files without dominating process memory. +const CHUNK_CACHE_MAX_ENTRIES = 16; +const caches = new WeakMap(); +function cacheFor(db) { + let cache = caches.get(db); + if (cache === undefined) { + cache = new Map(); + caches.set(db, cache); + } + return cache; +} +// Stringify a 32-byte hash so it can key a JS Map. Latin-1 +// preserves every byte exactly and avoids the allocation cost of +// hex encoding for what is a very hot path. +function hashKey(hash) { + let out = ""; + for (let i = 0; i < hash.byteLength; i++) { + out += String.fromCharCode(hash[i]); + } + return out; +} +// Look up blob bytes by hash. Cache hit returns the cached +// Uint8Array directly (callers must not mutate it). Cache miss +// queries vfs_blob_bytes and stores the result. Returns undefined +// if the blob isn't in the store. +export function getBlobBytes(db, hash) { + const cache = cacheFor(db); + const key = hashKey(hash); + const cached = cache.get(key); + if (cached !== undefined) { + // Reinsert to move to the most-recent position. Map iteration + // order is insertion order, so this gives us LRU eviction for + // free without a separate doubly-linked list. + cache.delete(key); + cache.set(key, cached); + return cached; + } + const row = db.one("SELECT bytes FROM vfs_blob_bytes WHERE hash = ?", hash); + if (row === undefined) + return undefined; + cache.set(key, row.bytes); + while (cache.size > CHUNK_CACHE_MAX_ENTRIES) { + const first = cache.keys().next(); + if (first.done === true) + break; + cache.delete(first.value); + } + return row.bytes; +} +// Reset the cache for `db`. Tests use this to keep cache state from +// leaking between cases that share a Database constructor pattern. +export function clearBlobCache(db) { + caches.delete(db); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/chmod.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/chmod.d.ts new file mode 100644 index 00000000..1f1c979b --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/chmod.d.ts @@ -0,0 +1,2 @@ +import type { Database } from "../storage.js"; +export declare function chmod(db: Database, path: string, mode: number, now: () => number): void; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/chmod.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/chmod.js new file mode 100644 index 00000000..86d23c70 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/chmod.js @@ -0,0 +1,25 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { incrementRev } from "../rev.js"; +import { assertNotReadOnly } from "./mount-guard.js"; +import { resolveInode } from "./resolve.js"; +// Change the file mode bits of a path. Follows symlinks like POSIX +// chmod — the change lands on the target, not the link. Bumps rev +// and mtime so the sync protocol carries the change. +// +// The supplied mode is masked to 12 bits (the permission bits and +// the setuid / setgid / sticky bits). Callers that pass a Node-style +// stat.mode with file-type bits in the upper byte get only the +// permission half stored. +export function chmod(db, path, mode, now) { + const { path: canonical } = canonicalizePath(path); + assertNotReadOnly(db, canonical); + db.transactionSync(() => { + const node = resolveInode(db, canonical); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); + } + const rev = incrementRev(db); + db.run("UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ? WHERE inode = ?", mode & 0o7777, now(), rev, node.inode); + }); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/filesystem.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/filesystem.d.ts new file mode 100644 index 00000000..44c7053d --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/filesystem.d.ts @@ -0,0 +1,32 @@ +import type { Database } from "../storage.js"; +import { type WorkspaceFoundEntry } from "./find.js"; +import { type GrepOptions, type WorkspaceGrepMatch } from "./grep.js"; +import { type MkdirOptions } from "./mkdir.js"; +import { type ReaddirOptions, type WorkspaceDirentResult } from "./readdir.js"; +import { type ReadFileOptions } from "./readFile.js"; +import { type RmOptions } from "./rm.js"; +import { type WorkspaceStatResult } from "./stat.js"; +import { type WriteFileContent, type WriteFileOptions } from "./writeFile.js"; +export interface WorkspaceFilesystemOptions { + now?: () => number; +} +export declare class WorkspaceFilesystem { + readonly db: Database; + readonly now: () => number; + constructor(db: Database, options?: WorkspaceFilesystemOptions); + readFile(path: string): Promise>; + readFile(path: string, encoding: "utf8"): Promise; + readFile(path: string, options: ReadFileOptions): Promise>; + stat(path: string): Promise; + lstat(path: string): Promise; + readlink(path: string): Promise; + readdir(path: string, options?: ReaddirOptions): Promise; + find(directory: string, pattern?: string): Promise; + ls(prefix: string): Promise; + grep(pattern: string, path: string, options?: GrepOptions): Promise; + writeFile(path: string, content: WriteFileContent, options?: WriteFileOptions): Promise; + mkdir(path: string, options?: MkdirOptions): Promise; + rm(path: string, options?: RmOptions): Promise; + chmod(path: string, mode: number): Promise; + symlink(target: string, path: string): Promise; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/filesystem.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/filesystem.js new file mode 100644 index 00000000..27c2b3e6 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/filesystem.js @@ -0,0 +1,91 @@ +// WorkspaceFilesystem — class wrapper that binds a Database and a +// clock to the free fs/* functions. +// +// Every method here is a thin forward to the matching free +// function. The class exists so callers (host-side Workspace, +// in-container tools, tests) get a single instance to thread +// through their code rather than passing (db, now) pairs into +// every call. +// +// Free functions remain exported for internal callers — the +// apply paths in sync/* operate on a Database directly, and the +// in-package tests skip the class wrapper when they only need a +// single op. +import { chmod } from "./chmod.js"; +import { find } from "./find.js"; +import { grep } from "./grep.js"; +import { ls } from "./ls.js"; +import { mkdir } from "./mkdir.js"; +import { readdir } from "./readdir.js"; +import { readFile } from "./readFile.js"; +import { readlink } from "./readlink.js"; +import { rm } from "./rm.js"; +import { lstat, stat } from "./stat.js"; +import { symlink } from "./symlink.js"; +import { writeFile } from "./writeFile.js"; +export class WorkspaceFilesystem { + db; + now; + constructor(db, options = {}) { + this.db = db; + this.now = options.now ?? Date.now; + } + readFile(path, optionsOrEncoding) { + // Forward through the free function's overload set. The + // individual overloads above let callers see the precise + // return type for each input shape. + // Cast through the union overload of the free function; + // the class's overloads above carry the precise return type + // for each input shape back to the caller. + return readFile(this.db, path, optionsOrEncoding); + } + async stat(path) { + return stat(this.db, path); + } + // POSIX lstat — like stat, but doesn't follow a trailing symlink. + // Use when the caller wants to inspect the link itself: readlink + // / unlink under a Node-style fs surface, or just-bash's adapter + // routing lstat through to the workspace. + async lstat(path) { + return lstat(this.db, path); + } + // Return the stored target of a symlink. EINVAL when path is + // not a symlink; ENOENT when path is missing. + async readlink(path) { + return readlink(this.db, path); + } + async readdir(path, options = {}) { + return readdir(this.db, path, options); + } + async find(directory, pattern) { + return find(this.db, directory, pattern); + } + async ls(prefix) { + return ls(this.db, prefix); + } + grep(pattern, path, options = {}) { + return grep(this.db, pattern, path, options); + } + // --- Mutations --------------------------------------------------- + writeFile(path, content, options = {}) { + return writeFile(this.db, path, content, options, this.now); + } + async mkdir(path, options = {}) { + mkdir(this.db, path, options, this.now); + } + async rm(path, options = {}) { + rm(this.db, path, options); + } + // Change the permission bits on a path. Follows symlinks like + // POSIX chmod — the change lands on the target, not the link. + // The supplied mode is masked to twelve bits. + async chmod(path, mode) { + chmod(this.db, path, mode, this.now); + } + // Create a symbolic link at `path` pointing at `target`. The + // target is stored verbatim; it can be relative or absolute and + // is allowed to dangle. + async symlink(target, path) { + symlink(this.db, target, path, this.now); + } +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/find.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/find.d.ts new file mode 100644 index 00000000..c084b27a --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/find.d.ts @@ -0,0 +1,6 @@ +import type { Database } from "../storage.js"; +export interface WorkspaceFoundEntry { + path: string; + type: "file" | "dir"; +} +export declare function find(db: Database, directory: string, pattern?: string): WorkspaceFoundEntry[]; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/find.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/find.js new file mode 100644 index 00000000..d09c3e2f --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/find.js @@ -0,0 +1,84 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { resolveInode } from "./resolve.js"; +export function find(db, directory, pattern) { + const { path: canonical } = canonicalizePath(directory); + const node = resolveInode(db, canonical); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); + } + if (node.type !== "dir") { + throw createWorkspaceError("ENOTDIR", `not a directory: ${canonical}`, canonical); + } + const out = []; + // An empty pattern is equivalent to no pattern: walk and return + // everything rather than compiling it into `^$`, which would match + // only empty relative paths and yield no results. + const regex = pattern ? compileGlob(pattern) : undefined; + walk(db, node.inode, canonical, out); + if (regex === undefined) { + return out; + } + // Glob matches against the path relative to the start directory. + const prefix = canonical === "/" ? "/" : `${canonical}/`; + return out.filter((entry) => { + if (!entry.path.startsWith(prefix)) + return false; + const rel = entry.path.slice(prefix.length); + return regex.test(rel); + }); +} +function walk(db, parentInode, parentPath, out) { + const children = db.all(`SELECT d.name AS name, d.child_inode AS child_inode, n.type AS type + FROM vfs_dirents d + JOIN vfs_nodes n ON n.inode = d.child_inode + WHERE d.parent_inode = ? + ORDER BY d.name`, parentInode); + for (const child of children) { + const childPath = parentPath === "/" ? `/${child.name}` : `${parentPath}/${child.name}`; + out.push({ path: childPath, type: child.type }); + if (child.type === "dir") { + walk(db, child.child_inode, childPath, out); + } + } +} +// Compile a simple glob into a regex. Supported: +// * matches any run of characters except '/' +// ** matches any run of characters including '/' +// Anything else is a literal. Regex metacharacters in literals are +// escaped so '.' in '*.ts' doesn't match an arbitrary character. +function compileGlob(pattern) { + let re = ""; + let i = 0; + while (i < pattern.length) { + const ch = pattern[i]; + if (ch === "*") { + if (pattern[i + 1] === "*") { + // '**/' matches zero or more path segments. Without the slash, '**' + // matches any run including slashes. + if (pattern[i + 2] === "/") { + re += "(?:.*/)?"; + i += 3; + } + else { + re += ".*"; + i += 2; + } + } + else { + re += "[^/]*"; + i += 1; + } + continue; + } + if (REGEX_METACHARS.has(ch)) { + re += `\\${ch}`; + } + else { + re += ch; + } + i += 1; + } + return new RegExp(`^${re}$`); +} +const REGEX_METACHARS = new Set([".", "+", "?", "^", "$", "(", ")", "[", "]", "{", "}", "|", "\\"]); diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/grep.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/grep.d.ts new file mode 100644 index 00000000..ea39d4a0 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/grep.d.ts @@ -0,0 +1,10 @@ +import type { Database } from "../storage.js"; +export interface WorkspaceGrepMatch { + path: string; + line: number; + text: string; +} +export interface GrepOptions { + ignoreCase?: boolean; +} +export declare function grep(db: Database, pattern: string, path: string, options?: GrepOptions): Promise; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/grep.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/grep.js new file mode 100644 index 00000000..a345769f --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/grep.js @@ -0,0 +1,74 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { find } from "./find.js"; +import { readFile } from "./readFile.js"; +import { resolveInode } from "./resolve.js"; +export async function grep(db, pattern, path, options = {}) { + const { path: canonical } = canonicalizePath(path); + const node = resolveInode(db, canonical); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); + } + const filePaths = node.type === "file" + ? [canonical] + : find(db, canonical) + .filter((entry) => entry.type === "file") + .map((entry) => entry.path); + const matches = []; + for (const filePath of filePaths) { + await scanFile(db, filePath, pattern, options, matches); + } + return matches; +} +// Stream the file in chunks so very large files don't load fully into +// memory. Carry a partial-line tail between chunks (everything after +// the last '\n') so a line that straddles a chunk boundary still +// matches as one line. Line numbers are 1-indexed. +async function scanFile(db, path, pattern, options, out) { + const stream = await readFile(db, path); + const reader = stream.getReader(); + const decoder = new TextDecoder("utf-8", { fatal: false }); + const needle = options.ignoreCase ? pattern.toUpperCase() : pattern; + let tail = ""; + let lineNo = 1; + while (true) { + const { value, done } = await reader.read(); + if (done) + break; + if (value === undefined) + continue; + const text = tail + decoder.decode(value, { stream: true }); + const newlineIdx = text.lastIndexOf("\n"); + const ready = newlineIdx === -1 ? "" : text.slice(0, newlineIdx); + tail = newlineIdx === -1 ? text : text.slice(newlineIdx + 1); + if (ready.length > 0) { + lineNo = scanLines(ready, lineNo, needle, options.ignoreCase === true, path, out); + } + } + // Drain the decoder and scan whatever's left (final line without a + // trailing newline). + tail += decoder.decode(); + if (tail.length > 0) { + scanLines(tail, lineNo, needle, options.ignoreCase === true, path, out); + } +} +// Walk `block` line-by-line, push matches into `out`, return the next +// 1-indexed line number to use for the following block. +function scanLines(block, startLine, needle, ignoreCase, path, out) { + let line = startLine; + let cursor = 0; + while (cursor <= block.length) { + const next = block.indexOf("\n", cursor); + const end = next === -1 ? block.length : next; + const text = block.slice(cursor, end); + const haystack = ignoreCase ? text.toUpperCase() : text; + if (haystack.includes(needle)) { + out.push({ path, line, text }); + } + line += 1; + if (next === -1) + break; + cursor = next + 1; + } + return line; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/link.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/link.d.ts new file mode 100644 index 00000000..92d31dfd --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/link.d.ts @@ -0,0 +1,2 @@ +import type { Database } from "../storage.js"; +export declare function link(db: Database, existingPath: string, newPath: string): void; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/link.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/link.js new file mode 100644 index 00000000..046fb6e5 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/link.js @@ -0,0 +1,53 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { incrementRev } from "../rev.js"; +import { ROOT_INODE } from "../schema/index.js"; +import { assertNotReadOnly } from "./mount-guard.js"; +import { resolveInode } from "./resolve.js"; +import { invalidateResolveExact } from "./resolveCache.js"; +function resolveParent(db, parts, canonical) { + let parentInode = ROOT_INODE; + for (let i = 0; i < parts.length - 1; i++) { + const child = db.one("SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", parentInode, parts[i]); + if (child === undefined) { + throw createWorkspaceError("ENOENT", `parent directory missing: ${canonical}`, canonical); + } + const next = db.one("SELECT inode, type FROM vfs_nodes WHERE inode = ?", child.child_inode); + if (next === undefined) { + throw createWorkspaceError("ENOENT", `dangling dirent: ${canonical}`, canonical); + } + if (next.type !== "dir") { + throw createWorkspaceError("ENOTDIR", `parent path segment is not a directory: ${canonical}`, canonical); + } + parentInode = next.inode; + } + return parentInode; +} +export function link(db, existingPath, newPath) { + const { parts, path: canonicalNew } = canonicalizePath(newPath); + if (parts.length === 0) { + throw createWorkspaceError("EEXIST", "cannot link onto root", canonicalNew); + } + assertNotReadOnly(db, canonicalNew); + db.transactionSync(() => { + const source = resolveInode(db, existingPath); + if (source === null) { + throw createWorkspaceError("ENOENT", `no such file: ${existingPath}`, existingPath); + } + if (source.type !== "file") { + throw createWorkspaceError("EPERM", `cannot hardlink non-file: ${existingPath}`, existingPath); + } + const parentInode = resolveParent(db, parts, canonicalNew); + const leafName = parts[parts.length - 1]; + const existing = db.one("SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", parentInode, leafName); + if (existing !== undefined) { + throw createWorkspaceError("EEXIST", `path exists: ${canonicalNew}`, canonicalNew); + } + db.run("INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", parentInode, leafName, source.inode); + const rev = incrementRev(db); + db.run("UPDATE vfs_nodes SET rev = ? WHERE inode = ?", rev, source.inode); + // A new hardlink name for an existing file: a leaf with no + // descendants, so drop just the (possibly negative) entry for it. + invalidateResolveExact(db, canonicalNew); + }); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/ls.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/ls.d.ts new file mode 100644 index 00000000..cafc1fae --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/ls.d.ts @@ -0,0 +1,2 @@ +import type { Database } from "../storage.js"; +export declare function ls(db: Database, prefix: string): string[]; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/ls.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/ls.js new file mode 100644 index 00000000..1e797da2 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/ls.js @@ -0,0 +1,49 @@ +import { canonicalizePath } from "../path.js"; +import { ROOT_INODE } from "../schema/index.js"; +// Recursive CTE that materializes the file paths under one listing +// root. Files only (no directory entries) because that's the +// documented "flat list of file paths" semantics. +// +// The walk is seeded at the listing root's inode: each row is +// (inode, path, type), built by concatenating dirent names with '/' +// separators onto the seed path. Scoping the seed to the requested +// directory keeps the walk O(subtree) instead of O(whole tree). +const LS_QUERY = ` + WITH RECURSIVE walk(inode, path, type) AS ( + SELECT inode, ?, type FROM vfs_nodes WHERE inode = ? + UNION ALL + SELECT n.inode, w.path || '/' || d.name, n.type + FROM walk w + JOIN vfs_dirents d ON d.parent_inode = w.inode + JOIN vfs_nodes n ON n.inode = d.child_inode + ) + SELECT path FROM walk + WHERE type = 'file' + ORDER BY path +`; +// Walk dirents from the root to `parts` without following symlinks, so +// the seed matches the CTE's structural view: a symlink component has +// no dirents and thus lists nothing, and a missing or non-directory +// component resolves to null (an empty listing). Returns the root +// inode for an empty path. +function resolvePrefixInode(db, parts) { + let inode = ROOT_INODE; + for (const name of parts) { + const child = db.one("SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", inode, name); + if (child === undefined) + return null; + inode = child.child_inode; + } + return inode; +} +export function ls(db, prefix) { + const { parts, path: canonical } = canonicalizePath(prefix); + const inode = resolvePrefixInode(db, parts); + if (inode === null) + return []; + // Root contributes the empty string so its children start with '/'; + // a non-root prefix seeds its own path so descendants read as + // absolute paths. + const seedPath = canonical === "/" ? "" : canonical; + return db.all(LS_QUERY, seedPath, inode).map((row) => row.path); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/mkdir.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/mkdir.d.ts new file mode 100644 index 00000000..6f6bafba --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/mkdir.d.ts @@ -0,0 +1,6 @@ +import type { Database } from "../storage.js"; +export interface MkdirOptions { + recursive?: boolean; + mode?: number; +} +export declare function mkdir(db: Database, path: string, options: MkdirOptions, now: () => number): void; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/mkdir.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/mkdir.js new file mode 100644 index 00000000..dab3d93a --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/mkdir.js @@ -0,0 +1,83 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { incrementRev } from "../rev.js"; +import { ROOT_INODE } from "../schema/index.js"; +import { assertNotReadOnly } from "./mount-guard.js"; +import { invalidateResolveExact } from "./resolveCache.js"; +// Look up a child by name under a parent directory. Returns undefined +// when there's no dirent. The caller decides whether that's an error. +function lookupChild(db, parentInode, name) { + const row = db.one("SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", parentInode, name); + if (row === undefined) { + return undefined; + } + const node = db.one("SELECT inode, type FROM vfs_nodes WHERE inode = ?", row.child_inode); + if (node === undefined) { + return undefined; + } + return node; +} +// Create one directory entry under `parentInode`, returning the new +// inode. The caller has already verified the name is not taken. +function createDir(db, parentInode, name, mode, mtime, rev) { + // RETURNING folds the rowid read into the INSERT. + const row = db.one("INSERT INTO vfs_nodes (type, mode, mtime, rev) VALUES ('dir', ?, ?, ?) RETURNING inode", mode, mtime, rev); + if (row === undefined) { + throw createWorkspaceError("EIO", "failed to allocate inode"); + } + const inode = row.inode; + db.run("INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", parentInode, name, inode); + return inode; +} +export function mkdir(db, path, options, now) { + const { parts, path: canonical } = canonicalizePath(path); + const recursive = options.recursive === true; + const mode = (options.mode ?? 0o755) & 0o7777; + if (parts.length === 0) { + // Root always exists post-initializeSchema; mkdir("/") is EEXIST + // even with recursive (matches Node fs.mkdir's "EEXIST on root" + // behaviour for non-recursive; for recursive Node returns + // undefined, but our docs treat mkdir("/") as nonsensical). + throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); + } + assertNotReadOnly(db, canonical); + db.transactionSync(() => { + const rev = incrementRev(db); + const mtime = now(); + let parentInode = ROOT_INODE; + // Walk all but the final segment. Each must already exist as a + // directory; if `recursive`, we create missing ones. + for (let i = 0; i < parts.length - 1; i++) { + const name = parts[i]; + const existing = lookupChild(db, parentInode, name); + if (existing === undefined) { + if (!recursive) { + throw createWorkspaceError("ENOENT", `parent directory missing: ${canonical}`, canonical); + } + parentInode = createDir(db, parentInode, name, 0o755, mtime, rev); + // A newly created directory is empty, so a cached negative for + // its own path is the only stale entry possible; drop it exact. + invalidateResolveExact(db, `/${parts.slice(0, i + 1).join("/")}`); + continue; + } + if (existing.type !== "dir") { + throw createWorkspaceError("ENOTDIR", `parent path segment is not a directory: ${canonical}`, canonical); + } + parentInode = existing.inode; + } + // Final segment. + const leafName = parts[parts.length - 1]; + const existing = lookupChild(db, parentInode, leafName); + if (existing !== undefined) { + // EEXIST is correct for both "already a directory" and + // "already a file" per docs/04. Recursive only swallows the + // already-a-directory case. + if (recursive && existing.type === "dir") { + return; + } + throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); + } + createDir(db, parentInode, leafName, mode, mtime, rev); + invalidateResolveExact(db, canonical); + }); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/mount-guard.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/mount-guard.d.ts new file mode 100644 index 00000000..e2f9dfe9 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/mount-guard.d.ts @@ -0,0 +1,5 @@ +import type { Database } from "../storage.js"; +export declare function invalidateReadOnlyMountCache(db: Database): void; +export declare function getReadOnlyMountRoots(db: Database): readonly string[]; +export declare function assertNotReadOnly(db: Database, path: string): void; +export declare function readOnlyRootFor(db: Database, path: string): string | undefined; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/mount-guard.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/mount-guard.js new file mode 100644 index 00000000..ab468c35 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/mount-guard.js @@ -0,0 +1,76 @@ +// Read-only mount guard. +// +// Every dofs mutating entry point (writeFile, mkdir, rm, and the +// apply path in sync/apply.ts) consults this module to reject +// writes that fall under a registered read-only mount root. The +// guard lives at the data layer so container-side writes that +// arrive via pullOnce -> applyChanges are caught too — the +// workspace-side surface wrapper alone cannot see them. +// +// The set of read-only roots is small (one row per registered +// mount per workspace, typically <10) and changes only at indexer +// write time. Cache it per Database in a WeakMap so repeat lookups +// don't hit SQLite. The mount indexer in @cloudflare/computer +// invalidates the cache via `invalidateReadOnlyMountCache(db)` after +// it writes _vfs_mounts. +import { createWorkspaceError } from "../errors.js"; +// undefined sentinel = "not loaded yet"; an empty array means +// "loaded, no read-only mounts registered". The two are not the +// same: the empty case must skip the SQL lookup on every check. +const cache = new WeakMap(); +// Public so the workspace-side indexer can drop the cache after it +// writes a new _vfs_mounts row. Tests also call it when they stage +// a mount fixture by direct SQL. +export function invalidateReadOnlyMountCache(db) { + cache.delete(db); +} +function loadReadOnlyRoots(db) { + const rows = db.all("SELECT root FROM _vfs_mounts WHERE mode = 'read-only'"); + const roots = rows.map((r) => r.root); + cache.set(db, roots); + return roots; +} +export function getReadOnlyMountRoots(db) { + const cached = cache.get(db); + if (cached !== undefined) + return cached; + return loadReadOnlyRoots(db); +} +// Symmetric overlap check between a candidate write path and a +// mount root. Either: +// - `path` is at or below `root` (a direct write or rm under the +// mount root), OR +// - `root` is below `path` (an ancestor rm that would recurse +// through the mount). +// Both shapes must be blocked so a read-only mount survives both +// vectors. Mirrors the predicate that lived in +// GuardedWorkspaceFilesystem before the data-layer move. +function overlapsRoot(path, root) { + return path === root || path.startsWith(`${root}/`) || root.startsWith(`${path}/`); +} +// Throws EROFS when the path overlaps any read-only mount root. +// Callers should invoke this before any DB mutation. The error +// shape matches the existing createWorkspaceError contract so +// surface callers see a normal WorkspaceFsError. +export function assertNotReadOnly(db, path) { + const roots = getReadOnlyMountRoots(db); + if (roots.length === 0) + return; + for (const root of roots) { + if (overlapsRoot(path, root)) { + throw createWorkspaceError("EROFS", `read-only mount at ${root}: cannot modify`, path); + } + } +} +// Variant for callers that already know the path is canonicalised +// and want to reject a single descendant during a recursive walk +// (rm's walkPostOrder). Returns the matching root or undefined; the +// caller decides whether to throw, log, or skip. +export function readOnlyRootFor(db, path) { + const roots = getReadOnlyMountRoots(db); + for (const root of roots) { + if (overlapsRoot(path, root)) + return root; + } + return undefined; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/readFile.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/readFile.d.ts new file mode 100644 index 00000000..71f4f363 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/readFile.d.ts @@ -0,0 +1,8 @@ +import type { Database } from "../storage.js"; +export interface ReadFileOptions { + encoding?: "utf8"; +} +export declare function readFile(db: Database, path: string): Promise>; +export declare function readFile(db: Database, path: string, encoding: "utf8"): Promise; +export declare function readFile(db: Database, path: string, options: ReadFileOptions): Promise>; +export declare function readRangeSync(db: Database, path: string, offset: number, length: number): Uint8Array; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/readFile.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/readFile.js new file mode 100644 index 00000000..a5a4243f --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/readFile.js @@ -0,0 +1,156 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { getBlobBytes } from "./blobCache.js"; +import { resolveInode } from "./resolve.js"; +import { getPendingWriteBufferByPath, getWriteBuffer } from "./writeBuffer.js"; +import { CHUNK_SIZE } from "./writeFile.js"; +export async function readFile(db, path, optionsOrEncoding) { + const wantString = optionsOrEncoding === "utf8" || + (typeof optionsOrEncoding === "object" && optionsOrEncoding?.encoding === "utf8"); + // Pending-create files surface through the path-keyed buffer. + const { path: canonical } = canonicalizePath(path); + const pending = getPendingWriteBufferByPath(db, canonical); + if (pending !== undefined) { + const snapshot = new Uint8Array(pending.size); + snapshot.set(pending.buf.subarray(0, pending.size)); + if (wantString) + return new TextDecoder().decode(snapshot); + return new ReadableStream({ + start(controller) { + controller.enqueue(snapshot); + controller.close(); + }, + }); + } + // Resolve up front so we surface ENOENT/EISDIR before doing any + // streaming work. + const node = resolveInode(db, path); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such file: ${path}`, path); + } + if (node.type !== "file") { + throw createWorkspaceError("EISDIR", `path is a directory: ${path}`, path); + } + // While a write buffer is open for this inode it is the source of + // truth. Skip the chunk store and serve the buffered bytes. + const buffered = getWriteBuffer(db, node.inode); + if (buffered?.dirty) { + const snapshot = new Uint8Array(buffered.size); + snapshot.set(buffered.buf.subarray(0, buffered.size)); + if (wantString) + return new TextDecoder().decode(snapshot); + return new ReadableStream({ + start(controller) { + controller.enqueue(snapshot); + controller.close(); + }, + }); + } + const chunks = db.all("SELECT hash, size FROM vfs_chunks WHERE inode = ? ORDER BY idx", node.inode); + if (wantString) { + // Fast path — concatenate everything and decode once. Matches the + // node:fs/promises.readFile semantics for an encoding argument: + // memory cost = whole file. + const totalSize = chunks.reduce((acc, c) => acc + c.size, 0); + const out = new Uint8Array(totalSize); + let offset = 0; + for (const chunk of chunks) { + const bytes = getBlobBytes(db, chunk.hash); + if (bytes === undefined) { + throw createWorkspaceError("EIO", `missing blob bytes for ${path}`, path); + } + out.set(bytes, offset); + offset += bytes.byteLength; + } + return new TextDecoder().decode(out); + } + // Stream form. We enqueue one Uint8Array per chunk, lazily pulled. + // Reads resolve bytes by hash and never restamp last_seen: a chunk + // being read is already linked to a node, so gc's orphan gate keeps + // it. last_seen only guards blobs staged but not yet linked. + let i = 0; + return new ReadableStream({ + pull(controller) { + if (i >= chunks.length) { + controller.close(); + return; + } + const chunk = chunks[i++]; + const bytes = getBlobBytes(db, chunk.hash); + if (bytes === undefined) { + controller.error(createWorkspaceError("EIO", `missing blob bytes for ${path}`, path)); + return; + } + controller.enqueue(bytes); + }, + }); +} +// Positional read primitive. Walks only the chunk rows that overlap +// [offset, offset+length), so the FUSE driver can serve a kernel +// read without materializing the whole file. +export function readRangeSync(db, path, offset, length) { + if (!Number.isInteger(offset) || offset < 0) { + throw createWorkspaceError("EINVAL", `invalid read offset: ${offset}`, path); + } + if (!Number.isInteger(length) || length < 0) { + throw createWorkspaceError("EINVAL", `invalid read length: ${length}`, path); + } + // Pending-create files have no inode yet. Serve reads from the + // path-keyed buffer until release commits the row. + const { path: canonical } = canonicalizePath(path); + const pending = getPendingWriteBufferByPath(db, canonical); + if (pending !== undefined) { + if (length === 0) + return new Uint8Array(); + if (offset >= pending.size) + return new Uint8Array(); + const end = Math.min(offset + length, pending.size); + return pending.buf.subarray(offset, end); + } + const node = resolveInode(db, path); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such file: ${path}`, path); + } + if (node.type !== "file") { + throw createWorkspaceError("EISDIR", `path is a directory: ${path}`, path); + } + if (length === 0) + return new Uint8Array(); + // If a write buffer is open for this inode, it is the source of + // truth: pending writes have not yet committed to vfs_chunks. + // Reading from SQLite here would return stale bytes. + const buffered = getWriteBuffer(db, node.inode); + if (buffered?.dirty) { + if (offset >= buffered.size) + return new Uint8Array(); + const end = Math.min(offset + length, buffered.size); + return buffered.buf.subarray(offset, end); + } + // node.size is the cached value resolveInode just loaded. + const totalSize = node.size; + if (offset >= totalSize) + return new Uint8Array(); + const end = Math.min(offset + length, totalSize); + const firstIdx = Math.floor(offset / CHUNK_SIZE); + const lastIdx = Math.floor((end - 1) / CHUNK_SIZE); + // Pull every overlapping chunk in one indexed range scan. Missing + // indices (a sparse file) simply don't come back, so the assembly + // below compacts around the gaps exactly as a per-index walk would. + const chunks = db.all("SELECT idx, hash FROM vfs_chunks WHERE inode = ? AND idx BETWEEN ? AND ? ORDER BY idx", node.inode, firstIdx, lastIdx); + const out = new Uint8Array(end - offset); + let written = 0; + for (const { idx, hash } of chunks) { + const start = idx * CHUNK_SIZE; + const bytes = getBlobBytes(db, hash); + if (bytes === undefined) { + throw createWorkspaceError("EIO", `missing blob bytes for ${path}`, path); + } + const srcStart = Math.max(0, offset - start); + const srcEnd = Math.min(bytes.byteLength, end - start); + if (srcEnd <= srcStart) + continue; + out.set(bytes.subarray(srcStart, srcEnd), written); + written += srcEnd - srcStart; + } + return written === out.byteLength ? out : out.subarray(0, written); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/readdir.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/readdir.d.ts new file mode 100644 index 00000000..0d6ec42d --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/readdir.d.ts @@ -0,0 +1,13 @@ +import type { Database } from "../storage.js"; +export interface WorkspaceDirentResult { + name: string; + parentPath: string; + isFile: boolean; + isDirectory: boolean; + isSymbolicLink: boolean; +} +export interface ReaddirOptions { + /** Maximum committed entries to materialize. Pending entries may extend the result. */ + limit?: number; +} +export declare function readdir(db: Database, path: string, options?: ReaddirOptions): WorkspaceDirentResult[]; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/readdir.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/readdir.js new file mode 100644 index 00000000..00a0258b --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/readdir.js @@ -0,0 +1,55 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { resolveInode } from "./resolve.js"; +import { listPendingByParent } from "./writeBuffer.js"; +export function readdir(db, path, options = {}) { + const { path: canonical } = canonicalizePath(path); + const node = resolveInode(db, canonical); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); + } + if (node.type !== "dir") { + throw createWorkspaceError("ENOTDIR", `not a directory: ${canonical}`, canonical); + } + const limit = options.limit; + if (limit !== undefined && (!Number.isSafeInteger(limit) || limit < 0)) { + throw new TypeError("readdir limit must be a non-negative safe integer"); + } + const rows = db.all(`SELECT d.name AS name, n.type AS type + FROM vfs_dirents d + JOIN vfs_nodes n ON n.inode = d.child_inode + WHERE d.parent_inode = ? + ORDER BY d.name + ${limit === undefined ? "" : "LIMIT ?"}`, ...(limit === undefined ? [node.inode] : [node.inode, limit])); + const entries = rows.map((row) => ({ + name: row.name, + parentPath: canonical, + isFile: row.type === "file", + isDirectory: row.type === "dir", + isSymbolicLink: row.type === "symlink", + })); + // Merge in pending-create buffers parented under this directory so + // a `readdir` between FUSE create and release still surfaces the + // file. Skip any whose name already appears in the SQL rows (in + // case a concurrent commit just landed it). + const pending = listPendingByParent(db, node.inode); + if (pending.length > 0) { + const seen = new Set(entries.map((e) => e.name)); + for (const entry of pending) { + if (entry.pending === undefined) + continue; + const { leafName } = entry.pending; + if (seen.has(leafName)) + continue; + entries.push({ + name: leafName, + parentPath: canonical, + isFile: true, + isDirectory: false, + isSymbolicLink: false, + }); + } + entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); + } + return entries; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/readlink.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/readlink.d.ts new file mode 100644 index 00000000..2336c412 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/readlink.d.ts @@ -0,0 +1,2 @@ +import type { Database } from "../storage.js"; +export declare function readlink(db: Database, path: string): string; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/readlink.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/readlink.js new file mode 100644 index 00000000..a3aeeb78 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/readlink.js @@ -0,0 +1,15 @@ +import { createWorkspaceError } from "../errors.js"; +import { resolveInode } from "./resolve.js"; +// Return the stored target of a symlink. Does not follow the link. +// Mirrors POSIX semantics: ENOENT for a missing path, EINVAL when +// the path resolves to something that isn't a symlink. +export function readlink(db, path) { + const node = resolveInode(db, path, { followSymlinks: false }); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such path: ${path}`, path); + } + if (node.type !== "symlink" || node.linkTarget === undefined) { + throw createWorkspaceError("EINVAL", `not a symlink: ${path}`, path); + } + return node.linkTarget; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/rename.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/rename.d.ts new file mode 100644 index 00000000..0062f67e --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/rename.d.ts @@ -0,0 +1,2 @@ +import type { Database } from "../storage.js"; +export declare function rename(db: Database, oldPath: string, newPath: string): void; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/rename.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/rename.js new file mode 100644 index 00000000..d836e01c --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/rename.js @@ -0,0 +1,164 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { incrementRev } from "../rev.js"; +import { recordDelete } from "../sync/changes.js"; +import { pathOf } from "../sync/paths.js"; +import { assertNotReadOnly } from "./mount-guard.js"; +import { resolveInode } from "./resolve.js"; +import { invalidateResolveExact, invalidateResolveSubtree } from "./resolveCache.js"; +import { unlinkDirent } from "./unlink.js"; +export function rename(db, oldPath, newPath) { + const { path: oldCanonical } = canonicalizePath(oldPath); + const { parts: newParts, path: newCanonical } = canonicalizePath(newPath); + if (oldCanonical === "/") { + throw createWorkspaceError("EINVAL", "cannot rename root", oldCanonical); + } + if (newParts.length === 0) { + throw createWorkspaceError("EINVAL", "cannot rename onto root", newCanonical); + } + assertNotReadOnly(db, oldCanonical); + assertNotReadOnly(db, newCanonical); + db.transactionSync(() => { + const source = resolveInode(db, oldCanonical, { followSymlinks: false }); + if (source === null) { + throw createWorkspaceError("ENOENT", `no such path: ${oldCanonical}`, oldCanonical); + } + // Resolve the source's real parent dirent. The parent path is + // resolved with symlinks followed so a request through a symlinked + // directory lands on the real container; the inode is then + // identified by (parent_inode, name) rather than by child_inode so + // a hardlinked source touches only the requested name. + const { parts: oldParts } = canonicalizePath(oldCanonical); + const oldName = oldParts[oldParts.length - 1]; + const oldParentPath = oldParts.length === 1 ? "/" : `/${oldParts.slice(0, -1).join("/")}`; + const oldParent = resolveInode(db, oldParentPath); + if (oldParent === null || oldParent.type !== "dir") { + throw createWorkspaceError("ENOENT", `no such path: ${oldCanonical}`, oldCanonical); + } + const oldParentReal = pathOf(db, oldParent.inode); + if (oldParentReal === null) { + throw createWorkspaceError("ENOENT", `no such path: ${oldCanonical}`, oldCanonical); + } + const oldRealPath = oldParentReal === "/" ? `/${oldName}` : `${oldParentReal}/${oldName}`; + assertNotReadOnly(db, oldRealPath); + if (oldCanonical === newCanonical) + return; + const newName = newParts[newParts.length - 1]; + const newParentPath = newParts.length === 1 ? "/" : `/${newParts.slice(0, -1).join("/")}`; + const newParent = resolveInode(db, newParentPath); + if (newParent === null || newParent.type !== "dir") { + throw createWorkspaceError("ENOENT", `parent directory missing: ${newCanonical}`, newCanonical); + } + const newParentReal = pathOf(db, newParent.inode); + if (newParentReal === null) { + throw createWorkspaceError("ENOENT", `parent directory missing: ${newCanonical}`, newCanonical); + } + const newRealPath = newParentReal === "/" ? `/${newName}` : `${newParentReal}/${newName}`; + assertNotReadOnly(db, newRealPath); + // A rename whose source and destination resolve to the very same + // dirent (same real parent and name, e.g. through a symlinked path) + // is a true no-op: leave the tree and the change stream untouched. + // This is distinct from renaming one hardlink onto another, where + // the names differ and the source link must still be removed. + if (oldParent.inode === newParent.inode && oldName === newName) + return; + const existing = db.one(`SELECT d.child_inode AS child_inode, n.type AS type + FROM vfs_dirents d + JOIN vfs_nodes n ON n.inode = d.child_inode + WHERE d.parent_inode = ? AND d.name = ?`, newParent.inode, newName); + // Authoritative directory self-move guard. It tests the *resolved* + // destination parent inode against the source subtree, so it catches + // a symlinked destination that lands inside the source and allows one + // that resolves outside it. A textual prefix test on the unresolved + // path could do neither and is intentionally absent. + if (source.type === "dir" && + renamedSubtreeContains(db, source.inode, oldRealPath, newParent.inode)) { + throw createWorkspaceError("EINVAL", `cannot rename a directory into itself: ${oldRealPath}`, newCanonical); + } + if (existing !== undefined) { + assertCompatibleOverwrite(source.type, existing.type, newCanonical); + if (existing.type === "dir") { + const childCount = db.scalar("SELECT COUNT(*) FROM vfs_dirents WHERE parent_inode = ?", existing.child_inode); + if ((childCount ?? 0) > 0) { + throw createWorkspaceError("ENOTEMPTY", `not empty: ${newCanonical}`, newCanonical); + } + } + // Displace only the destination name. The displaced inode may + // carry other hardlinks (or be the source inode itself), so reap + // its chunks and node row only once the final link disappears. + // Order matters: displace before unlinking the source so a + // hardlink-onto-hardlink rename never momentarily drops to zero + // links and reaps the inode it is about to re-point. + unlinkDirent(db, newParent.inode, newName, existing.child_inode, existing.type); + } + // Unlink only the source name; a hardlinked source keeps its other + // names alive. + db.run("DELETE FROM vfs_dirents WHERE parent_inode = ? AND name = ?", oldParent.inode, oldName); + db.run("INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", newParent.inode, newName, source.inode); + const rev = incrementRev(db); + // Rename is represented on the wire as old-path tombstones plus + // live entries for the moved inode subtree, so stamp only that + // subtree with the shared rev. Parent directory mtimes are left + // unchanged on purpose; this diverges from POSIX rename(2), but + // avoids treating the old and new parents as content changes. A + // directory move stamps and tombstones its whole subtree in two + // set-based statements; a file or symlink touches one inode and + // one path. + if (source.type === "dir") { + stampRenamedSubtree(db, source.inode, oldRealPath, rev); + } + else { + db.run("UPDATE vfs_nodes SET rev = ? WHERE inode = ?", rev, source.inode); + recordDelete(db, rev, oldRealPath); + } + // Drop cached resolutions for both endpoints. A directory move + // changes every descendant's path, so both sides need a subtree + // drop; a file/symlink move only touches the two leaf paths. The + // destination drop also covers any entry displaced by an overwrite. + if (source.type === "dir") { + invalidateResolveSubtree(db, oldRealPath); + invalidateResolveSubtree(db, newRealPath); + } + else { + invalidateResolveExact(db, oldRealPath); + invalidateResolveExact(db, newRealPath); + } + }); +} +function assertCompatibleOverwrite(sourceType, existingType, path) { + if (sourceType === "dir" && existingType === "dir") + return; + if (existingType === "dir") { + throw createWorkspaceError("EISDIR", `cannot overwrite directory: ${path}`, path); + } + if (sourceType === "dir") { + throw createWorkspaceError("ENOTDIR", `cannot overwrite non-directory: ${path}`, path); + } +} +// Recursive walk of a directory subtree seeded at an inode and its +// path. Descends through directory dirents only, so files and symlinks +// are leaves and each hardlink name yields its own row (matching the +// per-component collection it replaces). Bound as a reusable WITH +// clause whose two placeholders are the seed inode and path; callers +// append their own projection. +const SUBTREE_CTE = `WITH RECURSIVE subtree(inode, type, path) AS ( + SELECT ?, 'dir', ? + UNION ALL + SELECT n.inode, n.type, + CASE WHEN s.path = '/' THEN '/' || d.name ELSE s.path || '/' || d.name END + FROM subtree s + JOIN vfs_dirents d ON d.parent_inode = s.inode + JOIN vfs_nodes n ON n.inode = d.child_inode + WHERE s.type = 'dir' +)`; +function renamedSubtreeContains(db, rootInode, rootPath, targetInode) { + const hit = db.one(`${SUBTREE_CTE} SELECT 1 AS hit FROM subtree WHERE inode = ? LIMIT 1`, rootInode, rootPath, targetInode); + return hit !== undefined; +} +// Stamp the shared rev on every inode in the moved subtree and record +// an old-path tombstone for each entry, in two set-based statements +// over the same walk. +function stampRenamedSubtree(db, rootInode, rootPath, rev) { + db.run(`${SUBTREE_CTE} UPDATE vfs_nodes SET rev = ? WHERE inode IN (SELECT inode FROM subtree)`, rootInode, rootPath, rev); + db.run(`${SUBTREE_CTE} INSERT INTO vfs_changes (rev, path, op) SELECT ?, path, 'delete' FROM subtree ORDER BY path`, rootInode, rootPath, rev); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/resolve.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/resolve.d.ts new file mode 100644 index 00000000..f5d3c761 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/resolve.d.ts @@ -0,0 +1,13 @@ +import type { Database } from "../storage.js"; +export interface ResolvedInode { + inode: number; + type: "file" | "dir" | "symlink"; + mode: number; + mtime: number; + size: number; + linkTarget?: string; +} +export interface ResolveOptions { + followSymlinks?: boolean; +} +export declare function resolveInode(db: Database, path: string, options?: ResolveOptions): ResolvedInode | null; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/resolve.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/resolve.js new file mode 100644 index 00000000..8e31bff9 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/resolve.js @@ -0,0 +1,175 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { ROOT_INODE } from "../schema/index.js"; +import { lookupResolveCache, storeResolveCache } from "./resolveCache.js"; +// Cap the total number of symlinks resolved across a single +// resolveInode() call. Matches Linux's default SYMLOOP_MAX of 40. +const MAX_SYMLINK_FOLLOWS = 40; +// Walk vfs_dirents from ROOT_INODE down to `path`. Returns null when +// any segment is missing, when an intermediate segment is a file +// (which a real filesystem would surface as ENOTDIR — callers map +// the `null` to the appropriate POSIX code), or when a final-segment +// symlink dangles. Throws ELOOP when a cycle is detected. +// +// `path` is canonicalized internally so callers can pass user input +// directly. Pre-canonicalized paths are also accepted and incur the +// same trivial re-canonicalization cost. +export function resolveInode(db, path, options = {}) { + const followFinal = options.followSymlinks !== false; + const { parts, path: canonical } = canonicalizePath(path); + // Cache + single-statement CTE serve only cache-eligible reads: + // follow-symlinks resolutions outside a transaction. Everything else + // uses the per-component loop: + // * followSymlinks:false (lstat / readlink / the provider's + // pre-mutation captures) — not cached, and the loop is cheaper + // for these shallow one-shot resolves than the recursive CTE. + // * inside a transaction (every mutation path) — resolves are + // shallow and hot, the CTE competes with the mutation's own + // statements for the plan cache (recompiling it is far dearer + // than the loop), and the cache must not be populated + // mid-transaction anyway (rollback safety). + // Mutations still invalidate the cache; that is independent of this. + if (!followFinal || db.inTransaction) { + return resolveParts(db, parts, followFinal, 0); + } + // Repeat reads of the same path are served from the per-Database + // cache. Only the path -> inode mapping is cached; re-read the node + // row so mode/size/mtime/type are always current. A stale mapping + // (inode reaped without invalidation) reads back null and falls + // through to a full resolve that re-populates the cache. + const hit = lookupResolveCache(db, canonical); + if (hit !== undefined) { + if (hit.kind === "negative") { + return null; + } + const node = readNode(db, hit.inode); + if (node !== null) { + return toResolved(node); + } + } + // One recursive-CTE statement resolves the common symlink-free + // case. Any symlink on the path falls back to the per-component loop, + // which follows links and enforces ELOOP; those resolutions are not + // cached (a followed path is an alias whose invalidation can't be + // reasoned about structurally). + const cte = resolveViaCte(db, parts); + if (cte.kind === "symlink") { + return resolveParts(db, parts, followFinal, 0); + } + storeResolveCache(db, canonical, cte.node === null ? null : cte.node.inode); + return cte.node; +} +// Single-statement path walk. Binds the canonical path segments as a +// JSON array and walks vfs_dirents -> vfs_nodes from ROOT_INODE, one +// level per segment. Descends only through directories (WHERE +// w.type = 'dir'), so a file intermediate stalls the walk (ENOTDIR) +// and a missing dirent produces no row (ENOENT) — both surface as a +// missing level-D row, matching the loop's null. Every node the walk +// touches is returned so the caller can detect any symlink and fall +// back. +function resolveViaCte(db, parts) { + const rows = db.all(`WITH RECURSIVE + segs(level, name) AS ( + SELECT key, value FROM json_each(?) + ), + walk(level, inode, type, mode, mtime, size, link_target) AS ( + SELECT 0, n.inode, n.type, n.mode, n.mtime, n.size, n.link_target + FROM vfs_nodes n + WHERE n.inode = ? + UNION ALL + SELECT w.level + 1, n.inode, n.type, n.mode, n.mtime, n.size, n.link_target + FROM walk w + JOIN segs s ON s.level = w.level + JOIN vfs_dirents d ON d.parent_inode = w.inode AND d.name = s.name + JOIN vfs_nodes n ON n.inode = d.child_inode + WHERE w.type = 'dir' + ) + SELECT level, inode, type, mode, mtime, size, link_target + FROM walk + ORDER BY level`, JSON.stringify(parts), ROOT_INODE); + const depth = parts.length; + let target; + for (const row of rows) { + // Any symlink on the walk (root is level 0 and always a dir) means + // the loop must take over to follow it. + if (row.level >= 1 && row.type === "symlink") { + return { kind: "symlink" }; + } + if (row.level === depth) { + target = row; + } + } + return { + kind: "resolved", + node: target === undefined ? null : toResolved(target), + }; +} +function toResolved(node) { + return { + inode: node.inode, + type: node.type, + mode: node.mode, + mtime: node.mtime, + size: node.size, + linkTarget: node.link_target ?? undefined, + }; +} +function resolveParts(db, parts, followFinal, follows) { + const root = readNode(db, ROOT_INODE); + if (root === null) { + return null; + } + let current = root; + for (let i = 0; i < parts.length; i++) { + const isFinal = i === parts.length - 1; + if (current.type !== "dir") { + return null; + } + const child = db.one("SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", current.inode, parts[i]); + if (child === undefined) { + return null; + } + const next = readNode(db, child.child_inode); + if (next === null) { + return null; + } + // Intermediate symlinks always get followed; final-segment symlinks + // are only followed when the caller wants. A dangling intermediate + // is the same as a missing intermediate (return null). + if (next.type === "symlink" && (!isFinal || followFinal)) { + follows += 1; + if (follows > MAX_SYMLINK_FOLLOWS) { + throw createWorkspaceError("ELOOP", "too many symlinks resolving path"); + } + const target = next.link_target ?? ""; + const resolved = resolveParts(db, canonicalizePath(target).parts, true, follows); + if (resolved === null) { + return null; + } + // Replace the current dirent-resolved node with the followed + // result, then keep walking remaining segments (if any). + current = { + inode: resolved.inode, + type: resolved.type, + mode: resolved.mode, + mtime: resolved.mtime, + size: resolved.size, + link_target: resolved.linkTarget ?? null, + }; + continue; + } + current = next; + } + return { + inode: current.inode, + type: current.type, + mode: current.mode, + mtime: current.mtime, + size: current.size, + linkTarget: current.link_target ?? undefined, + }; +} +function readNode(db, inode) { + const row = db.one("SELECT inode, type, mode, mtime, size, link_target FROM vfs_nodes WHERE inode = ?", inode); + return row ?? null; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/resolveCache.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/resolveCache.d.ts new file mode 100644 index 00000000..9ddda6d2 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/resolveCache.d.ts @@ -0,0 +1,12 @@ +import type { Database } from "../storage.js"; +export type ResolveCacheHit = { + kind: "inode"; + inode: number; +} | { + kind: "negative"; +}; +export declare function lookupResolveCache(db: Database, canonicalPath: string): ResolveCacheHit | undefined; +export declare function storeResolveCache(db: Database, canonicalPath: string, inode: number | null): void; +export declare function invalidateResolveExact(db: Database, canonicalPath: string): void; +export declare function invalidateResolveSubtree(db: Database, canonicalPath: string): void; +export declare function clearResolveCache(db: Database): void; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/resolveCache.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/resolveCache.js new file mode 100644 index 00000000..6ae85ce2 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/resolveCache.js @@ -0,0 +1,113 @@ +// Per-Database path -> inode resolution cache. +// +// Maps a canonical absolute path (as produced by +// canonicalizePath().path) to the inode that resolveInode(path, +// { followSymlinks: true }) lands on, or a NEGATIVE marker when the +// path does not resolve. It turns repeat stat/exists/read of the same +// path into an O(1) lookup instead of an O(depth) walk. +// +// Deliberately narrow, for correctness: +// +// * Only the path -> inode MAPPING is cached. resolveInode always +// re-reads the node row on a hit, so content changes (chmod, +// size/mtime, type) are never served stale — only structural +// mutations that move a dirent can invalidate an entry. +// +// * Only symlink-free resolutions are cached. Following a symlink +// makes the cached path an alias of the target whose invalidation +// can't be reasoned about from the path alone, so resolveInode +// stores nothing when a symlink was traversed. +// +// * Population is gated on Database.inTransaction: entries are only +// written outside a transaction, so a rolled-back mutation can +// never leave a positive/negative entry reflecting uncommitted +// state. Mutations invalidate (drop) freely — dropping is safe +// under rollback because the worst case is a recompute. +// +// The cache is per-Database (WeakMap) and bounded (LRU by Map +// insertion order, same discipline as blobCache). +// Sentinel value for "this path resolves to nothing" (ENOENT/ENOTDIR). +const NEGATIVE = -1; +// Upper bound on cached paths per Database. Entries are tiny (a string +// key and a number), so this caps memory at a few MB while covering +// the working set of a busy tree. +const MAX_ENTRIES = 8192; +// Keyed by the Database instance, so correctness assumes exactly one +// Database wraps each SqlStorage. Two Databases over the same storage +// would hold independent caches and could serve each other stale +// results; the DO owns a single Database, which upholds this. +const caches = new WeakMap(); +function cacheFor(db) { + let cache = caches.get(db); + if (cache === undefined) { + cache = new Map(); + caches.set(db, cache); + } + return cache; +} +// Look up a canonical path. Returns undefined on a miss, a positive +// inode hit, or a negative (known-absent) hit. Bumps LRU recency. +export function lookupResolveCache(db, canonicalPath) { + const cache = cacheFor(db); + const value = cache.get(canonicalPath); + if (value === undefined) { + return undefined; + } + // Move to most-recent position for LRU eviction. + cache.delete(canonicalPath); + cache.set(canonicalPath, value); + return value === NEGATIVE ? { kind: "negative" } : { kind: "inode", inode: value }; +} +// Cache a resolution. `inode === null` records a negative entry. No-op +// while a transaction is active so the cache never reflects +// uncommitted state (rollback safety). +export function storeResolveCache(db, canonicalPath, inode) { + if (db.inTransaction) { + return; + } + const cache = cacheFor(db); + cache.set(canonicalPath, inode === null ? NEGATIVE : inode); + while (cache.size > MAX_ENTRIES) { + const oldest = cache.keys().next(); + if (oldest.done === true) { + break; + } + cache.delete(oldest.value); + } +} +// Drop the entry for exactly `canonicalPath`. Use after a mutation +// that changes a single leaf's existence without affecting anything +// beneath it: creating/removing a file, symlink, hardlink, or an +// empty directory. O(1). +export function invalidateResolveExact(db, canonicalPath) { + const cache = caches.get(db); + cache?.delete(canonicalPath); +} +// Drop `canonicalPath` and every entry beneath it (keys prefixed +// `canonicalPath + "/"`). Use when a mutation changes a whole subtree's +// resolution: a recursive delete, any directory rename (every +// descendant's path changes), a structural subtree replacement, or a +// symlink create (paths *through* the new link become resolvable, so +// stale negatives beneath it must go). Root ("/") clears everything. +export function invalidateResolveSubtree(db, canonicalPath) { + const cache = caches.get(db); + if (cache === undefined || cache.size === 0) { + return; + } + if (canonicalPath === "/") { + cache.clear(); + return; + } + cache.delete(canonicalPath); + const prefix = `${canonicalPath}/`; + for (const key of cache.keys()) { + if (key.startsWith(prefix)) { + cache.delete(key); + } + } +} +// Drop the entire cache for a Database. Used by tests and available as +// a blunt reset. +export function clearResolveCache(db) { + caches.get(db)?.clear(); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/rm.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/rm.d.ts new file mode 100644 index 00000000..be9e4701 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/rm.d.ts @@ -0,0 +1,6 @@ +import type { Database } from "../storage.js"; +export interface RmOptions { + recursive?: boolean; + force?: boolean; +} +export declare function rm(db: Database, path: string, options: RmOptions): void; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/rm.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/rm.js new file mode 100644 index 00000000..99eb16cd --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/rm.js @@ -0,0 +1,131 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { incrementRev } from "../rev.js"; +import { recordDelete } from "../sync/changes.js"; +import { pathOf } from "../sync/paths.js"; +import { assertNotReadOnly } from "./mount-guard.js"; +import { resolveInode } from "./resolve.js"; +import { invalidateResolveExact, invalidateResolveSubtree } from "./resolveCache.js"; +import { unlinkDirent } from "./unlink.js"; +// Walk a directory subtree post-order so we delete leaves before +// parents. Yields each node together with the parent inode and name +// the walk already knows, so the caller can unlink the dirent by +// (parent, name) without re-resolving the parent from root. The caller +// appends one tombstone per yielded path and clears vfs_chunks for +// file inodes. +function* walkPostOrder(db, rootInode, rootPath, rootParentInode, rootName) { + const stack = [ + { + inode: rootInode, + path: rootPath, + type: "dir", + parentInode: rootParentInode, + name: rootName, + expanded: false, + }, + ]; + while (stack.length > 0) { + const top = stack[stack.length - 1]; + if (top.type !== "dir" || top.expanded) { + stack.pop(); + yield { + path: top.path, + inode: top.inode, + type: top.type, + parentInode: top.parentInode, + name: top.name, + }; + continue; + } + top.expanded = true; + const children = db.all(`SELECT d.name AS name, d.child_inode AS child_inode, n.type AS type + FROM vfs_dirents d + JOIN vfs_nodes n ON n.inode = d.child_inode + WHERE d.parent_inode = ? + ORDER BY d.name`, top.inode); + for (const child of children) { + const childPath = top.path === "/" ? `/${child.name}` : `${top.path}/${child.name}`; + stack.push({ + inode: child.child_inode, + path: childPath, + type: child.type, + parentInode: top.inode, + name: child.name, + expanded: false, + }); + } + } +} +export function rm(db, path, options) { + const { parts, path: canonical } = canonicalizePath(path); + if (parts.length === 0) { + // The workspace root is structural; refuse to delete it even with + // recursive+force. Matches the doc's example. + throw createWorkspaceError("EPERM", `cannot remove the root directory`, canonical); + } + // assertNotReadOnly uses the symmetric overlap predicate, so a + // recursive rm of an ancestor whose subtree contains a read-only + // mount root is caught here without walking the tree. + assertNotReadOnly(db, canonical); + const force = options.force === true; + const recursive = options.recursive === true; + db.transactionSync(() => { + const node = resolveInode(db, canonical, { followSymlinks: false }); + if (node === null) { + if (force) + return; + throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); + } + if (node.type === "dir" && !recursive) { + const childCount = db.scalar("SELECT COUNT(*) FROM vfs_dirents WHERE parent_inode = ?", node.inode); + if ((childCount ?? 0) > 0) { + throw createWorkspaceError("ENOTEMPTY", `directory not empty: ${canonical}`, canonical); + } + } + // Resolve the entry's real path from its parent rather than from + // the inode: a hardlinked file has several names, and pathOf would + // pick an arbitrary one. Following symlinks on the parent lets a + // request through a symlinked directory land on the real container + // while still removing exactly the requested name. + const name = parts[parts.length - 1]; + const parentPath = parts.length === 1 ? "/" : `/${parts.slice(0, -1).join("/")}`; + const parent = resolveInode(db, parentPath); + if (parent === null || parent.type !== "dir") { + throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); + } + const parentReal = pathOf(db, parent.inode); + if (parentReal === null) { + throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); + } + const realPath = parentReal === "/" ? `/${name}` : `${parentReal}/${name}`; + assertNotReadOnly(db, realPath); + const rev = incrementRev(db); + if (node.type !== "dir" || !recursive) { + // Single entry removal — file, symlink, or empty directory. A + // file inode may have multiple dirents (hardlinks), so remove + // only the requested name and reap chunks/node after the final + // link disappears. `parent` is already resolved above, so unlink + // by (parent, name) directly rather than re-resolving. The + // tombstone is recorded at the resolved real path so sync sees + // the move-aware location. + unlinkDirent(db, parent.inode, name, node.inode, node.type); + recordDelete(db, rev, realPath); + // A single removed entry is a file, symlink, or empty directory: + // no cached descendants to worry about, so drop it exact. + invalidateResolveExact(db, realPath); + return; + } + // Recursive directory removal. Walk leaves first so each delete + // sees an empty parent by the time we get to it. File entries may + // be hardlinked outside this subtree, so delete by path rather + // than by child inode. The walk carries each node's parent inode + // and name, so unlinkDirent needs no per-node re-resolve from root. + for (const entry of walkPostOrder(db, node.inode, realPath, parent.inode, name)) { + unlinkDirent(db, entry.parentInode, entry.name, entry.inode, entry.type); + recordDelete(db, rev, entry.path); + } + // The whole subtree under realPath is gone; one subtree drop covers + // every descendant's cached resolution. + invalidateResolveSubtree(db, realPath); + }); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/stat.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/stat.d.ts new file mode 100644 index 00000000..b4a2670d --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/stat.d.ts @@ -0,0 +1,13 @@ +import type { Database } from "../storage.js"; +export interface WorkspaceStatResult { + name: string; + inode: number; + mode: number; + mtime: number; + size: number; + isFile: boolean; + isDirectory: boolean; + isSymbolicLink: boolean; +} +export declare function stat(db: Database, path: string): WorkspaceStatResult; +export declare function lstat(db: Database, path: string): WorkspaceStatResult; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/stat.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/stat.js new file mode 100644 index 00000000..cee4304d --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/stat.js @@ -0,0 +1,64 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { resolveInode } from "./resolve.js"; +import { getPendingWriteBufferByPath, getWriteBuffer } from "./writeBuffer.js"; +export function stat(db, path) { + return statShared(db, path, true); +} +// Like stat, but does not follow a trailing symlink. Mirrors POSIX +// lstat: the returned size for a symlink is the byte length of the +// stored target, and mode is the symlink node's own mode. +export function lstat(db, path) { + return statShared(db, path, false); +} +function statShared(db, path, followFinal) { + const { name, path: canonical } = canonicalizePath(path); + // Pending-create files have no inode yet; serve the buffer state + // so callers between create and release see the file as it stands. + // Pending creates never apply to symlinks, so this is safe to run + // even on the lstat path — a hit here always corresponds to a + // file mid-open. + const pending = getPendingWriteBufferByPath(db, canonical); + if (pending !== undefined && pending.pending !== undefined) { + return { + name, + // A pending create has no inode until releaseWriteBufferSync + // commits it; report 0, which yields nlink 1 in the provider. + inode: 0, + mode: pending.mode & 0o7777, + mtime: pending.pending.mtime, + size: pending.size, + isFile: true, + isDirectory: false, + isSymbolicLink: false, + }; + } + const node = resolveInode(db, path, { followSymlinks: followFinal }); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such path: ${path}`, path); + } + const isDirectory = node.type === "dir"; + const isFile = node.type === "file"; + const isSymbolicLink = node.type === "symlink"; + let size = 0; + if (isFile) { + // Prefer the in-memory buffer when an open file has unflushed + // writes; otherwise read the cached size off vfs_nodes that + // resolveInode just loaded for us, no extra SQL. + const buffered = getWriteBuffer(db, node.inode); + size = buffered?.dirty ? buffered.size : node.size; + } + else if (isSymbolicLink) { + size = (node.linkTarget ?? "").length; + } + return { + name, + inode: node.inode, + mode: node.mode, + mtime: node.mtime, + size, + isFile, + isDirectory, + isSymbolicLink, + }; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/symlink.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/symlink.d.ts new file mode 100644 index 00000000..882617a6 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/symlink.d.ts @@ -0,0 +1,2 @@ +import type { Database } from "../storage.js"; +export declare function symlink(db: Database, target: string, path: string, now: () => number): void; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/symlink.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/symlink.js new file mode 100644 index 00000000..ad2aa794 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/symlink.js @@ -0,0 +1,50 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { incrementRev } from "../rev.js"; +import { ROOT_INODE } from "../schema/index.js"; +import { assertNotReadOnly } from "./mount-guard.js"; +import { invalidateResolveSubtree } from "./resolveCache.js"; +// Create a symlink node. The target is stored as-is — it can be a +// relative or absolute path, dangling or live. resolveInode follows +// it transparently when callers walk through this entry. +export function symlink(db, target, path, now) { + const { parts, path: canonical } = canonicalizePath(path); + if (parts.length === 0) { + throw createWorkspaceError("EEXIST", "cannot symlink onto root", canonical); + } + assertNotReadOnly(db, canonical); + db.transactionSync(() => { + // Walk to the parent dirent. Intermediate segments must be real + // directories; we don't auto-create them. + let parentInode = ROOT_INODE; + for (let i = 0; i < parts.length - 1; i++) { + const child = db.one("SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", parentInode, parts[i]); + if (child === undefined) { + throw createWorkspaceError("ENOENT", `parent directory missing: ${canonical}`, canonical); + } + const next = db.one("SELECT inode, type FROM vfs_nodes WHERE inode = ?", child.child_inode); + if (next === undefined || next.type !== "dir") { + throw createWorkspaceError("ENOTDIR", `parent path segment is not a directory: ${canonical}`, canonical); + } + parentInode = next.inode; + } + const leafName = parts[parts.length - 1]; + const existing = db.one("SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", parentInode, leafName); + if (existing !== undefined) { + throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); + } + const rev = incrementRev(db); + const mtime = now(); + // RETURNING folds the rowid read into the INSERT. + const row = db.one("INSERT INTO vfs_nodes (type, mode, mtime, rev, link_target) VALUES ('symlink', ?, ?, ?, ?) RETURNING inode", 0o777, mtime, rev, target); + if (row === undefined) { + throw createWorkspaceError("EIO", "failed to allocate inode"); + } + const inode = row.inode; + db.run("INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", parentInode, leafName, inode); + // Subtree, not exact: paths *through* the new link (e.g. /s/x when + // /s -> a populated dir) now resolve, so any cached negative + // beneath the link must be dropped. + invalidateResolveSubtree(db, canonical); + }); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/unlink.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/unlink.d.ts new file mode 100644 index 00000000..d6b654e6 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/unlink.d.ts @@ -0,0 +1,4 @@ +import type { Database } from "../storage.js"; +type NodeType = "file" | "dir" | "symlink"; +export declare function unlinkDirent(db: Database, parentInode: number, name: string, childInode: number, type: NodeType): boolean; +export {}; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/unlink.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/unlink.js new file mode 100644 index 00000000..d3b4fa38 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/unlink.js @@ -0,0 +1,21 @@ +// Remove a single (parent, name) dirent and reap the child inode's +// node and chunk rows only once its last link disappears. A file inode +// can carry several hardlink names, so the node and its chunks survive +// until the final dirent is gone. Returns true when the inode was +// reaped, false when other links keep it alive. +// +// Callers own rev bumps and tombstones; this helper touches only +// vfs_dirents, vfs_chunks, and vfs_nodes. It is the single place the +// refcount-gated reap is implemented — rm, rename, and the sync apply +// path all funnel through here so the invariant lives once. +export function unlinkDirent(db, parentInode, name, childInode, type) { + db.run("DELETE FROM vfs_dirents WHERE parent_inode = ? AND name = ?", parentInode, name); + const remaining = db.scalar("SELECT COUNT(*) FROM vfs_dirents WHERE child_inode = ?", childInode); + if ((remaining ?? 0) > 0) + return false; + if (type === "file") { + db.run("DELETE FROM vfs_chunks WHERE inode = ?", childInode); + } + db.run("DELETE FROM vfs_nodes WHERE inode = ?", childInode); + return true; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/writeBuffer.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/writeBuffer.d.ts new file mode 100644 index 00000000..423f2b2a --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/writeBuffer.d.ts @@ -0,0 +1,23 @@ +import type { Database } from "../storage.js"; +export interface WriteBufferEntry { + buf: Uint8Array; + size: number; + dirty: boolean; + openCount: number; + mode: number; + pending?: { + parentInode: number; + leafName: string; + canonicalPath: string; + pendingInode: number; + mtime: number; + }; +} +export declare function getWriteBuffer(db: Database, inode: number): WriteBufferEntry | undefined; +export declare function getPendingWriteBufferByPath(db: Database, canonicalPath: string): WriteBufferEntry | undefined; +export declare function listPendingByParent(db: Database, parentInode: number): WriteBufferEntry[]; +export declare function setWriteBuffer(db: Database, inode: number, entry: WriteBufferEntry): void; +export declare function deleteWriteBuffer(db: Database, inode: number): void; +export declare function allocatePendingInode(db: Database): number; +export declare function promotePendingToInode(db: Database, pendingInode: number, realInode: number): void; +export declare function ensureCapacity(entry: WriteBufferEntry, needed: number): void; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/writeBuffer.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/writeBuffer.js new file mode 100644 index 00000000..13cb2f5e --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/writeBuffer.js @@ -0,0 +1,93 @@ +// In-process write buffer cache. +// +// Holds per-inode mutable byte buffers between an explicit open and +// release. While a buffer is open, all reads and writes for that +// inode go through the buffer rather than the SQLite blob/chunk +// store. Release commits the bytes to chunks once per file +// and evicts the entry, so per-syscall writes no longer accumulate +// orphan blob rows in the store. +// +// The cache is keyed by Database so a fresh database (a test, a +// rebooted DO incarnation) starts with an empty cache. +const caches = new WeakMap(); +function cacheFor(db) { + let cache = caches.get(db); + if (cache === undefined) { + cache = { byInode: new Map(), byPendingPath: new Map(), nextPendingInode: -1 }; + caches.set(db, cache); + } + return cache; +} +export function getWriteBuffer(db, inode) { + return caches.get(db)?.byInode.get(inode); +} +export function getPendingWriteBufferByPath(db, canonicalPath) { + return caches.get(db)?.byPendingPath.get(canonicalPath); +} +// List pending-create buffers whose parent dirent matches `parentInode`. +// Used by readdir so freshly-created-but-not-yet-released files show +// up in directory listings between open and release. +export function listPendingByParent(db, parentInode) { + const cache = caches.get(db); + if (cache === undefined) + return []; + const out = []; + for (const entry of cache.byPendingPath.values()) { + if (entry.pending?.parentInode === parentInode) + out.push(entry); + } + return out; +} +export function setWriteBuffer(db, inode, entry) { + const cache = cacheFor(db); + cache.byInode.set(inode, entry); + if (entry.pending !== undefined) { + cache.byPendingPath.set(entry.pending.canonicalPath, entry); + } +} +export function deleteWriteBuffer(db, inode) { + const cache = caches.get(db); + if (cache === undefined) + return; + const entry = cache.byInode.get(inode); + if (entry?.pending !== undefined) { + cache.byPendingPath.delete(entry.pending.canonicalPath); + } + cache.byInode.delete(inode); +} +// Allocate a synthetic negative inode id for a pending file. The +// real id is assigned by SQLite when release INSERTs the node row; +// the synthetic value just lets the buffer cache key entries +// before that point. +export function allocatePendingInode(db) { + const cache = cacheFor(db); + const next = cache.nextPendingInode; + cache.nextPendingInode -= 1; + return next; +} +// Re-key a pending entry to the real inode assigned by SQLite at +// commit time, dropping the pending-path index. +export function promotePendingToInode(db, pendingInode, realInode) { + const cache = caches.get(db); + if (cache === undefined) + return; + const entry = cache.byInode.get(pendingInode); + if (entry === undefined) + return; + if (entry.pending !== undefined) { + cache.byPendingPath.delete(entry.pending.canonicalPath); + entry.pending = undefined; + } + cache.byInode.delete(pendingInode); + cache.byInode.set(realInode, entry); +} +export function ensureCapacity(entry, needed) { + if (entry.buf.byteLength >= needed) + return; + let cap = Math.max(entry.buf.byteLength * 2, 64 * 1024); + while (cap < needed) + cap *= 2; + const next = new Uint8Array(cap); + next.set(entry.buf.subarray(0, entry.size), 0); + entry.buf = next; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/writeFile.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/writeFile.d.ts new file mode 100644 index 00000000..a844457d --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/writeFile.d.ts @@ -0,0 +1,37 @@ +import type { Database } from "../storage.js"; +export declare const CHUNK_SIZE: number; +export type WriteFileContent = string | Uint8Array | ReadableStream; +export interface WriteFileOptions { + mode?: number; + /** Fail with EEXIST when the target already exists. */ + exclusive?: boolean; +} +export interface WriteFileRange { + start: number; + end: number; +} +interface PreparedChunk { + hash: Uint8Array; + bytes: Uint8Array; + size: number; +} +export declare function chunksOf(bytes: Uint8Array): PreparedChunk[]; +export declare function writeFile(db: Database, path: string, content: WriteFileContent, options: WriteFileOptions, now: () => number): Promise; +export declare function createFileSync(db: Database, path: string, options: WriteFileOptions, now: () => number): void; +export declare function openWriteBufferSync(db: Database, path: string): void; +export declare function openWriteBufferForCreateSync(db: Database, path: string, options: WriteFileOptions, now: () => number): void; +export declare function releaseWriteBufferSync(db: Database, path: string, now: () => number): void; +/** + * @internal + * Bridges a pending-create write buffer into the SQL world ahead of a + * dirent-mutating provider operation (link, rename, unlink). Leaves + * the open count untouched so a still-open handle keeps writing into + * the now-promoted buffer. Returns true when a pending buffer was + * committed. External callers should never invoke this directly. + */ +export declare function flushPendingByPath(db: Database, path: string, now: () => number): boolean; +export declare function writeRangeSync(db: Database, path: string, bytes: Uint8Array, offset: number, options: WriteFileOptions, now: () => number): number; +export declare function truncateFileSync(db: Database, path: string, size: number, now: () => number): void; +export declare function writeFileSync(db: Database, path: string, bytes: Uint8Array, options: WriteFileOptions, now: () => number): void; +export declare function writeFileRangesSync(db: Database, path: string, bytes: Uint8Array, dirtyRanges: WriteFileRange[], options: WriteFileOptions, now: () => number): void; +export {}; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/writeFile.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/writeFile.js new file mode 100644 index 00000000..820a4bcf --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/fs/writeFile.js @@ -0,0 +1,743 @@ +import { createHash } from "node:crypto"; +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { incrementRev } from "../rev.js"; +import { ROOT_INODE } from "../schema/index.js"; +import { stageBlob } from "../sync/blobs.js"; +import { buildManifest } from "../sync/manifests.js"; +import { getBlobBytes } from "./blobCache.js"; +import { assertNotReadOnly } from "./mount-guard.js"; +import { invalidateResolveExact } from "./resolveCache.js"; +import { allocatePendingInode, deleteWriteBuffer, ensureCapacity as ensureBufferCapacity, getPendingWriteBufferByPath, getWriteBuffer, promotePendingToInode, setWriteBuffer, } from "./writeBuffer.js"; +// Fixed chunk size. Exported so tests can size inputs precisely +// without hard-coding the magic number twice. +export const CHUNK_SIZE = 512 * 1024; +// Resolve directory-only paths (the parent of the target file). The +// final segment is handled by the caller. Returns the parent inode or +// throws ENOENT/ENOTDIR. +function resolveParent(db, parts, canonical) { + let parentInode = ROOT_INODE; + for (let i = 0; i < parts.length - 1; i++) { + const name = parts[i]; + const child = db.one("SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", parentInode, name); + if (child === undefined) { + throw createWorkspaceError("ENOENT", `parent directory missing: ${canonical}`, canonical); + } + const next = db.one("SELECT inode, type FROM vfs_nodes WHERE inode = ?", child.child_inode); + if (next === undefined) { + throw createWorkspaceError("ENOENT", `dangling dirent: ${canonical}`, canonical); + } + if (next.type !== "dir") { + throw createWorkspaceError("ENOTDIR", `parent path segment is not a directory: ${canonical}`, canonical); + } + parentInode = next.inode; + } + return parentInode; +} +async function materialize(content) { + if (typeof content === "string") { + return new TextEncoder().encode(content); + } + return content; +} +// sha256 with a synchronous code path so writeFile can be called both +// from async drivers (the FS API) and from sync drivers (the +// VirtualProvider). node:crypto is available natively on Node and +// polyfilled by workerd. +function sha256(bytes) { + const hash = createHash("sha256"); + hash.update(bytes); + return new Uint8Array(hash.digest()); +} +export function chunksOf(bytes) { + const chunks = []; + for (let offset = 0; offset < bytes.byteLength; offset += CHUNK_SIZE) { + const end = Math.min(offset + CHUNK_SIZE, bytes.byteLength); + // subarray (not slice) avoids an extra copy; sha256() takes its own + // copy when needed. + const slice = bytes.subarray(offset, end); + const hash = sha256(slice); + chunks.push({ hash, bytes: slice, size: slice.byteLength }); + } + return chunks; +} +export async function writeFile(db, path, content, options, now) { + if (content instanceof ReadableStream) { + await writeFileStreaming(db, path, content, options, now); + return; + } + const bytes = await materialize(content); + writeFileSync(db, path, bytes, options, now); +} +// Streaming write path. Reads the source one source-chunk at a time, +// re-windows into fixed CHUNK_SIZE pieces, hashes each window, and +// stages it into vfs_blobs / vfs_blob_bytes as it goes. The final +// inode / dirent / vfs_chunks / manifest writes happen in a single +// short transaction once the source is drained, against a list of +// {hash, size} entries that's O(file_size / CHUNK_SIZE) bytes — not +// O(file_size). +// +// Failure mid-stream leaves blob rows behind; gc() reaps orphans on +// its next pass since no node references them. +async function writeFileStreaming(db, path, source, options, now) { + const { parts, path: canonical } = canonicalizePath(path); + if (parts.length === 0) { + throw createWorkspaceError("EISDIR", "cannot write to the root directory", canonical); + } + // Reject before we stage any blob bytes so known failures do not grow + // orphan blob rows that gc() then has to reap. + assertNotReadOnly(db, canonical); + if (options.exclusive) { + const parentInode = resolveParent(db, parts, canonical); + const existing = db.one("SELECT 1 FROM vfs_dirents WHERE parent_inode = ? AND name = ?", parentInode, parts[parts.length - 1]); + if (existing !== undefined) { + throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); + } + } + const mode = (options.mode ?? 0o644) & 0o7777; + const mtime = now(); + const chunkRefs = []; + // Carry-over buffer: bytes left over from the previous source chunk + // that didn't fill a CHUNK_SIZE window. + let carry; + const flush = (chunk) => { + const hash = sha256(chunk); + stageBlob(db, hash, chunk, mtime); + chunkRefs.push({ hash, size: chunk.byteLength }); + }; + const reader = source.getReader(); + try { + while (true) { + const { value, done } = await reader.read(); + if (done) + break; + if (value === undefined || value.byteLength === 0) + continue; + let input = value; + if (carry !== undefined) { + // Splice carry-over onto the front of this source chunk so + // we can re-window cleanly. + const merged = new Uint8Array(carry.byteLength + input.byteLength); + merged.set(carry, 0); + merged.set(input, carry.byteLength); + input = merged; + carry = undefined; + } + let offset = 0; + while (input.byteLength - offset >= CHUNK_SIZE) { + // Copy the window so the staged blob doesn't alias a + // larger backing buffer. + const window = input.slice(offset, offset + CHUNK_SIZE); + flush(window); + offset += CHUNK_SIZE; + } + if (offset < input.byteLength) { + carry = input.slice(offset); + } + } + } + finally { + reader.releaseLock(); + } + if (carry !== undefined && carry.byteLength > 0) { + flush(carry); + } + // Wire up the inode against the staged blobs in one short + // transaction. From this point on the SQL is the same shape as the + // synchronous path — only the chunk-bytes step is skipped because + // stageBlob already landed them above. + db.transactionSync(() => { + const parentInode = resolveParent(db, parts, canonical); + const leafName = parts[parts.length - 1]; + const existing = db.one("SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", parentInode, leafName); + let inode; + if (existing !== undefined) { + if (options.exclusive) { + throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); + } + const node = db.one("SELECT type FROM vfs_nodes WHERE inode = ?", existing.child_inode); + if (node?.type === "dir") { + throw createWorkspaceError("EISDIR", `path is a directory: ${canonical}`, canonical); + } + inode = existing.child_inode; + db.run("DELETE FROM vfs_chunks WHERE inode = ?", inode); + } + else { + inode = insertFileNode(db, mode, mtime); + insertFileDirent(db, parentInode, leafName, inode, canonical); + } + for (let idx = 0; idx < chunkRefs.length; idx++) { + const ref = chunkRefs[idx]; + db.run("INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", inode, idx, ref.hash, ref.size); + } + const manifestHash = buildManifest(db, chunkRefs, mtime); + const rev = incrementRev(db); + let totalSize = 0; + for (const ref of chunkRefs) + totalSize += ref.size; + db.run("UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ?, size = ?, manifest_hash = ? WHERE inode = ?", mode, mtime, rev, totalSize, manifestHash, inode); + }); +} +// Allocate a fresh file inode row with the supplied mode and mtime, +// using SQLite's RETURNING so the new rowid comes back in the same +// statement instead of through a follow-up SELECT last_insert_rowid(). +// Link a freshly created file inode into its parent directory and drop +// any cached negative resolution for the new path. The single choke +// point for every new-file dirent, so the resolve cache stays correct +// on create without touching the overwrite path (which reuses the +// existing inode and dirent, so its resolution is unchanged). A new +// file is a leaf with no descendants, so exact invalidation suffices. +function insertFileDirent(db, parentInode, leafName, childInode, canonicalPath) { + db.run("INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", parentInode, leafName, childInode); + invalidateResolveExact(db, canonicalPath); +} +function insertFileNode(db, mode, mtime) { + const row = db.one("INSERT INTO vfs_nodes (type, mode, mtime, rev) VALUES ('file', ?, ?, 0) RETURNING inode", mode, mtime); + if (row === undefined) { + throw createWorkspaceError("EIO", "failed to allocate inode"); + } + return row.inode; +} +function upsertChunkBlob(db, chunk, lastSeen) { + db.run("INSERT INTO vfs_blobs (hash, size, last_seen) VALUES (?, ?, ?) ON CONFLICT(hash) DO UPDATE SET last_seen = excluded.last_seen", chunk.hash, chunk.size, lastSeen); + db.run("INSERT INTO vfs_blob_bytes (hash, bytes) VALUES (?, ?) ON CONFLICT(hash) DO NOTHING", chunk.hash, chunk.bytes); +} +function replaceChunkRows(db, inode, chunks, manifestTime) { + db.run("DELETE FROM vfs_chunks WHERE inode = ?", inode); + for (let idx = 0; idx < chunks.length; idx++) { + const chunk = chunks[idx]; + db.run("INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", inode, idx, chunk.hash, chunk.size); + } + return buildManifest(db, chunks, manifestTime); +} +function rangesOverlap(start, end, ranges) { + for (const range of ranges) { + if (range.start < end && start < range.end) + return true; + } + return false; +} +function normalizeRanges(ranges, size) { + const normalized = ranges + .map((range) => ({ + start: Math.max(0, Math.min(size, Math.floor(range.start))), + end: Math.max(0, Math.min(size, Math.ceil(range.end))), + })) + .filter((range) => range.start < range.end) + .sort((a, b) => a.start - b.start); + const merged = []; + for (const range of normalized) { + const previous = merged.at(-1); + if (previous === undefined || previous.end < range.start) { + merged.push({ ...range }); + } + else { + previous.end = Math.max(previous.end, range.end); + } + } + return merged; +} +function existingChunkRefs(db, inode) { + return db.all("SELECT hash, size FROM vfs_chunks WHERE inode = ? ORDER BY idx", inode); +} +function fileSizeForInode(db, inode) { + return db.scalar("SELECT size FROM vfs_nodes WHERE inode = ?", inode) ?? 0; +} +function readChunkBytes(db, inode, idx) { + const chunk = db.one("SELECT hash FROM vfs_chunks WHERE inode = ? AND idx = ?", inode, idx); + if (chunk === undefined) + return new Uint8Array(); + const bytes = getBlobBytes(db, chunk.hash); + if (bytes === undefined) { + throw createWorkspaceError("EIO", "missing blob bytes"); + } + return bytes; +} +function resolveFileInode(db, path) { + const { path: canonical } = canonicalizePath(path); + const node = db.one(`SELECT n.inode AS inode, n.type AS type, n.mode AS mode + FROM vfs_nodes n + WHERE n.inode = ( + SELECT child_inode + FROM vfs_dirents + WHERE parent_inode = ? AND name = ? + )`, ...parentAndNameForResolvedPath(db, path)); + if (node === undefined) { + throw createWorkspaceError("ENOENT", `no such file: ${canonical}`, canonical); + } + if (node.type !== "file") { + throw createWorkspaceError("EISDIR", `path is a directory: ${canonical}`, canonical); + } + return { inode: node.inode, mode: node.mode }; +} +function parentAndNameForResolvedPath(db, path) { + const { parts, path: canonical } = canonicalizePath(path); + if (parts.length === 0) { + throw createWorkspaceError("EISDIR", "cannot write to the root directory", canonical); + } + return [resolveParent(db, parts, canonical), parts[parts.length - 1]]; +} +// Update an inode's chunk-backed representation in place. Iterates over +// the full chunk grid but only touches `vfs_chunks` rows whose contents +// or size actually changed, so untouched chunk rows keep their +// rowids and the surrounding rows do not churn. The manifest is +// invalidated rather than recomputed; sync rebuilds it lazily. +function applyChunkedInodeUpdate(db, inode, size, mode, mtime, isTouched, buildChunkBytes) { + const oldChunks = existingChunkRefs(db, inode); + const chunkCount = Math.ceil(size / CHUNK_SIZE); + const oldChunkCount = oldChunks.length; + for (let idx = 0; idx < chunkCount; idx++) { + const start = idx * CHUNK_SIZE; + const end = Math.min(start + CHUNK_SIZE, size); + const intendedSize = end - start; + const old = oldChunks[idx]; + const touched = isTouched(idx, start, end); + // Stable chunk: existed before with the same logical size and the + // caller did not flag it as touched. Skip without issuing SQL so + // its rowid stays put. + if (old !== undefined && old.size === intendedSize && !touched) + continue; + const existingBytes = old !== undefined ? readChunkBytes(db, inode, idx) : new Uint8Array(); + const chunkBytes = buildChunkBytes(idx, start, end, existingBytes); + if (chunkBytes.byteLength !== intendedSize) { + throw createWorkspaceError("EIO", "chunk builder returned wrong size"); + } + const chunk = { hash: sha256(chunkBytes), bytes: chunkBytes, size: chunkBytes.byteLength }; + upsertChunkBlob(db, chunk, mtime); + db.run("INSERT OR REPLACE INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", inode, idx, chunk.hash, chunk.size); + } + // Drop any old chunks past the new end of file (shrink case). + if (oldChunkCount > chunkCount) { + db.run("DELETE FROM vfs_chunks WHERE inode = ? AND idx >= ?", inode, chunkCount); + } + const rev = incrementRev(db); + db.run("UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ?, size = ?, manifest_hash = NULL WHERE inode = ?", mode, mtime, rev, size, inode); +} +export function createFileSync(db, path, options, now) { + const { path: canonical } = canonicalizePath(path); + assertNotReadOnly(db, canonical); + const [parentInode, leafName] = parentAndNameForResolvedPath(db, path); + const mode = (options.mode ?? 0o644) & 0o7777; + const mtime = now(); + db.transactionSync(() => { + const existing = db.one("SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", parentInode, leafName); + if (existing !== undefined) { + throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); + } + const rev = incrementRev(db); + // INSERT with RETURNING folds the last_insert_rowid lookup into + // the same statement, and computing rev up front lets us write + // the node row with its final stamp in one shot. + const row = db.one("INSERT INTO vfs_nodes (type, mode, mtime, rev, manifest_hash) VALUES ('file', ?, ?, ?, NULL) RETURNING inode", mode, mtime, rev); + if (row === undefined) + throw createWorkspaceError("EIO", "failed to allocate inode"); + insertFileDirent(db, parentInode, leafName, row.inode, canonical); + }); +} +// Open a write buffer for an existing file. Subsequent writes, +// truncates, and reads against the same Database operate on the +// buffer instead of the SQLite chunk/blob store. Release commits +// the bytes back to chunks. +export function openWriteBufferSync(db, path) { + const { path: canonical } = canonicalizePath(path); + const pending = getPendingWriteBufferByPath(db, canonical); + if (pending !== undefined) { + pending.openCount += 1; + return; + } + const { inode, mode } = resolveFileInode(db, path); + const existing = getWriteBuffer(db, inode); + if (existing !== undefined) { + existing.openCount += 1; + return; + } + setWriteBuffer(db, inode, { + buf: new Uint8Array(0), + size: 0, + dirty: false, + openCount: 1, + mode, + }); +} +// Create a new file lazily: stash a pending-create write buffer +// keyed by path, without touching SQL until release. createFileSync +// + openWriteBufferSync + writes + releaseWriteBufferSync would +// otherwise spend two transactions per file (one INSERT round and +// one chunk-commit round); this collapses them into a single +// INSERT-and-chunks transaction at release time. +// +// Throws EEXIST if a path already resolves to a live node or to +// another pending buffer. +export function openWriteBufferForCreateSync(db, path, options, now) { + const { path: canonical } = canonicalizePath(path); + assertNotReadOnly(db, canonical); + if (getPendingWriteBufferByPath(db, canonical) !== undefined) { + throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); + } + const [parentInode, leafName] = parentAndNameForResolvedPath(db, path); + const existing = db.one("SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", parentInode, leafName); + if (existing !== undefined) { + throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); + } + const mode = (options.mode ?? 0o644) & 0o7777; + const mtime = now(); + const pendingInode = allocatePendingInode(db); + setWriteBuffer(db, pendingInode, { + buf: new Uint8Array(0), + size: 0, + dirty: true, + openCount: 1, + mode, + pending: { parentInode, leafName, canonicalPath: canonical, pendingInode, mtime }, + }); +} +// Release one open of an inode's write buffer. When the open count +// reaches zero, commit the buffered bytes to chunk rows and drop +// the entry. The committed mode is the buffer's mode at release +// time so an intermediate chmod survives. Pending-create entries +// emit their INSERT + dirent + chunks in the same transaction. +export function releaseWriteBufferSync(db, path, now) { + const { path: canonical } = canonicalizePath(path); + const pending = getPendingWriteBufferByPath(db, canonical); + if (pending !== undefined) { + releasePendingBuffer(db, pending, now); + return; + } + const node = resolveFileInode(db, path); + const entry = getWriteBuffer(db, node.inode); + if (entry === undefined) + return; + entry.openCount -= 1; + if (entry.openCount > 0) + return; + if (!entry.dirty) { + deleteWriteBuffer(db, node.inode); + return; + } + const mtime = now(); + const mode = entry.mode & 0o7777; + const buffered = entry.buf.subarray(0, entry.size); + db.transactionSync(() => { + if (entry.size === 0) { + // An empty file owns no chunk rows; clear any old ones the + // buffer would otherwise have replaced and bump metadata. + db.run("DELETE FROM vfs_chunks WHERE inode = ?", node.inode); + const rev = incrementRev(db); + db.run("UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ?, size = 0, manifest_hash = NULL WHERE inode = ?", mode, mtime, rev, node.inode); + return; + } + applyChunkedInodeUpdate(db, node.inode, entry.size, mode, mtime, (_idx, start, end) => start < entry.size && end > 0, (_idx, start, end) => buffered.subarray(start, Math.min(end, entry.size))); + }); + deleteWriteBuffer(db, node.inode); +} +// Commit a pending-create buffer to SQLite. Returns the real inode +// allocated by the INSERT, or throws. Promotes the cache entry's key +// from the synthetic pending id to the real inode so subsequent +// reads/writes through the inode-keyed cache still see the same +// buffer. Caller owns the lifecycle of the now-promoted entry. +function commitPendingBuffer(db, entry, now) { + if (entry.pending === undefined) { + throw createWorkspaceError("EIO", "commitPendingBuffer called on non-pending entry"); + } + const { parentInode, leafName, canonicalPath, pendingInode } = entry.pending; + const mtime = now(); + const mode = entry.mode & 0o7777; + const buffered = entry.buf.subarray(0, entry.size); + let realInode = 0; + try { + db.transactionSync(() => { + // Re-check at commit time: a non-buffered writeFile or another + // out-of-band path could have landed between open and release. + const collision = db.one("SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", parentInode, leafName); + if (collision !== undefined) { + throw createWorkspaceError("EEXIST", `path exists at commit time: ${canonicalPath}`, canonicalPath); + } + const rev = incrementRev(db); + const row = db.one("INSERT INTO vfs_nodes (type, mode, mtime, rev, size, manifest_hash) VALUES ('file', ?, ?, ?, ?, NULL) RETURNING inode", mode, mtime, rev, entry.size); + if (row === undefined) { + throw createWorkspaceError("EIO", "failed to allocate inode"); + } + insertFileDirent(db, parentInode, leafName, row.inode, canonicalPath); + if (entry.size > 0) { + const inode = row.inode; + const chunkCount = Math.ceil(entry.size / CHUNK_SIZE); + for (let idx = 0; idx < chunkCount; idx++) { + const start = idx * CHUNK_SIZE; + const end = Math.min(start + CHUNK_SIZE, entry.size); + const chunkBytes = buffered.subarray(start, end); + const chunk = { + hash: sha256(chunkBytes), + bytes: chunkBytes, + size: chunkBytes.byteLength, + }; + upsertChunkBlob(db, chunk, mtime); + db.run("INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", inode, idx, chunk.hash, chunk.size); + } + } + realInode = row.inode; + }); + } + catch (error) { + // Transaction rolled back; drop the buffer so the next caller + // starts clean. + deleteWriteBuffer(db, pendingInode); + throw error; + } + promotePendingToInode(db, pendingInode, realInode); + return realInode; +} +/** + * @internal + * Bridges a pending-create write buffer into the SQL world ahead of a + * dirent-mutating provider operation (link, rename, unlink). Leaves + * the open count untouched so a still-open handle keeps writing into + * the now-promoted buffer. Returns true when a pending buffer was + * committed. External callers should never invoke this directly. + */ +export function flushPendingByPath(db, path, now) { + const { path: canonical } = canonicalizePath(path); + const entry = getPendingWriteBufferByPath(db, canonical); + if (entry === undefined || entry.pending === undefined) + return false; + commitPendingBuffer(db, entry, now); + return true; +} +function releasePendingBuffer(db, entry, now) { + if (entry.pending === undefined) + return; + entry.openCount -= 1; + if (entry.openCount > 0) + return; + const inode = commitPendingBuffer(db, entry, now); + // File is closed; drop the now-promoted entry. A subsequent open + // hits the SQL path and gets a fresh buffer if needed. + deleteWriteBuffer(db, inode); +} +// Hydrate a freshly-opened buffer with the inode's current bytes +// the first time we mutate it. Avoids paying the read cost when the +// caller opens a file just to truncate or overwrite it. +function hydrateBufferIfNeeded(db, inode, entry) { + if (entry.dirty) + return; + const existingSize = fileSizeForInode(db, inode); + if (existingSize === 0) { + entry.dirty = true; + return; + } + ensureBufferCapacity(entry, existingSize); + let copied = 0; + for (let idx = 0; copied < existingSize; idx++) { + const chunk = readChunkBytes(db, inode, idx); + if (chunk.byteLength === 0) + break; + entry.buf.set(chunk, copied); + copied += chunk.byteLength; + } + entry.size = existingSize; + entry.dirty = true; +} +export function writeRangeSync(db, path, bytes, offset, options, now) { + const { path: canonical } = canonicalizePath(path); + assertNotReadOnly(db, canonical); + if (!Number.isInteger(offset) || offset < 0) { + throw createWorkspaceError("EINVAL", `invalid write offset: ${offset}`, canonical); + } + if (bytes.byteLength === 0) + return 0; + const mtime = now(); + // Pending-create files don't have an inode yet; route the write + // straight into the path-keyed buffer. + const pending = getPendingWriteBufferByPath(db, canonical); + if (pending !== undefined) { + const writeEnd = offset + bytes.byteLength; + ensureBufferCapacity(pending, writeEnd); + if (offset > pending.size) { + pending.buf.fill(0, pending.size, offset); + } + pending.buf.set(bytes, offset); + if (writeEnd > pending.size) + pending.size = writeEnd; + pending.mode = (options.mode ?? pending.mode) & 0o7777; + pending.dirty = true; + return bytes.byteLength; + } + const { inode, mode: existingMode } = resolveFileInode(db, path); + const mode = (options.mode ?? existingMode) & 0o7777; + const buffered = getWriteBuffer(db, inode); + // Buffered path: mutate the in-memory bytes and defer storage + // writes until release. Reads through the same Database see the + // buffer's current bytes via readRangeSync's buffer check. + if (buffered !== undefined) { + hydrateBufferIfNeeded(db, inode, buffered); + const writeEnd = offset + bytes.byteLength; + ensureBufferCapacity(buffered, writeEnd); + if (offset > buffered.size) { + buffered.buf.fill(0, buffered.size, offset); + } + buffered.buf.set(bytes, offset); + if (writeEnd > buffered.size) + buffered.size = writeEnd; + buffered.mode = mode; + buffered.dirty = true; + return bytes.byteLength; + } + db.transactionSync(() => { + const oldSize = fileSizeForInode(db, inode); + const writeEnd = offset + bytes.byteLength; + const nextSize = Math.max(oldSize, writeEnd); + applyChunkedInodeUpdate(db, inode, nextSize, mode, mtime, (_idx, start, end) => offset < end && start < writeEnd, (_idx, start, end, existing) => { + const chunkBytes = new Uint8Array(end - start); + chunkBytes.set(existing.subarray(0, Math.min(existing.byteLength, chunkBytes.byteLength))); + if (offset < end && start < writeEnd) { + const copyStart = Math.max(start, offset); + const copyEnd = Math.min(end, writeEnd); + chunkBytes.set(bytes.subarray(copyStart - offset, copyEnd - offset), copyStart - start); + } + return chunkBytes; + }); + }); + return bytes.byteLength; +} +export function truncateFileSync(db, path, size, now) { + const { path: canonical } = canonicalizePath(path); + assertNotReadOnly(db, canonical); + if (!Number.isInteger(size) || size < 0) { + throw createWorkspaceError("EINVAL", `invalid truncate size: ${size}`, canonical); + } + const mtime = now(); + // Pending-create files truncate in-place on the path-keyed buffer. + const pending = getPendingWriteBufferByPath(db, canonical); + if (pending !== undefined) { + if (size > pending.size) { + ensureBufferCapacity(pending, size); + pending.buf.fill(0, pending.size, size); + } + pending.size = size; + pending.dirty = true; + return; + } + const { inode, mode } = resolveFileInode(db, path); + const buffered = getWriteBuffer(db, inode); + if (buffered !== undefined) { + hydrateBufferIfNeeded(db, inode, buffered); + if (size > buffered.size) { + ensureBufferCapacity(buffered, size); + buffered.buf.fill(0, buffered.size, size); + } + buffered.size = size; + buffered.dirty = true; + return; + } + db.transactionSync(() => { + const oldSize = fileSizeForInode(db, inode); + if (oldSize === size) + return; + if (size === 0) { + db.run("DELETE FROM vfs_chunks WHERE inode = ?", inode); + const rev = incrementRev(db); + db.run("UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ?, size = 0, manifest_hash = NULL WHERE inode = ?", mode, mtime, rev, inode); + return; + } + applyChunkedInodeUpdate(db, inode, size, mode, mtime, () => false, (_idx, start, end, existing) => { + const chunkBytes = new Uint8Array(end - start); + chunkBytes.set(existing.subarray(0, Math.min(existing.byteLength, chunkBytes.byteLength))); + return chunkBytes; + }); + }); +} +// Synchronous entry point used by the VirtualProvider. Identical SQL +// to the async path; differs only in that the bytes have already been +// materialized. +export function writeFileSync(db, path, bytes, options, now) { + const { parts, path: canonical } = canonicalizePath(path); + if (parts.length === 0) { + throw createWorkspaceError("EISDIR", "cannot write to the root directory", canonical); + } + assertNotReadOnly(db, canonical); + const mode = (options.mode ?? 0o644) & 0o7777; + const mtime = now(); + db.transactionSync(() => { + const parentInode = resolveParent(db, parts, canonical); + const leafName = parts[parts.length - 1]; + const existing = db.one("SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", parentInode, leafName); + let inode; + if (existing !== undefined) { + if (options.exclusive) { + throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); + } + const node = db.one("SELECT type FROM vfs_nodes WHERE inode = ?", existing.child_inode); + if (node?.type === "dir") { + throw createWorkspaceError("EISDIR", `path is a directory: ${canonical}`, canonical); + } + inode = existing.child_inode; + // Replace the existing representation. Orphaned blobs (if any) + // are cleaned up by a later gc() pass. + db.run("DELETE FROM vfs_chunks WHERE inode = ?", inode); + } + else { + inode = insertFileNode(db, mode, mtime); + insertFileDirent(db, parentInode, leafName, inode, canonical); + } + const rev = incrementRev(db); + const chunks = chunksOf(bytes); + // Upsert blobs and write the new chunk list. + for (let idx = 0; idx < chunks.length; idx++) { + const chunk = chunks[idx]; + upsertChunkBlob(db, chunk, mtime); + db.run("INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", inode, idx, chunk.hash, chunk.size); + } + const manifestHash = buildManifest(db, chunks, mtime); + db.run("UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ?, size = ?, manifest_hash = ? WHERE inode = ?", mode, mtime, rev, bytes.byteLength, manifestHash, inode); + }); +} +export function writeFileRangesSync(db, path, bytes, dirtyRanges, options, now) { + const { parts, path: canonical } = canonicalizePath(path); + if (parts.length === 0) { + throw createWorkspaceError("EISDIR", "cannot write to the root directory", canonical); + } + assertNotReadOnly(db, canonical); + const mode = (options.mode ?? 0o644) & 0o7777; + const ranges = normalizeRanges(dirtyRanges, bytes.byteLength); + const mtime = now(); + db.transactionSync(() => { + const parentInode = resolveParent(db, parts, canonical); + const leafName = parts[parts.length - 1]; + const existing = db.one("SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", parentInode, leafName); + let inode; + let oldChunks = []; + if (existing !== undefined) { + const node = db.one("SELECT type FROM vfs_nodes WHERE inode = ?", existing.child_inode); + if (node?.type === "dir") { + throw createWorkspaceError("EISDIR", `path is a directory: ${canonical}`, canonical); + } + inode = existing.child_inode; + oldChunks = existingChunkRefs(db, inode); + } + else { + inode = insertFileNode(db, mode, mtime); + insertFileDirent(db, parentInode, leafName, inode, canonical); + } + const rev = incrementRev(db); + const nextChunks = []; + const chunkCount = Math.ceil(bytes.byteLength / CHUNK_SIZE); + for (let idx = 0; idx < chunkCount; idx++) { + const start = idx * CHUNK_SIZE; + const end = Math.min(start + CHUNK_SIZE, bytes.byteLength); + const size = end - start; + const oldChunk = oldChunks[idx]; + if (oldChunk !== undefined && oldChunk.size === size && !rangesOverlap(start, end, ranges)) { + nextChunks.push(oldChunk); + continue; + } + const chunk = { + hash: sha256(bytes.subarray(start, end)), + bytes: bytes.subarray(start, end), + size, + }; + upsertChunkBlob(db, chunk, mtime); + nextChunks.push({ hash: chunk.hash, size: chunk.size }); + } + const manifestHash = replaceChunkRows(db, inode, nextChunks, mtime); + db.run("UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ?, size = ?, manifest_hash = ? WHERE inode = ?", mode, mtime, rev, bytes.byteLength, manifestHash, inode); + }); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/path.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/path.d.ts new file mode 100644 index 00000000..ff251652 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/path.d.ts @@ -0,0 +1,7 @@ +export interface CanonicalPath { + path: string; + parts: string[]; + name: string; + parentPath: string | undefined; +} +export declare function canonicalizePath(path: string): CanonicalPath; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/path.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/path.js new file mode 100644 index 00000000..3cdf5572 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/path.js @@ -0,0 +1,36 @@ +import { invalidPath } from "./errors.js"; +export function canonicalizePath(path) { + if (path.length === 0) { + throw invalidPath(path, "empty"); + } + if (!path.startsWith("/")) { + throw invalidPath(path, "must be absolute"); + } + if (path.includes("\0")) { + throw invalidPath(path, "contains NUL byte"); + } + const parts = []; + for (const part of path.split("/")) { + if (part === "" || part === ".") { + continue; + } + if (part === "..") { + if (parts.length === 0) { + throw invalidPath(path, "escapes root"); + } + parts.pop(); + continue; + } + parts.push(part); + } + const canonical = parts.length === 0 ? "/" : `/${parts.join("/")}`; + const name = parts.length === 0 ? "" : parts[parts.length - 1]; + const parentParts = parts.slice(0, -1); + const parentPath = parts.length === 0 ? undefined : parentParts.length === 0 ? "/" : `/${parentParts.join("/")}`; + return { + path: canonical, + parts, + name, + parentPath, + }; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/rev.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/rev.d.ts new file mode 100644 index 00000000..92183de3 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/rev.d.ts @@ -0,0 +1,2 @@ +import type { Database } from "./storage.js"; +export declare function incrementRev(db: Database): number; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/rev.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/rev.js new file mode 100644 index 00000000..2d4cd9b6 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/rev.js @@ -0,0 +1,19 @@ +// Atomic monotonic rev counter. Every FS mutation (mkdir, writeFile, +// rm, ...) calls incrementRev once per transaction and stamps the returned +// value into vfs_nodes.rev. The sync layer reads vfs_meta.rev as +// currentRev and consumes vfs_changes.rev for tombstones. +// +// Must be called inside a transactionSync — the UPDATE and SELECT +// otherwise race with concurrent mutations. The DO single-writer model +// makes that unlikely in practice, but the contract is "wrap me". +export function incrementRev(db) { + // RETURNING folds the read into the same statement so each mutation + // pays one round-trip instead of two. SQLite has supported it since + // 3.35; both node:sqlite and Cloudflare DO SqlStorage are on newer + // versions. + const row = db.one("UPDATE vfs_meta SET v = v + 1 WHERE k = 'rev' RETURNING v"); + if (row === undefined) { + throw new Error("vfs_meta.rev row missing; was initializeSchema run?"); + } + return row.v; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/core.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/core.d.ts new file mode 100644 index 00000000..b0a4930e --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/core.d.ts @@ -0,0 +1,3 @@ +export declare const SCHEMA_VERSION = 5; +export declare const ROOT_INODE = 1; +export declare const CORE_STATEMENTS: readonly ["CREATE TABLE IF NOT EXISTS vfs_meta (\n k TEXT PRIMARY KEY,\n v INTEGER NOT NULL\n )", "CREATE TABLE IF NOT EXISTS vfs_nodes (\n inode INTEGER PRIMARY KEY AUTOINCREMENT,\n type TEXT NOT NULL CHECK(type IN ('file','dir','symlink')),\n mode INTEGER NOT NULL DEFAULT 493,\n mtime INTEGER NOT NULL,\n rev INTEGER NOT NULL DEFAULT 0,\n mount_root TEXT,\n stub_size INTEGER,\n manifest_hash BLOB,\n link_target TEXT,\n size INTEGER NOT NULL DEFAULT 0\n )", "CREATE TABLE IF NOT EXISTS vfs_dirents (\n parent_inode INTEGER NOT NULL,\n name TEXT NOT NULL,\n child_inode INTEGER NOT NULL,\n PRIMARY KEY (parent_inode, name)\n ) WITHOUT ROWID", "CREATE INDEX IF NOT EXISTS vfs_dirents_by_child ON vfs_dirents(child_inode)", "CREATE INDEX IF NOT EXISTS vfs_nodes_by_rev ON vfs_nodes(rev)", "CREATE INDEX IF NOT EXISTS vfs_nodes_by_manifest_hash\n ON vfs_nodes(manifest_hash) WHERE manifest_hash IS NOT NULL", "CREATE TABLE IF NOT EXISTS vfs_blobs (\n hash BLOB PRIMARY KEY,\n size INTEGER NOT NULL,\n last_seen INTEGER NOT NULL\n )", "CREATE TABLE IF NOT EXISTS vfs_blob_bytes (\n hash BLOB PRIMARY KEY REFERENCES vfs_blobs(hash) ON DELETE CASCADE,\n bytes BLOB NOT NULL\n )", "CREATE TABLE IF NOT EXISTS vfs_chunks (\n inode INTEGER NOT NULL,\n idx INTEGER NOT NULL,\n hash BLOB NOT NULL,\n size INTEGER NOT NULL,\n PRIMARY KEY (inode, idx)\n ) WITHOUT ROWID", "CREATE INDEX IF NOT EXISTS vfs_chunks_by_hash ON vfs_chunks(hash)"]; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/core.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/core.js new file mode 100644 index 00000000..f38c869d --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/core.js @@ -0,0 +1,79 @@ +// Filesystem-side tables. These hold the inode graph and the +// content-addressed blob store. See docs/03_filesystem_schema.md. +// Bumped to 2 when `_vfs_mounts.mode` landed (read-only mount +// enforcement at the data layer). Bumped to 3 when `vfs_nodes` +// gained a cached `size` column so stat() doesn't have to SUM +// chunks on every call. Bumped to 4 when `_vfs_watermark` gained +// a `backend` column so a single workspace can host more than +// one backend with independent sync cursors. Bumped to 5 when +// `vfs_dirents` and `vfs_chunks` became WITHOUT ROWID: their +// composite-PK lookups now read straight from the PK b-tree leaf +// with no rowid indirection, and `child_inode` lives in the +// dirents leaf so the (parent, name) resolve read is covering +// (no separate index needed). See `schema/migrations.ts` for the +// migration list; `sync.ts` carries the fresh-install DDL. +export const SCHEMA_VERSION = 5; +export const ROOT_INODE = 1; +export const CORE_STATEMENTS = [ + `CREATE TABLE IF NOT EXISTS vfs_meta ( + k TEXT PRIMARY KEY, + v INTEGER NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS vfs_nodes ( + inode INTEGER PRIMARY KEY AUTOINCREMENT, + type TEXT NOT NULL CHECK(type IN ('file','dir','symlink')), + mode INTEGER NOT NULL DEFAULT 493, + mtime INTEGER NOT NULL, + rev INTEGER NOT NULL DEFAULT 0, + mount_root TEXT, + stub_size INTEGER, + manifest_hash BLOB, + link_target TEXT, + size INTEGER NOT NULL DEFAULT 0 + )`, + // WITHOUT ROWID: the row lives in the (parent_inode, name) PK + // b-tree leaf, so resolving a path segment reads child_inode + // directly from the leaf — no autoindex -> rowid hop, and no + // separate covering index. Legal here because the PK is composite + // and the table has no AUTOINCREMENT. Existing databases are + // rebuilt by the v4 -> v5 migration in schema/migrations.ts; keep + // this DDL and that migrator's CREATE in lockstep. + `CREATE TABLE IF NOT EXISTS vfs_dirents ( + parent_inode INTEGER NOT NULL, + name TEXT NOT NULL, + child_inode INTEGER NOT NULL, + PRIMARY KEY (parent_inode, name) + ) WITHOUT ROWID`, + `CREATE INDEX IF NOT EXISTS vfs_dirents_by_child ON vfs_dirents(child_inode)`, + `CREATE INDEX IF NOT EXISTS vfs_nodes_by_rev ON vfs_nodes(rev)`, + // gc/manifests checks every manifest row against vfs_nodes via a + // correlated NOT EXISTS (manifest_hash = ?). Without this index + // gc full-scans vfs_nodes per candidate manifest — O(N×M). + // Partial because the column is null on every dir and symlink + // node, and on files until they get their first content write. + `CREATE INDEX IF NOT EXISTS vfs_nodes_by_manifest_hash + ON vfs_nodes(manifest_hash) WHERE manifest_hash IS NOT NULL`, + `CREATE TABLE IF NOT EXISTS vfs_blobs ( + hash BLOB PRIMARY KEY, + size INTEGER NOT NULL, + last_seen INTEGER NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS vfs_blob_bytes ( + hash BLOB PRIMARY KEY REFERENCES vfs_blobs(hash) ON DELETE CASCADE, + bytes BLOB NOT NULL + )`, + // WITHOUT ROWID: clustered on (inode, idx) so a file's chunks are + // stored and scanned in index order straight from the PK leaf. + // Legal here — composite PK, no AUTOINCREMENT. The bytes live in + // vfs_blob_bytes (content-addressed), so these rows stay small, + // which is what WITHOUT ROWID wants. Rebuilt for existing DBs by + // the v4 -> v5 migration; keep in lockstep with that migrator. + `CREATE TABLE IF NOT EXISTS vfs_chunks ( + inode INTEGER NOT NULL, + idx INTEGER NOT NULL, + hash BLOB NOT NULL, + size INTEGER NOT NULL, + PRIMARY KEY (inode, idx) + ) WITHOUT ROWID`, + `CREATE INDEX IF NOT EXISTS vfs_chunks_by_hash ON vfs_chunks(hash)`, +]; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/index.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/index.d.ts new file mode 100644 index 00000000..8e14176a --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/index.d.ts @@ -0,0 +1,3 @@ +import type { Database } from "../storage.js"; +export { ROOT_INODE, SCHEMA_VERSION } from "./core.js"; +export declare function initializeSchema(db: Database, now: () => number): void; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/index.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/index.js new file mode 100644 index 00000000..dc701261 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/index.js @@ -0,0 +1,47 @@ +import { createWorkspaceError } from "../errors.js"; +import { CORE_STATEMENTS, ROOT_INODE, SCHEMA_VERSION } from "./core.js"; +import { runMigrations } from "./migrations.js"; +import { SYNC_STATEMENTS } from "./sync.js"; +export { ROOT_INODE, SCHEMA_VERSION } from "./core.js"; +export function initializeSchema(db, now) { + db.transactionSync(() => { + // 1. Baseline DDL. Every statement is "CREATE TABLE IF NOT + // EXISTS" / "CREATE INDEX IF NOT EXISTS" so this is a no-op + // on already-initialized databases. Fresh databases come out + // of this step at the latest column shape (SCHEMA_VERSION). + for (const statement of CORE_STATEMENTS) { + db.run(statement); + } + for (const statement of SYNC_STATEMENTS) { + db.run(statement); + } + // 2. Read the on-disk schema version. Absent → 0 (very first + // boot of this database). The baseline above just created + // every table at the latest shape, so a 0 → SCHEMA_VERSION + // jump has nothing to migrate. + const storedVersion = db.one("SELECT v FROM vfs_meta WHERE k = ?", "schema_version")?.v; + const onDiskVersion = storedVersion ?? 0; + if (onDiskVersion > SCHEMA_VERSION) { + throw createWorkspaceError("EIO", `Unsupported workspace filesystem schema version ${onDiskVersion}`); + } + // 3. Migrate. Skip when the database is fresh (0) — the + // baseline DDL already shipped the latest shape. Otherwise + // dispatch each registered migrator until we hit the + // target. + if (onDiskVersion > 0 && onDiskVersion < SCHEMA_VERSION) { + runMigrations(db, onDiskVersion, SCHEMA_VERSION); + } + // 4. Stamp the version and seed the boot rows. Both shapes + // (insert-if-missing, then update) keep this idempotent so + // repeat calls do nothing. + db.run("INSERT OR IGNORE INTO vfs_meta (k, v) VALUES (?, ?)", "schema_version", SCHEMA_VERSION); + db.run("UPDATE vfs_meta SET v = ? WHERE k = ?", SCHEMA_VERSION, "schema_version"); + db.run("INSERT OR IGNORE INTO vfs_meta (k, v) VALUES (?, ?)", "rev", 1); + db.run("INSERT OR IGNORE INTO _vfs_watermark (k, backend, v) VALUES (?, 'default', ?)", "pushRev", 0); + db.run("INSERT OR IGNORE INTO _vfs_watermark (k, backend, v) VALUES (?, 'default', ?)", "fetchRev", 0); + db.run("INSERT OR IGNORE INTO _vfs_fetch_cursor (k, backend, path) VALUES (?, 'default', ?)", "fetch", null); + db.run(`INSERT OR IGNORE INTO vfs_nodes + (inode, type, mode, mtime, rev) + VALUES (?, 'dir', ?, ?, 0)`, ROOT_INODE, 0o755, now()); + }); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/migrations.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/migrations.d.ts new file mode 100644 index 00000000..9597ce73 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/migrations.d.ts @@ -0,0 +1,8 @@ +import type { Database } from "../storage.js"; +export interface Migration { + readonly from: number; + readonly to: number; + readonly migrator: (db: Database) => void; +} +export declare const MIGRATIONS: readonly Migration[]; +export declare function runMigrations(db: Database, current: number, target: number): number; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/migrations.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/migrations.js new file mode 100644 index 00000000..524d72eb --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/migrations.js @@ -0,0 +1,138 @@ +// Schema migration runner. +// +// The schema's "CREATE TABLE IF NOT EXISTS" baseline handles fresh +// databases. When a schema column changes shape — added, dropped, +// renamed, retyped — IF NOT EXISTS does nothing and the older +// rows stay incompatible. Migrations close that gap. +// +// Shape: an ordered list of `(from, to, migrator)` tuples. The +// runner reads `vfs_meta.schema_version` (defaulting to 0 when the +// row is absent), picks every migration whose `from === current`, +// runs it, advances `current`, and repeats until `current >= +// SCHEMA_VERSION`. The whole pass runs inside the caller's +// transactionSync so a partial migration rolls back. +// +// Each migrator is a `(db: Database) => void` and may assume the +// previous version's schema is in place. Migrators land schema +// changes only; they don't touch user data unless the column shape +// requires it. +// v1 → v2 — add `_vfs_mounts.mode` so dofs can enforce read-only +// mounts at the data layer. Existing rows default to 'read-only'; +// the workspace re-stamps them with the registered mount's mode on +// the next index pass. +// +// The CHECK constraint is duplicated in `sync.ts`'s fresh-install +// DDL; both paths must keep the same allowed set. +function v1_to_v2_add_mounts_mode(db) { + db.run(`ALTER TABLE _vfs_mounts + ADD COLUMN mode TEXT NOT NULL DEFAULT 'read-only' + CHECK(mode IN ('read-only', 'read-write'))`); +} +// v2 → v3 — denormalise file size onto vfs_nodes so stat doesn't +// have to SUM the chunk rows on every call. The column is +// backfilled from existing vfs_chunks; later writes maintain it. +function v2_to_v3_add_size_column(db) { + const hasColumn = db + .all("PRAGMA table_info(vfs_nodes)") + .some((column) => column.name === "size"); + if (!hasColumn) { + db.run("ALTER TABLE vfs_nodes ADD COLUMN size INTEGER NOT NULL DEFAULT 0"); + } + db.run(`UPDATE vfs_nodes + SET size = COALESCE( + (SELECT SUM(size) FROM vfs_chunks WHERE vfs_chunks.inode = vfs_nodes.inode), + 0 + ) + WHERE type = 'file'`); +} +// v3 → v4 — add a `backend` column to `_vfs_watermark` so a +// workspace can host more than one backend with independent sync +// cursors. SQLite's ALTER TABLE can't change a primary key; copy +// existing rows into a fresh table with the composite +// (k, backend) primary key, then swap the tables. +// +// Existing rows land under the `default` backend id, which the +// dofs sync helpers also use as the fallback when a caller +// doesn't pass an id. Pre-multi-backend workspaces keep their +// pushRev / fetchRev cursors intact through the upgrade. +function v3_to_v4_watermark_backend_column(db) { + db.run(`ALTER TABLE _vfs_watermark RENAME TO _vfs_watermark_v3`); + db.run(`CREATE TABLE _vfs_watermark ( + k TEXT NOT NULL, + backend TEXT NOT NULL DEFAULT 'default', + v INTEGER NOT NULL, + PRIMARY KEY (k, backend) + )`); + db.run(`INSERT INTO _vfs_watermark (k, backend, v) + SELECT k, 'default', v FROM _vfs_watermark_v3`); + db.run(`DROP TABLE _vfs_watermark_v3`); +} +// v4 → v5 — rebuild `vfs_dirents` and `vfs_chunks` as WITHOUT ROWID. +// SQLite can't convert a table to WITHOUT ROWID in place, so for each +// table: rename it aside, create the WITHOUT ROWID replacement, copy +// the rows, drop the old table. +// +// Both targets are FK-inert (neither is an FK parent or child; the +// schema's only foreign key is vfs_blob_bytes -> vfs_blobs) and have +// composite primary keys with no AUTOINCREMENT, so WITHOUT ROWID is +// legal and sqlite_sequence is untouched. `vfs_blob_bytes` is left +// alone on purpose — it holds the large blob payloads and the FK. +// +// A RENAME carries the table's secondary index along to the temp +// name, and the following DROP takes the index with it. The baseline +// `CREATE INDEX IF NOT EXISTS` in initializeSchema already ran, before +// migrations, and does not re-run — so this migrator must recreate +// vfs_dirents_by_child and vfs_chunks_by_hash itself, or upgraded +// databases silently lose them. Keep the CREATE bodies in lockstep +// with the fresh-install DDL in core.ts. +function v4_to_v5_without_rowid(db) { + // vfs_dirents + db.run(`ALTER TABLE vfs_dirents RENAME TO vfs_dirents_v4`); + db.run(`CREATE TABLE vfs_dirents ( + parent_inode INTEGER NOT NULL, + name TEXT NOT NULL, + child_inode INTEGER NOT NULL, + PRIMARY KEY (parent_inode, name) + ) WITHOUT ROWID`); + db.run(`INSERT INTO vfs_dirents (parent_inode, name, child_inode) + SELECT parent_inode, name, child_inode FROM vfs_dirents_v4`); + db.run(`DROP TABLE vfs_dirents_v4`); + db.run(`CREATE INDEX vfs_dirents_by_child ON vfs_dirents(child_inode)`); + // vfs_chunks + db.run(`ALTER TABLE vfs_chunks RENAME TO vfs_chunks_v4`); + db.run(`CREATE TABLE vfs_chunks ( + inode INTEGER NOT NULL, + idx INTEGER NOT NULL, + hash BLOB NOT NULL, + size INTEGER NOT NULL, + PRIMARY KEY (inode, idx) + ) WITHOUT ROWID`); + db.run(`INSERT INTO vfs_chunks (inode, idx, hash, size) + SELECT inode, idx, hash, size FROM vfs_chunks_v4`); + db.run(`DROP TABLE vfs_chunks_v4`); + db.run(`CREATE INDEX vfs_chunks_by_hash ON vfs_chunks(hash)`); +} +export const MIGRATIONS = [ + { from: 1, to: 2, migrator: v1_to_v2_add_mounts_mode }, + { from: 2, to: 3, migrator: v2_to_v3_add_size_column }, + { from: 3, to: 4, migrator: v3_to_v4_watermark_backend_column }, + { from: 4, to: 5, migrator: v4_to_v5_without_rowid }, +]; +// Apply every migration whose `from` matches the current version, +// in order, until we reach the target. The caller has already +// wrapped this in a transactionSync; failures here roll the whole +// initializeSchema call back. +export function runMigrations(db, current, target) { + let version = current; + while (version < target) { + const next = MIGRATIONS.find((m) => m.from === version); + if (next === undefined) { + // No migration registered for this jump. This is a bug — the + // version was bumped without a matching migration. + throw new Error(`dofs schema: no migration registered for v${version} -> v${target}`); + } + next.migrator(db); + version = next.to; + } + return version; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/sync.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/sync.d.ts new file mode 100644 index 00000000..49b80fbb --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/sync.d.ts @@ -0,0 +1 @@ +export declare const SYNC_STATEMENTS: readonly ["CREATE TABLE IF NOT EXISTS vfs_manifests (\n hash BLOB PRIMARY KEY,\n size INTEGER NOT NULL,\n encoded BLOB NOT NULL,\n last_seen INTEGER NOT NULL DEFAULT 0\n )", "CREATE TABLE IF NOT EXISTS vfs_changes (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n rev INTEGER NOT NULL,\n path TEXT NOT NULL,\n op TEXT NOT NULL CHECK(op IN ('delete'))\n )", "CREATE INDEX IF NOT EXISTS vfs_changes_by_rev ON vfs_changes(rev)", "CREATE INDEX IF NOT EXISTS vfs_changes_by_path ON vfs_changes(path, id DESC)", "CREATE TABLE IF NOT EXISTS _vfs_watermark (\n k TEXT NOT NULL,\n backend TEXT NOT NULL DEFAULT 'default',\n v INTEGER NOT NULL,\n PRIMARY KEY (k, backend)\n )", "CREATE TABLE IF NOT EXISTS _vfs_fetch_cursor (\n k TEXT NOT NULL CHECK(k = 'fetch'),\n backend TEXT NOT NULL DEFAULT 'default',\n path TEXT,\n PRIMARY KEY (k, backend)\n )", "CREATE TABLE IF NOT EXISTS _vfs_mounts (\n root TEXT PRIMARY KEY,\n kind TEXT NOT NULL,\n indexed INTEGER NOT NULL DEFAULT 0,\n mode TEXT NOT NULL DEFAULT 'read-only'\n CHECK(mode IN ('read-only', 'read-write'))\n )"]; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/sync.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/sync.js new file mode 100644 index 00000000..9dcc3c6c --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/schema/sync.js @@ -0,0 +1,58 @@ +// Sync-protocol tables. Populated by the sync module; the FS module +// only writes to vfs_changes (via sync/changes.ts) on rm. The rest +// of these tables stay empty until the sync task is implemented. +export const SYNC_STATEMENTS = [ + `CREATE TABLE IF NOT EXISTS vfs_manifests ( + hash BLOB PRIMARY KEY, + size INTEGER NOT NULL, + encoded BLOB NOT NULL, + last_seen INTEGER NOT NULL DEFAULT 0 + )`, + `CREATE TABLE IF NOT EXISTS vfs_changes ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + rev INTEGER NOT NULL, + path TEXT NOT NULL, + op TEXT NOT NULL CHECK(op IN ('delete')) + )`, + `CREATE INDEX IF NOT EXISTS vfs_changes_by_rev ON vfs_changes(rev)`, + // changes.ts looks up the latest op for a path via + // `WHERE path = ? ORDER BY id DESC LIMIT 1`. Without the index + // SQLite falls back to a full scan; with (path, id DESC) the + // lookup is O(log n) and the ORDER BY drains straight from the + // index. Used on every recordDelete and on every push-tick that + // processes tombstones. + `CREATE INDEX IF NOT EXISTS vfs_changes_by_path ON vfs_changes(path, id DESC)`, + // Watermarks are keyed by (k, backend) so a workspace hosting + // multiple backends keeps each backend's sync cursors + // independent. The `backend` column was added at schema v3; + // `schema/migrations.ts` owns the ALTER for existing + // databases. Fresh installs land the composite key directly. + `CREATE TABLE IF NOT EXISTS _vfs_watermark ( + k TEXT NOT NULL, + backend TEXT NOT NULL DEFAULT 'default', + v INTEGER NOT NULL, + PRIMARY KEY (k, backend) + )`, + // The fetch cursor's same-rev `path` component, keyed by + // (k, backend) so each backend resumes a partially-drained rev + // independently. The rev component lives in _vfs_watermark under + // 'fetchRev'; this table only holds the in-rev path. `backend` + // mirrors _vfs_watermark and defaults to 'default'. + `CREATE TABLE IF NOT EXISTS _vfs_fetch_cursor ( + k TEXT NOT NULL CHECK(k = 'fetch'), + backend TEXT NOT NULL DEFAULT 'default', + path TEXT, + PRIMARY KEY (k, backend) + )`, + // The `mode` column was added at schema v2; `schema/migrations.ts` + // owns the ALTER for existing databases. Keep the CHECK + // constraint here aligned with the migration's CHECK so fresh + // installs and upgrades enforce the same allowed set. + `CREATE TABLE IF NOT EXISTS _vfs_mounts ( + root TEXT PRIMARY KEY, + kind TEXT NOT NULL, + indexed INTEGER NOT NULL DEFAULT 0, + mode TEXT NOT NULL DEFAULT 'read-only' + CHECK(mode IN ('read-only', 'read-write')) + )`, +]; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/storage.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/storage.d.ts new file mode 100644 index 00000000..047402c3 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/storage.d.ts @@ -0,0 +1,12 @@ +import type { DurableObjectStorageLike, SQLStorageLike } from "./types.js"; +export declare class Database { + #private; + readonly sql: SQLStorageLike; + readonly transactionSync: (closure: () => T) => T; + constructor(storage: DurableObjectStorageLike); + get inTransaction(): boolean; + run(query: string, ...bindings: unknown[]): void; + all(query: string, ...bindings: unknown[]): Row[]; + one(query: string, ...bindings: unknown[]): Row | undefined; + scalar(query: string, ...bindings: unknown[]): T | undefined; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/storage.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/storage.js new file mode 100644 index 00000000..c5f3b2cb --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/storage.js @@ -0,0 +1,106 @@ +export class Database { + sql; + transactionSync; + // Depth counter so reentrant transactionSync() calls work. The + // outer call uses the storage adapter's transactionSync (or + // BEGIN/COMMIT under the hood); nested calls use SAVEPOINTs + // through sql.exec directly. SQLite forbids a real BEGIN inside + // an active transaction. + #txDepth = 0; + constructor(storage) { + this.sql = storage.sql; + this.transactionSync = (closure) => { + if (this.#txDepth > 0) { + // Reentrant call: use a savepoint. SQLite's RELEASE on a + // savepoint inside an outer transaction commits the inner + // work without ending the outer one. + const sp = `_t${this.#txDepth}`; + this.sql.exec(`SAVEPOINT ${sp}`); + this.#txDepth++; + try { + const result = closure(); + this.sql.exec(`RELEASE ${sp}`); + return result; + } + catch (error) { + this.sql.exec(`ROLLBACK TO ${sp}`); + this.sql.exec(`RELEASE ${sp}`); + throw error; + } + finally { + this.#txDepth--; + } + } + // Outer call: hand off to the storage adapter so the DO + // runtime's transaction semantics apply. + this.#txDepth++; + try { + if (storage.transactionSync !== undefined) { + return storage.transactionSync(closure); + } + if (storage.transaction !== undefined) { + const result = storage.transaction(closure); + if (result !== undefined && + result !== null && + typeof result === "object" && + "then" in result) { + throw new Error("Durable Object storage adapter requires synchronous transactions"); + } + return result; + } + return closure(); + } + finally { + this.#txDepth--; + } + }; + } + // True while a transactionSync closure is on the stack. The resolve + // cache uses this to refuse populating entries mid-transaction, so a + // rolled-back mutation can never leave the cache reflecting + // uncommitted state. (Invalidation still runs freely inside a + // transaction — dropping an entry is always safe.) + // + // Invariant: #txDepth only tracks transactionSync. A raw + // BEGIN/SAVEPOINT issued through run() would open a transaction this + // flag can't see, letting the cache populate mid-transaction and + // survive a rollback — so transactionSync is the only sanctioned way + // to open one. + get inTransaction() { + return this.#txDepth > 0; + } + run(query, ...bindings) { + this.sql.exec(query, ...bindings); + } + all(query, ...bindings) { + const rows = this.sql.exec(query, ...bindings).toArray(); + return rows.map((row) => normalizeRow(row)); + } + one(query, ...bindings) { + return this.all(query, ...bindings)[0]; + } + scalar(query, ...bindings) { + const row = this.one(query, ...bindings); + if (row === undefined) { + return undefined; + } + const [value] = Object.values(row); + return value; + } +} +// Cloudflare's DO SqlStorage returns BLOB columns as ArrayBuffer, +// whereas node:sqlite returns Uint8Array. Normalise to Uint8Array so +// the rest of the code only has to handle one shape. +function normalizeRow(row) { + // node:sqlite hands back rows with a null prototype; the DO SQL + // flavour returns ArrayBuffer for BLOB columns. Re-key into a plain + // {} so consumers get Object.prototype-shaped rows (capnweb's + // serializer keys off Object.prototype to detect "object") and + // convert any ArrayBuffer to Uint8Array in the same pass. + const out = {}; + for (const key of Object.keys(row)) { + const value = row[key]; + out[key] = value instanceof ArrayBuffer ? new Uint8Array(value) : value; + } + return out; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/blobs.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/blobs.d.ts new file mode 100644 index 00000000..39debd94 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/blobs.d.ts @@ -0,0 +1,2 @@ +import type { Database } from "../storage.js"; +export declare function stageBlob(db: Database, hash: Uint8Array, bytes: Uint8Array, now: number): void; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/blobs.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/blobs.js new file mode 100644 index 00000000..3d2a7695 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/blobs.js @@ -0,0 +1,21 @@ +import { clearBlobCache } from "../fs/blobCache.js"; +// Stage a chunk directly into vfs_blobs + vfs_blob_bytes without +// creating a node or a manifest. The receiver-side push path uses +// this to land bytes the sender shipped via pushObjects so a +// subsequent applyChanges call can find them by hash. +// +// Idempotent: a second call with the same hash refreshes +// last_seen so the bytes don't get reaped by an interleaved gc. +// Conflict updates also repair incomplete or size-mismatched rows +// left by an interrupted or corrupt write. +// +// Callers are expected to have verified that hash === sha256(bytes) +// before calling. The function trusts the caller; a mismatched +// pair would silently land under the wrong key. +export function stageBlob(db, hash, bytes, now) { + db.transactionSync(() => { + db.run("INSERT INTO vfs_blobs (hash, size, last_seen) VALUES (?, ?, ?) ON CONFLICT(hash) DO UPDATE SET size = excluded.size, last_seen = excluded.last_seen", hash, bytes.byteLength, now); + db.run("INSERT INTO vfs_blob_bytes (hash, bytes) VALUES (?, ?) ON CONFLICT(hash) DO UPDATE SET bytes = excluded.bytes", hash, bytes); + }); + clearBlobCache(db); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/changes.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/changes.d.ts new file mode 100644 index 00000000..d64fb41a --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/changes.d.ts @@ -0,0 +1,32 @@ +import type { Database } from "../storage.js"; +export declare function recordDelete(db: Database, rev: number, path: string): void; +export type ChangeEntry = { + kind: "file"; + rev: number; + path: string; + mode: number; + mtime: number; + size: number; + chunks: { + hash: Uint8Array; + size: number; + }[]; +} | { + kind: "dir"; + rev: number; + path: string; + mode: number; + mtime: number; +} | { + kind: "symlink"; + rev: number; + path: string; + target: string; + mode: number; + mtime: number; +} | { + kind: "delete"; + rev: number; + path: string; +}; +export declare function materialiseChange(db: Database, path: string): ChangeEntry | null; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/changes.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/changes.js new file mode 100644 index 00000000..c3e703ea --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/changes.js @@ -0,0 +1,66 @@ +import { resolveInode } from "../fs/resolve.js"; +import { canonicalizePath } from "../path.js"; +// Record a tombstone for a deleted path so the next push to the +// container learns the path is gone. Called by fs/rm inside the same +// transaction that bumped rev and removed the inode rows; the caller +// passes the post-bump rev value. +export function recordDelete(db, rev, path) { + db.run("INSERT INTO vfs_changes (rev, path, op) VALUES (?, ?, 'delete')", rev, path); +} +// Read the current state of `path` and turn it into a wire entry. +// Returns null when the path was never touched (no live inode and no +// tombstone in vfs_changes). Live inodes win over tombstones, which +// handles the delete-then-recreate case correctly. +// +// Symlinks are returned as symlink entries; we never follow them on +// the sync wire. Callers that want "the file the link points at" +// resolve it themselves after applying the symlink entry. +export function materialiseChange(db, path) { + const canonical = canonicalizePath(path).path; + const live = resolveInode(db, canonical, { followSymlinks: false }); + if (live !== null) { + // Read the rev stamped on this inode. Used as the per-entry + // cursor on the sync wire; coalesceChanges yields entries in + // ascending rev order so the puller can checkpoint per batch. + const revRow = db.one("SELECT rev FROM vfs_nodes WHERE inode = ?", live.inode); + const rev = revRow?.rev ?? 0; + if (live.type === "dir") { + return { kind: "dir", rev, path: canonical, mode: live.mode, mtime: live.mtime }; + } + if (live.type === "symlink") { + return { + kind: "symlink", + rev, + path: canonical, + target: live.linkTarget ?? "", + mode: live.mode, + mtime: live.mtime, + }; + } + // file: collect chunk rows in index order. Each row carries hash + // and size so the receiver can probe hasObjects without a + // separate manifest lookup. An empty file has zero chunk rows + // and reports size 0. + const chunks = db.all("SELECT hash, size FROM vfs_chunks WHERE inode = ? ORDER BY idx", live.inode); + let size = 0; + for (const c of chunks) + size += c.size; + return { + kind: "file", + rev, + path: canonical, + mode: live.mode, + mtime: live.mtime, + size, + chunks, + }; + } + // No live inode — check for a tombstone. The last row wins if the + // path was deleted and never recreated; an indexed scan by path is + // cheap because vfs_changes is bounded by the watermark window. + const tomb = db.one("SELECT rev, op FROM vfs_changes WHERE path = ? ORDER BY id DESC LIMIT 1", canonical); + if (tomb?.op === "delete") { + return { kind: "delete", rev: tomb.rev, path: canonical }; + } + return null; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/manifests.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/manifests.d.ts new file mode 100644 index 00000000..912fc554 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/manifests.d.ts @@ -0,0 +1,8 @@ +import type { Database } from "../storage.js"; +export interface ManifestChunk { + hash: Uint8Array; + size: number; +} +export declare const MANIFEST_VERSION = 1; +export declare function computeManifestHash(chunks: ManifestChunk[]): Uint8Array; +export declare function buildManifest(db: Database, chunks: ManifestChunk[], now: number): Uint8Array; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/manifests.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/manifests.js new file mode 100644 index 00000000..d202641a --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/manifests.js @@ -0,0 +1,41 @@ +import { createHash } from "node:crypto"; +export const MANIFEST_VERSION = 1; +function toHex(bytes) { + let out = ""; + for (let i = 0; i < bytes.byteLength; i++) { + out += bytes[i].toString(16).padStart(2, "0"); + } + return out; +} +function sha256(bytes) { + return new Uint8Array(createHash("sha256").update(bytes).digest()); +} +// Serialize a chunk list into the canonical manifest bytes. The +// hash is taken over these bytes and the same bytes are stored, so +// producing them once keeps the two in step. +function encodeManifest(chunks) { + const encoded = { + version: MANIFEST_VERSION, + chunks: chunks.map((c) => ({ hash: toHex(c.hash), size: c.size })), + }; + return new TextEncoder().encode(JSON.stringify(encoded)); +} +// Compute the manifest hash for a chunk list without touching the +// DB. Used by the apply path to short-circuit when an upstream +// entry already matches the local node — the manifest hash is +// content-addressed so identical chunks always produce the same +// hash. +export function computeManifestHash(chunks) { + return sha256(encodeManifest(chunks)); +} +// Build a manifest row for the given chunk list. Idempotent: a +// second call with the same chunks no-ops on the UNIQUE(hash). The +// returned hash is what the caller writes onto +// `vfs_nodes.manifest_hash`. +export function buildManifest(db, chunks, now) { + const bytes = encodeManifest(chunks); + const hash = sha256(bytes); + const size = chunks.reduce((acc, c) => acc + c.size, 0); + db.run("INSERT INTO vfs_manifests (hash, size, encoded, last_seen) VALUES (?, ?, ?, ?) ON CONFLICT(hash) DO UPDATE SET last_seen = excluded.last_seen", hash, size, bytes, now); + return hash; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/paths.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/paths.d.ts new file mode 100644 index 00000000..3925c4b2 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/paths.d.ts @@ -0,0 +1,3 @@ +import type { Database } from "../storage.js"; +export declare function pathOf(db: Database, inode: number): string | null; +export declare function pathsOf(db: Database, inode: number): string[]; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/paths.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/paths.js new file mode 100644 index 00000000..ee2916e1 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/sync/paths.js @@ -0,0 +1,41 @@ +import { ROOT_INODE } from "../schema/index.js"; +// Walk vfs_dirents from `inode` up to ROOT_INODE, gathering the path +// segments along the way. Returns null when the inode is unreachable. +export function pathOf(db, inode) { + if (inode === ROOT_INODE) + return "/"; + const segments = []; + let current = inode; + // Bound the walk: a million levels deep is well past any real FS; + // anything beyond that is corruption and should not loop forever. + for (let i = 0; i < 1_000_000; i++) { + const row = db.one("SELECT parent_inode, name FROM vfs_dirents WHERE child_inode = ?", current); + if (row === undefined) + return null; + segments.push(row.name); + if (row.parent_inode === ROOT_INODE) { + segments.reverse(); + return `/${segments.join("/")}`; + } + current = row.parent_inode; + } + return null; +} +// Every path that currently names `inode`. A file may carry several +// hardlink names; pathOf collapses them to one arbitrary name, which +// is wrong for the change stream — every name has to reach the wire so +// the receiver materialises each. Directories cannot be hardlinked, so +// each parent walk is unambiguous. +export function pathsOf(db, inode) { + if (inode === ROOT_INODE) + return ["/"]; + const dirents = db.all("SELECT parent_inode, name FROM vfs_dirents WHERE child_inode = ?", inode); + const paths = []; + for (const { parent_inode, name } of dirents) { + const parent = pathOf(db, parent_inode); + if (parent === null) + continue; + paths.push(parent === "/" ? `/${name}` : `${parent}/${name}`); + } + return paths; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/types.d.ts b/packages/workflow/vendor/cloudflare-computer-dofs/generated/types.d.ts new file mode 100644 index 00000000..b998b8b5 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/types.d.ts @@ -0,0 +1,11 @@ +export interface SQLCursorLike> { + toArray(): Row[]; +} +export interface SQLStorageLike { + exec>(query: string, ...bindings: unknown[]): SQLCursorLike; +} +export interface DurableObjectStorageLike { + sql: SQLStorageLike; + transaction?(closure: () => T | Promise): T | Promise; + transactionSync?(closure: () => T): T; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/generated/types.js b/packages/workflow/vendor/cloudflare-computer-dofs/generated/types.js new file mode 100644 index 00000000..cb0ff5c3 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/generated/types.js @@ -0,0 +1 @@ +export {}; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/errors.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/errors.ts new file mode 100644 index 00000000..1c21257d --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/errors.ts @@ -0,0 +1,36 @@ +export type WorkspaceErrorCode = + | "ENOENT" + | "ENOTEMPTY" + | "ENOTDIR" + | "EISDIR" + | "EEXIST" + | "EINVAL" + | "EACCES" + | "EPERM" + | "EROFS" + | "ENOSYS" + | "EBADF" + | "ELOOP" + | "EUNKNOWN_HASH" + | "EIO"; + +export interface WorkspaceFsError extends Error { + code: WorkspaceErrorCode; + path?: string; +} + +export function createWorkspaceError( + code: WorkspaceErrorCode, + message: string, + path?: string, +): WorkspaceFsError { + const error = new Error(path === undefined ? message : `${message}: ${path}`) as WorkspaceFsError; + error.name = "WorkspaceFsError"; + error.code = code; + error.path = path; + return error; +} + +export function invalidPath(path: string, reason: string): WorkspaceFsError { + return createWorkspaceError("EINVAL", `Invalid path (${reason})`, path); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/blobCache.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/blobCache.ts new file mode 100644 index 00000000..b997ee82 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/blobCache.ts @@ -0,0 +1,87 @@ +// In-process LRU cache of vfs_blob_bytes payloads, keyed by hash. +// +// FUSE reads up to 128 KiB at a time (the kernel's default max_read); +// our chunk size is 512 KiB. A sequential read of a chunk-backed file +// re-fetches the same blob 4x by default. Worse, a 64 MiB file of +// repeated content (e.g. `dd if=/dev/zero`) deduplicates to a single +// blob in vfs_blobs, and we then re-fetch that one blob 512 times +// over the lifetime of one read pass. +// +// vfs_blob_bytes is content-addressed. The normal write path +// (upsertChunkBlob) uses ON CONFLICT DO NOTHING, so a correct +// (hash, bytes) pair is never overwritten and the cache stays valid +// for it. The one exception is repair: stageBlob (the sync receiver +// path) uses ON CONFLICT DO UPDATE SET bytes to replace an incomplete +// or size-mismatched payload left by an interrupted or corrupt write, +// and clears this cache afterward so a stale payload is never served +// after a repair. +// +// The cache is bounded (CHUNK_CACHE_MAX_ENTRIES) and per-Database so +// independent test databases don't pollute each other. Eviction is +// LRU; access moves an entry to the most-recent position. + +import type { Database } from "../storage.js"; + +// Number of distinct blob payloads kept in memory per Database. +// At 512 KiB per blob this caps the cache at ~8 MiB, large enough +// to hold a handful of hot chunks for sequential reads of large +// files without dominating process memory. +const CHUNK_CACHE_MAX_ENTRIES = 16; + +const caches = new WeakMap>(); + +function cacheFor(db: Database): Map { + let cache = caches.get(db); + if (cache === undefined) { + cache = new Map(); + caches.set(db, cache); + } + return cache; +} + +// Stringify a 32-byte hash so it can key a JS Map. Latin-1 +// preserves every byte exactly and avoids the allocation cost of +// hex encoding for what is a very hot path. +function hashKey(hash: Uint8Array): string { + let out = ""; + for (let i = 0; i < hash.byteLength; i++) { + out += String.fromCharCode(hash[i]); + } + return out; +} + +// Look up blob bytes by hash. Cache hit returns the cached +// Uint8Array directly (callers must not mutate it). Cache miss +// queries vfs_blob_bytes and stores the result. Returns undefined +// if the blob isn't in the store. +export function getBlobBytes(db: Database, hash: Uint8Array): Uint8Array | undefined { + const cache = cacheFor(db); + const key = hashKey(hash); + const cached = cache.get(key); + if (cached !== undefined) { + // Reinsert to move to the most-recent position. Map iteration + // order is insertion order, so this gives us LRU eviction for + // free without a separate doubly-linked list. + cache.delete(key); + cache.set(key, cached); + return cached; + } + const row = db.one<{ bytes: Uint8Array }>( + "SELECT bytes FROM vfs_blob_bytes WHERE hash = ?", + hash, + ); + if (row === undefined) return undefined; + cache.set(key, row.bytes); + while (cache.size > CHUNK_CACHE_MAX_ENTRIES) { + const first = cache.keys().next(); + if (first.done === true) break; + cache.delete(first.value); + } + return row.bytes; +} + +// Reset the cache for `db`. Tests use this to keep cache state from +// leaking between cases that share a Database constructor pattern. +export function clearBlobCache(db: Database): void { + caches.delete(db); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/chmod.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/chmod.ts new file mode 100644 index 00000000..5c0800fb --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/chmod.ts @@ -0,0 +1,34 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { incrementRev } from "../rev.js"; +import type { Database } from "../storage.js"; +import { assertNotReadOnly } from "./mount-guard.js"; +import { resolveInode } from "./resolve.js"; + +// Change the file mode bits of a path. Follows symlinks like POSIX +// chmod — the change lands on the target, not the link. Bumps rev +// and mtime so the sync protocol carries the change. +// +// The supplied mode is masked to 12 bits (the permission bits and +// the setuid / setgid / sticky bits). Callers that pass a Node-style +// stat.mode with file-type bits in the upper byte get only the +// permission half stored. +export function chmod(db: Database, path: string, mode: number, now: () => number): void { + const { path: canonical } = canonicalizePath(path); + assertNotReadOnly(db, canonical); + + db.transactionSync(() => { + const node = resolveInode(db, canonical); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); + } + const rev = incrementRev(db); + db.run( + "UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ? WHERE inode = ?", + mode & 0o7777, + now(), + rev, + node.inode, + ); + }); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/filesystem.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/filesystem.ts new file mode 100644 index 00000000..82ea588c --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/filesystem.ts @@ -0,0 +1,128 @@ +// WorkspaceFilesystem — class wrapper that binds a Database and a +// clock to the free fs/* functions. +// +// Every method here is a thin forward to the matching free +// function. The class exists so callers (host-side Workspace, +// in-container tools, tests) get a single instance to thread +// through their code rather than passing (db, now) pairs into +// every call. +// +// Free functions remain exported for internal callers — the +// apply paths in sync/* operate on a Database directly, and the +// in-package tests skip the class wrapper when they only need a +// single op. + +import type { Database } from "../storage.js"; + +import { chmod } from "./chmod.js"; +import { find, type WorkspaceFoundEntry } from "./find.js"; +import { type GrepOptions, grep, type WorkspaceGrepMatch } from "./grep.js"; +import { ls } from "./ls.js"; +import { type MkdirOptions, mkdir } from "./mkdir.js"; +import { type ReaddirOptions, readdir, type WorkspaceDirentResult } from "./readdir.js"; +import { type ReadFileOptions, readFile } from "./readFile.js"; +import { readlink } from "./readlink.js"; +import { type RmOptions, rm } from "./rm.js"; +import { lstat, stat, type WorkspaceStatResult } from "./stat.js"; +import { symlink } from "./symlink.js"; +import { type WriteFileContent, type WriteFileOptions, writeFile } from "./writeFile.js"; + +export interface WorkspaceFilesystemOptions { + // Clock used for mtime / last_seen. Defaults to Date.now. + // Override for deterministic tests. + now?: () => number; +} + +export class WorkspaceFilesystem { + readonly db: Database; + readonly now: () => number; + + constructor(db: Database, options: WorkspaceFilesystemOptions = {}) { + this.db = db; + this.now = options.now ?? Date.now; + } + + // --- Reads ------------------------------------------------------- + + readFile(path: string): Promise>; + readFile(path: string, encoding: "utf8"): Promise; + readFile(path: string, options: ReadFileOptions): Promise>; + readFile( + path: string, + optionsOrEncoding?: "utf8" | ReadFileOptions, + ): Promise> { + // Forward through the free function's overload set. The + // individual overloads above let callers see the precise + // return type for each input shape. + // Cast through the union overload of the free function; + // the class's overloads above carry the precise return type + // for each input shape back to the caller. + return readFile(this.db, path, optionsOrEncoding as ReadFileOptions); + } + + async stat(path: string): Promise { + return stat(this.db, path); + } + + // POSIX lstat — like stat, but doesn't follow a trailing symlink. + // Use when the caller wants to inspect the link itself: readlink + // / unlink under a Node-style fs surface, or just-bash's adapter + // routing lstat through to the workspace. + async lstat(path: string): Promise { + return lstat(this.db, path); + } + + // Return the stored target of a symlink. EINVAL when path is + // not a symlink; ENOENT when path is missing. + async readlink(path: string): Promise { + return readlink(this.db, path); + } + + async readdir(path: string, options: ReaddirOptions = {}): Promise { + return readdir(this.db, path, options); + } + + async find(directory: string, pattern?: string): Promise { + return find(this.db, directory, pattern); + } + + async ls(prefix: string): Promise { + return ls(this.db, prefix); + } + + grep(pattern: string, path: string, options: GrepOptions = {}): Promise { + return grep(this.db, pattern, path, options); + } + + // --- Mutations --------------------------------------------------- + + writeFile( + path: string, + content: WriteFileContent, + options: WriteFileOptions = {}, + ): Promise { + return writeFile(this.db, path, content, options, this.now); + } + + async mkdir(path: string, options: MkdirOptions = {}): Promise { + mkdir(this.db, path, options, this.now); + } + + async rm(path: string, options: RmOptions = {}): Promise { + rm(this.db, path, options); + } + + // Change the permission bits on a path. Follows symlinks like + // POSIX chmod — the change lands on the target, not the link. + // The supplied mode is masked to twelve bits. + async chmod(path: string, mode: number): Promise { + chmod(this.db, path, mode, this.now); + } + + // Create a symbolic link at `path` pointing at `target`. The + // target is stored verbatim; it can be relative or absolute and + // is allowed to dangle. + async symlink(target: string, path: string): Promise { + symlink(this.db, target, path, this.now); + } +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/find.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/find.ts new file mode 100644 index 00000000..67e1f747 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/find.ts @@ -0,0 +1,102 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import type { Database } from "../storage.js"; +import { resolveInode } from "./resolve.js"; + +export interface WorkspaceFoundEntry { + path: string; + type: "file" | "dir"; +} + +interface ChildRow { + name: string; + child_inode: number; + type: "file" | "dir"; +} + +export function find(db: Database, directory: string, pattern?: string): WorkspaceFoundEntry[] { + const { path: canonical } = canonicalizePath(directory); + const node = resolveInode(db, canonical); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); + } + if (node.type !== "dir") { + throw createWorkspaceError("ENOTDIR", `not a directory: ${canonical}`, canonical); + } + + const out: WorkspaceFoundEntry[] = []; + // An empty pattern is equivalent to no pattern: walk and return + // everything rather than compiling it into `^$`, which would match + // only empty relative paths and yield no results. + const regex = pattern ? compileGlob(pattern) : undefined; + + walk(db, node.inode, canonical, out); + + if (regex === undefined) { + return out; + } + // Glob matches against the path relative to the start directory. + const prefix = canonical === "/" ? "/" : `${canonical}/`; + return out.filter((entry) => { + if (!entry.path.startsWith(prefix)) return false; + const rel = entry.path.slice(prefix.length); + return regex.test(rel); + }); +} + +function walk(db: Database, parentInode: number, parentPath: string, out: WorkspaceFoundEntry[]) { + const children = db.all( + `SELECT d.name AS name, d.child_inode AS child_inode, n.type AS type + FROM vfs_dirents d + JOIN vfs_nodes n ON n.inode = d.child_inode + WHERE d.parent_inode = ? + ORDER BY d.name`, + parentInode, + ); + for (const child of children) { + const childPath = parentPath === "/" ? `/${child.name}` : `${parentPath}/${child.name}`; + out.push({ path: childPath, type: child.type }); + if (child.type === "dir") { + walk(db, child.child_inode, childPath, out); + } + } +} + +// Compile a simple glob into a regex. Supported: +// * matches any run of characters except '/' +// ** matches any run of characters including '/' +// Anything else is a literal. Regex metacharacters in literals are +// escaped so '.' in '*.ts' doesn't match an arbitrary character. +function compileGlob(pattern: string): RegExp { + let re = ""; + let i = 0; + while (i < pattern.length) { + const ch = pattern[i]; + if (ch === "*") { + if (pattern[i + 1] === "*") { + // '**/' matches zero or more path segments. Without the slash, '**' + // matches any run including slashes. + if (pattern[i + 2] === "/") { + re += "(?:.*/)?"; + i += 3; + } else { + re += ".*"; + i += 2; + } + } else { + re += "[^/]*"; + i += 1; + } + continue; + } + if (REGEX_METACHARS.has(ch)) { + re += `\\${ch}`; + } else { + re += ch; + } + i += 1; + } + return new RegExp(`^${re}$`); +} + +const REGEX_METACHARS = new Set([".", "+", "?", "^", "$", "(", ")", "[", "]", "{", "}", "|", "\\"]); diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/grep.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/grep.ts new file mode 100644 index 00000000..fd28471c --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/grep.ts @@ -0,0 +1,107 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import type { Database } from "../storage.js"; +import { find } from "./find.js"; +import { readFile } from "./readFile.js"; +import { resolveInode } from "./resolve.js"; + +export interface WorkspaceGrepMatch { + path: string; + line: number; + text: string; +} + +export interface GrepOptions { + ignoreCase?: boolean; +} + +export async function grep( + db: Database, + pattern: string, + path: string, + options: GrepOptions = {}, +): Promise { + const { path: canonical } = canonicalizePath(path); + const node = resolveInode(db, canonical); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); + } + + const filePaths = + node.type === "file" + ? [canonical] + : find(db, canonical) + .filter((entry) => entry.type === "file") + .map((entry) => entry.path); + + const matches: WorkspaceGrepMatch[] = []; + for (const filePath of filePaths) { + await scanFile(db, filePath, pattern, options, matches); + } + return matches; +} + +// Stream the file in chunks so very large files don't load fully into +// memory. Carry a partial-line tail between chunks (everything after +// the last '\n') so a line that straddles a chunk boundary still +// matches as one line. Line numbers are 1-indexed. +async function scanFile( + db: Database, + path: string, + pattern: string, + options: GrepOptions, + out: WorkspaceGrepMatch[], +): Promise { + const stream = await readFile(db, path); + const reader = stream.getReader(); + const decoder = new TextDecoder("utf-8", { fatal: false }); + const needle = options.ignoreCase ? pattern.toUpperCase() : pattern; + + let tail = ""; + let lineNo = 1; + while (true) { + const { value, done } = await reader.read(); + if (done) break; + if (value === undefined) continue; + const text = tail + decoder.decode(value, { stream: true }); + const newlineIdx = text.lastIndexOf("\n"); + const ready = newlineIdx === -1 ? "" : text.slice(0, newlineIdx); + tail = newlineIdx === -1 ? text : text.slice(newlineIdx + 1); + if (ready.length > 0) { + lineNo = scanLines(ready, lineNo, needle, options.ignoreCase === true, path, out); + } + } + // Drain the decoder and scan whatever's left (final line without a + // trailing newline). + tail += decoder.decode(); + if (tail.length > 0) { + scanLines(tail, lineNo, needle, options.ignoreCase === true, path, out); + } +} + +// Walk `block` line-by-line, push matches into `out`, return the next +// 1-indexed line number to use for the following block. +function scanLines( + block: string, + startLine: number, + needle: string, + ignoreCase: boolean, + path: string, + out: WorkspaceGrepMatch[], +): number { + let line = startLine; + let cursor = 0; + while (cursor <= block.length) { + const next = block.indexOf("\n", cursor); + const end = next === -1 ? block.length : next; + const text = block.slice(cursor, end); + const haystack = ignoreCase ? text.toUpperCase() : text; + if (haystack.includes(needle)) { + out.push({ path, line, text }); + } + line += 1; + if (next === -1) break; + cursor = next + 1; + } + return line; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/link.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/link.ts new file mode 100644 index 00000000..c5e21d17 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/link.ts @@ -0,0 +1,84 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { incrementRev } from "../rev.js"; +import { ROOT_INODE } from "../schema/index.js"; +import type { Database } from "../storage.js"; +import { assertNotReadOnly } from "./mount-guard.js"; +import { resolveInode } from "./resolve.js"; +import { invalidateResolveExact } from "./resolveCache.js"; + +function resolveParent(db: Database, parts: string[], canonical: string): number { + let parentInode = ROOT_INODE; + for (let i = 0; i < parts.length - 1; i++) { + const child = db.one<{ child_inode: number }>( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + parentInode, + parts[i], + ); + if (child === undefined) { + throw createWorkspaceError("ENOENT", `parent directory missing: ${canonical}`, canonical); + } + const next = db.one<{ inode: number; type: "file" | "dir" | "symlink" }>( + "SELECT inode, type FROM vfs_nodes WHERE inode = ?", + child.child_inode, + ); + if (next === undefined) { + throw createWorkspaceError("ENOENT", `dangling dirent: ${canonical}`, canonical); + } + if (next.type !== "dir") { + throw createWorkspaceError( + "ENOTDIR", + `parent path segment is not a directory: ${canonical}`, + canonical, + ); + } + parentInode = next.inode; + } + return parentInode; +} + +export function link(db: Database, existingPath: string, newPath: string): void { + const { parts, path: canonicalNew } = canonicalizePath(newPath); + if (parts.length === 0) { + throw createWorkspaceError("EEXIST", "cannot link onto root", canonicalNew); + } + + assertNotReadOnly(db, canonicalNew); + + db.transactionSync(() => { + const source = resolveInode(db, existingPath); + if (source === null) { + throw createWorkspaceError("ENOENT", `no such file: ${existingPath}`, existingPath); + } + if (source.type !== "file") { + throw createWorkspaceError( + "EPERM", + `cannot hardlink non-file: ${existingPath}`, + existingPath, + ); + } + + const parentInode = resolveParent(db, parts, canonicalNew); + const leafName = parts[parts.length - 1]; + const existing = db.one<{ child_inode: number }>( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + parentInode, + leafName, + ); + if (existing !== undefined) { + throw createWorkspaceError("EEXIST", `path exists: ${canonicalNew}`, canonicalNew); + } + + db.run( + "INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", + parentInode, + leafName, + source.inode, + ); + const rev = incrementRev(db); + db.run("UPDATE vfs_nodes SET rev = ? WHERE inode = ?", rev, source.inode); + // A new hardlink name for an existing file: a leaf with no + // descendants, so drop just the (possibly negative) entry for it. + invalidateResolveExact(db, canonicalNew); + }); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/ls.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/ls.ts new file mode 100644 index 00000000..6a5c2387 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/ls.ts @@ -0,0 +1,59 @@ +import { canonicalizePath } from "../path.js"; +import { ROOT_INODE } from "../schema/index.js"; +import type { Database } from "../storage.js"; + +interface PathRow { + path: string; +} + +// Recursive CTE that materializes the file paths under one listing +// root. Files only (no directory entries) because that's the +// documented "flat list of file paths" semantics. +// +// The walk is seeded at the listing root's inode: each row is +// (inode, path, type), built by concatenating dirent names with '/' +// separators onto the seed path. Scoping the seed to the requested +// directory keeps the walk O(subtree) instead of O(whole tree). +const LS_QUERY = ` + WITH RECURSIVE walk(inode, path, type) AS ( + SELECT inode, ?, type FROM vfs_nodes WHERE inode = ? + UNION ALL + SELECT n.inode, w.path || '/' || d.name, n.type + FROM walk w + JOIN vfs_dirents d ON d.parent_inode = w.inode + JOIN vfs_nodes n ON n.inode = d.child_inode + ) + SELECT path FROM walk + WHERE type = 'file' + ORDER BY path +`; + +// Walk dirents from the root to `parts` without following symlinks, so +// the seed matches the CTE's structural view: a symlink component has +// no dirents and thus lists nothing, and a missing or non-directory +// component resolves to null (an empty listing). Returns the root +// inode for an empty path. +function resolvePrefixInode(db: Database, parts: string[]): number | null { + let inode = ROOT_INODE; + for (const name of parts) { + const child = db.one<{ child_inode: number }>( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + inode, + name, + ); + if (child === undefined) return null; + inode = child.child_inode; + } + return inode; +} + +export function ls(db: Database, prefix: string): string[] { + const { parts, path: canonical } = canonicalizePath(prefix); + const inode = resolvePrefixInode(db, parts); + if (inode === null) return []; + // Root contributes the empty string so its children start with '/'; + // a non-root prefix seeds its own path so descendants read as + // absolute paths. + const seedPath = canonical === "/" ? "" : canonical; + return db.all(LS_QUERY, seedPath, inode).map((row) => row.path); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/mkdir.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/mkdir.ts new file mode 100644 index 00000000..4d664c61 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/mkdir.ts @@ -0,0 +1,130 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { incrementRev } from "../rev.js"; +import { ROOT_INODE } from "../schema/index.js"; +import type { Database } from "../storage.js"; +import { assertNotReadOnly } from "./mount-guard.js"; +import { invalidateResolveExact } from "./resolveCache.js"; + +export interface MkdirOptions { + recursive?: boolean; + mode?: number; +} + +interface ResolvedSegment { + inode: number; + type: "file" | "dir"; +} + +// Look up a child by name under a parent directory. Returns undefined +// when there's no dirent. The caller decides whether that's an error. +function lookupChild(db: Database, parentInode: number, name: string): ResolvedSegment | undefined { + const row = db.one<{ child_inode: number }>( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + parentInode, + name, + ); + if (row === undefined) { + return undefined; + } + const node = db.one<{ inode: number; type: "file" | "dir" }>( + "SELECT inode, type FROM vfs_nodes WHERE inode = ?", + row.child_inode, + ); + if (node === undefined) { + return undefined; + } + return node; +} + +// Create one directory entry under `parentInode`, returning the new +// inode. The caller has already verified the name is not taken. +function createDir( + db: Database, + parentInode: number, + name: string, + mode: number, + mtime: number, + rev: number, +): number { + // RETURNING folds the rowid read into the INSERT. + const row = db.one<{ inode: number }>( + "INSERT INTO vfs_nodes (type, mode, mtime, rev) VALUES ('dir', ?, ?, ?) RETURNING inode", + mode, + mtime, + rev, + ); + if (row === undefined) { + throw createWorkspaceError("EIO", "failed to allocate inode"); + } + const inode = row.inode; + db.run( + "INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", + parentInode, + name, + inode, + ); + return inode; +} + +export function mkdir(db: Database, path: string, options: MkdirOptions, now: () => number): void { + const { parts, path: canonical } = canonicalizePath(path); + const recursive = options.recursive === true; + const mode = (options.mode ?? 0o755) & 0o7777; + + if (parts.length === 0) { + // Root always exists post-initializeSchema; mkdir("/") is EEXIST + // even with recursive (matches Node fs.mkdir's "EEXIST on root" + // behaviour for non-recursive; for recursive Node returns + // undefined, but our docs treat mkdir("/") as nonsensical). + throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); + } + assertNotReadOnly(db, canonical); + + db.transactionSync(() => { + const rev = incrementRev(db); + const mtime = now(); + + let parentInode = ROOT_INODE; + // Walk all but the final segment. Each must already exist as a + // directory; if `recursive`, we create missing ones. + for (let i = 0; i < parts.length - 1; i++) { + const name = parts[i]; + const existing = lookupChild(db, parentInode, name); + if (existing === undefined) { + if (!recursive) { + throw createWorkspaceError("ENOENT", `parent directory missing: ${canonical}`, canonical); + } + parentInode = createDir(db, parentInode, name, 0o755, mtime, rev); + // A newly created directory is empty, so a cached negative for + // its own path is the only stale entry possible; drop it exact. + invalidateResolveExact(db, `/${parts.slice(0, i + 1).join("/")}`); + continue; + } + if (existing.type !== "dir") { + throw createWorkspaceError( + "ENOTDIR", + `parent path segment is not a directory: ${canonical}`, + canonical, + ); + } + parentInode = existing.inode; + } + + // Final segment. + const leafName = parts[parts.length - 1]; + const existing = lookupChild(db, parentInode, leafName); + if (existing !== undefined) { + // EEXIST is correct for both "already a directory" and + // "already a file" per docs/04. Recursive only swallows the + // already-a-directory case. + if (recursive && existing.type === "dir") { + return; + } + throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); + } + + createDir(db, parentInode, leafName, mode, mtime, rev); + invalidateResolveExact(db, canonical); + }); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/mount-guard.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/mount-guard.ts new file mode 100644 index 00000000..6a2aadd3 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/mount-guard.ts @@ -0,0 +1,82 @@ +// Read-only mount guard. +// +// Every dofs mutating entry point (writeFile, mkdir, rm, and the +// apply path in sync/apply.ts) consults this module to reject +// writes that fall under a registered read-only mount root. The +// guard lives at the data layer so container-side writes that +// arrive via pullOnce -> applyChanges are caught too — the +// workspace-side surface wrapper alone cannot see them. +// +// The set of read-only roots is small (one row per registered +// mount per workspace, typically <10) and changes only at indexer +// write time. Cache it per Database in a WeakMap so repeat lookups +// don't hit SQLite. The mount indexer in @cloudflare/computer +// invalidates the cache via `invalidateReadOnlyMountCache(db)` after +// it writes _vfs_mounts. + +import { createWorkspaceError } from "../errors.js"; +import type { Database } from "../storage.js"; + +// undefined sentinel = "not loaded yet"; an empty array means +// "loaded, no read-only mounts registered". The two are not the +// same: the empty case must skip the SQL lookup on every check. +const cache = new WeakMap(); + +// Public so the workspace-side indexer can drop the cache after it +// writes a new _vfs_mounts row. Tests also call it when they stage +// a mount fixture by direct SQL. +export function invalidateReadOnlyMountCache(db: Database): void { + cache.delete(db); +} + +function loadReadOnlyRoots(db: Database): readonly string[] { + const rows = db.all<{ root: string }>("SELECT root FROM _vfs_mounts WHERE mode = 'read-only'"); + const roots = rows.map((r) => r.root); + cache.set(db, roots); + return roots; +} + +export function getReadOnlyMountRoots(db: Database): readonly string[] { + const cached = cache.get(db); + if (cached !== undefined) return cached; + return loadReadOnlyRoots(db); +} + +// Symmetric overlap check between a candidate write path and a +// mount root. Either: +// - `path` is at or below `root` (a direct write or rm under the +// mount root), OR +// - `root` is below `path` (an ancestor rm that would recurse +// through the mount). +// Both shapes must be blocked so a read-only mount survives both +// vectors. Mirrors the predicate that lived in +// GuardedWorkspaceFilesystem before the data-layer move. +function overlapsRoot(path: string, root: string): boolean { + return path === root || path.startsWith(`${root}/`) || root.startsWith(`${path}/`); +} + +// Throws EROFS when the path overlaps any read-only mount root. +// Callers should invoke this before any DB mutation. The error +// shape matches the existing createWorkspaceError contract so +// surface callers see a normal WorkspaceFsError. +export function assertNotReadOnly(db: Database, path: string): void { + const roots = getReadOnlyMountRoots(db); + if (roots.length === 0) return; + for (const root of roots) { + if (overlapsRoot(path, root)) { + throw createWorkspaceError("EROFS", `read-only mount at ${root}: cannot modify`, path); + } + } +} + +// Variant for callers that already know the path is canonicalised +// and want to reject a single descendant during a recursive walk +// (rm's walkPostOrder). Returns the matching root or undefined; the +// caller decides whether to throw, log, or skip. +export function readOnlyRootFor(db: Database, path: string): string | undefined { + const roots = getReadOnlyMountRoots(db); + for (const root of roots) { + if (overlapsRoot(path, root)) return root; + } + return undefined; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/readFile.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/readFile.ts new file mode 100644 index 00000000..7f89de40 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/readFile.ts @@ -0,0 +1,194 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import type { Database } from "../storage.js"; +import { getBlobBytes } from "./blobCache.js"; +import { resolveInode } from "./resolve.js"; +import { getPendingWriteBufferByPath, getWriteBuffer } from "./writeBuffer.js"; +import { CHUNK_SIZE } from "./writeFile.js"; + +export interface ReadFileOptions { + encoding?: "utf8"; +} + +interface ChunkRow { + hash: Uint8Array; + size: number; +} + +// Overloads match docs/04_filesystem_interface.md exactly. +export function readFile(db: Database, path: string): Promise>; +export function readFile(db: Database, path: string, encoding: "utf8"): Promise; +export function readFile( + db: Database, + path: string, + options: ReadFileOptions, +): Promise>; +export async function readFile( + db: Database, + path: string, + optionsOrEncoding?: "utf8" | ReadFileOptions, +): Promise> { + const wantString = + optionsOrEncoding === "utf8" || + (typeof optionsOrEncoding === "object" && optionsOrEncoding?.encoding === "utf8"); + + // Pending-create files surface through the path-keyed buffer. + const { path: canonical } = canonicalizePath(path); + const pending = getPendingWriteBufferByPath(db, canonical); + if (pending !== undefined) { + const snapshot = new Uint8Array(pending.size); + snapshot.set(pending.buf.subarray(0, pending.size)); + if (wantString) return new TextDecoder().decode(snapshot); + return new ReadableStream({ + start(controller) { + controller.enqueue(snapshot); + controller.close(); + }, + }); + } + + // Resolve up front so we surface ENOENT/EISDIR before doing any + // streaming work. + const node = resolveInode(db, path); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such file: ${path}`, path); + } + if (node.type !== "file") { + throw createWorkspaceError("EISDIR", `path is a directory: ${path}`, path); + } + + // While a write buffer is open for this inode it is the source of + // truth. Skip the chunk store and serve the buffered bytes. + const buffered = getWriteBuffer(db, node.inode); + if (buffered?.dirty) { + const snapshot = new Uint8Array(buffered.size); + snapshot.set(buffered.buf.subarray(0, buffered.size)); + if (wantString) return new TextDecoder().decode(snapshot); + return new ReadableStream({ + start(controller) { + controller.enqueue(snapshot); + controller.close(); + }, + }); + } + + const chunks = db.all( + "SELECT hash, size FROM vfs_chunks WHERE inode = ? ORDER BY idx", + node.inode, + ); + + if (wantString) { + // Fast path — concatenate everything and decode once. Matches the + // node:fs/promises.readFile semantics for an encoding argument: + // memory cost = whole file. + const totalSize = chunks.reduce((acc, c) => acc + c.size, 0); + const out = new Uint8Array(totalSize); + let offset = 0; + for (const chunk of chunks) { + const bytes = getBlobBytes(db, chunk.hash); + if (bytes === undefined) { + throw createWorkspaceError("EIO", `missing blob bytes for ${path}`, path); + } + out.set(bytes, offset); + offset += bytes.byteLength; + } + return new TextDecoder().decode(out); + } + + // Stream form. We enqueue one Uint8Array per chunk, lazily pulled. + // Reads resolve bytes by hash and never restamp last_seen: a chunk + // being read is already linked to a node, so gc's orphan gate keeps + // it. last_seen only guards blobs staged but not yet linked. + let i = 0; + return new ReadableStream({ + pull(controller) { + if (i >= chunks.length) { + controller.close(); + return; + } + const chunk = chunks[i++]; + const bytes = getBlobBytes(db, chunk.hash); + if (bytes === undefined) { + controller.error(createWorkspaceError("EIO", `missing blob bytes for ${path}`, path)); + return; + } + controller.enqueue(bytes); + }, + }); +} + +// Positional read primitive. Walks only the chunk rows that overlap +// [offset, offset+length), so the FUSE driver can serve a kernel +// read without materializing the whole file. +export function readRangeSync( + db: Database, + path: string, + offset: number, + length: number, +): Uint8Array { + if (!Number.isInteger(offset) || offset < 0) { + throw createWorkspaceError("EINVAL", `invalid read offset: ${offset}`, path); + } + if (!Number.isInteger(length) || length < 0) { + throw createWorkspaceError("EINVAL", `invalid read length: ${length}`, path); + } + // Pending-create files have no inode yet. Serve reads from the + // path-keyed buffer until release commits the row. + const { path: canonical } = canonicalizePath(path); + const pending = getPendingWriteBufferByPath(db, canonical); + if (pending !== undefined) { + if (length === 0) return new Uint8Array(); + if (offset >= pending.size) return new Uint8Array(); + const end = Math.min(offset + length, pending.size); + return pending.buf.subarray(offset, end); + } + const node = resolveInode(db, path); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such file: ${path}`, path); + } + if (node.type !== "file") { + throw createWorkspaceError("EISDIR", `path is a directory: ${path}`, path); + } + if (length === 0) return new Uint8Array(); + + // If a write buffer is open for this inode, it is the source of + // truth: pending writes have not yet committed to vfs_chunks. + // Reading from SQLite here would return stale bytes. + const buffered = getWriteBuffer(db, node.inode); + if (buffered?.dirty) { + if (offset >= buffered.size) return new Uint8Array(); + const end = Math.min(offset + length, buffered.size); + return buffered.buf.subarray(offset, end); + } + + // node.size is the cached value resolveInode just loaded. + const totalSize = node.size; + if (offset >= totalSize) return new Uint8Array(); + const end = Math.min(offset + length, totalSize); + const firstIdx = Math.floor(offset / CHUNK_SIZE); + const lastIdx = Math.floor((end - 1) / CHUNK_SIZE); + // Pull every overlapping chunk in one indexed range scan. Missing + // indices (a sparse file) simply don't come back, so the assembly + // below compacts around the gaps exactly as a per-index walk would. + const chunks = db.all<{ idx: number; hash: Uint8Array }>( + "SELECT idx, hash FROM vfs_chunks WHERE inode = ? AND idx BETWEEN ? AND ? ORDER BY idx", + node.inode, + firstIdx, + lastIdx, + ); + const out = new Uint8Array(end - offset); + let written = 0; + for (const { idx, hash } of chunks) { + const start = idx * CHUNK_SIZE; + const bytes = getBlobBytes(db, hash); + if (bytes === undefined) { + throw createWorkspaceError("EIO", `missing blob bytes for ${path}`, path); + } + const srcStart = Math.max(0, offset - start); + const srcEnd = Math.min(bytes.byteLength, end - start); + if (srcEnd <= srcStart) continue; + out.set(bytes.subarray(srcStart, srcEnd), written); + written += srcEnd - srcStart; + } + return written === out.byteLength ? out : out.subarray(0, written); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/readdir.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/readdir.ts new file mode 100644 index 00000000..dddfa8d1 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/readdir.ts @@ -0,0 +1,84 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import type { Database } from "../storage.js"; +import { resolveInode } from "./resolve.js"; +import { listPendingByParent } from "./writeBuffer.js"; + +export interface WorkspaceDirentResult { + name: string; + parentPath: string; + isFile: boolean; + isDirectory: boolean; + isSymbolicLink: boolean; +} + +interface DirentRow { + name: string; + type: "file" | "dir" | "symlink"; +} + +export interface ReaddirOptions { + /** Maximum committed entries to materialize. Pending entries may extend the result. */ + limit?: number; +} + +export function readdir( + db: Database, + path: string, + options: ReaddirOptions = {}, +): WorkspaceDirentResult[] { + const { path: canonical } = canonicalizePath(path); + const node = resolveInode(db, canonical); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); + } + if (node.type !== "dir") { + throw createWorkspaceError("ENOTDIR", `not a directory: ${canonical}`, canonical); + } + + const limit = options.limit; + if (limit !== undefined && (!Number.isSafeInteger(limit) || limit < 0)) { + throw new TypeError("readdir limit must be a non-negative safe integer"); + } + const rows = db.all( + `SELECT d.name AS name, n.type AS type + FROM vfs_dirents d + JOIN vfs_nodes n ON n.inode = d.child_inode + WHERE d.parent_inode = ? + ORDER BY d.name + ${limit === undefined ? "" : "LIMIT ?"}`, + ...(limit === undefined ? [node.inode] : [node.inode, limit]), + ); + + const entries = rows.map((row) => ({ + name: row.name, + parentPath: canonical, + isFile: row.type === "file", + isDirectory: row.type === "dir", + isSymbolicLink: row.type === "symlink", + })); + + // Merge in pending-create buffers parented under this directory so + // a `readdir` between FUSE create and release still surfaces the + // file. Skip any whose name already appears in the SQL rows (in + // case a concurrent commit just landed it). + const pending = listPendingByParent(db, node.inode); + if (pending.length > 0) { + const seen = new Set(entries.map((e) => e.name)); + for (const entry of pending) { + if (entry.pending === undefined) continue; + const { leafName } = entry.pending; + if (seen.has(leafName)) continue; + entries.push({ + name: leafName, + parentPath: canonical, + isFile: true, + isDirectory: false, + isSymbolicLink: false, + }); + } + entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); + } + + return entries; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/readlink.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/readlink.ts new file mode 100644 index 00000000..8b9b5048 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/readlink.ts @@ -0,0 +1,17 @@ +import { createWorkspaceError } from "../errors.js"; +import type { Database } from "../storage.js"; +import { resolveInode } from "./resolve.js"; + +// Return the stored target of a symlink. Does not follow the link. +// Mirrors POSIX semantics: ENOENT for a missing path, EINVAL when +// the path resolves to something that isn't a symlink. +export function readlink(db: Database, path: string): string { + const node = resolveInode(db, path, { followSymlinks: false }); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such path: ${path}`, path); + } + if (node.type !== "symlink" || node.linkTarget === undefined) { + throw createWorkspaceError("EINVAL", `not a symlink: ${path}`, path); + } + return node.linkTarget; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/rename.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/rename.ts new file mode 100644 index 00000000..c2692b1e --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/rename.ts @@ -0,0 +1,230 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { incrementRev } from "../rev.js"; +import type { Database } from "../storage.js"; +import { recordDelete } from "../sync/changes.js"; +import { pathOf } from "../sync/paths.js"; +import { assertNotReadOnly } from "./mount-guard.js"; +import { resolveInode } from "./resolve.js"; +import { invalidateResolveExact, invalidateResolveSubtree } from "./resolveCache.js"; +import { unlinkDirent } from "./unlink.js"; + +type NodeType = "file" | "dir" | "symlink"; + +export function rename(db: Database, oldPath: string, newPath: string): void { + const { path: oldCanonical } = canonicalizePath(oldPath); + const { parts: newParts, path: newCanonical } = canonicalizePath(newPath); + + if (oldCanonical === "/") { + throw createWorkspaceError("EINVAL", "cannot rename root", oldCanonical); + } + if (newParts.length === 0) { + throw createWorkspaceError("EINVAL", "cannot rename onto root", newCanonical); + } + + assertNotReadOnly(db, oldCanonical); + assertNotReadOnly(db, newCanonical); + + db.transactionSync(() => { + const source = resolveInode(db, oldCanonical, { followSymlinks: false }); + if (source === null) { + throw createWorkspaceError("ENOENT", `no such path: ${oldCanonical}`, oldCanonical); + } + + // Resolve the source's real parent dirent. The parent path is + // resolved with symlinks followed so a request through a symlinked + // directory lands on the real container; the inode is then + // identified by (parent_inode, name) rather than by child_inode so + // a hardlinked source touches only the requested name. + const { parts: oldParts } = canonicalizePath(oldCanonical); + const oldName = oldParts[oldParts.length - 1]; + const oldParentPath = oldParts.length === 1 ? "/" : `/${oldParts.slice(0, -1).join("/")}`; + const oldParent = resolveInode(db, oldParentPath); + if (oldParent === null || oldParent.type !== "dir") { + throw createWorkspaceError("ENOENT", `no such path: ${oldCanonical}`, oldCanonical); + } + const oldParentReal = pathOf(db, oldParent.inode); + if (oldParentReal === null) { + throw createWorkspaceError("ENOENT", `no such path: ${oldCanonical}`, oldCanonical); + } + const oldRealPath = oldParentReal === "/" ? `/${oldName}` : `${oldParentReal}/${oldName}`; + assertNotReadOnly(db, oldRealPath); + + if (oldCanonical === newCanonical) return; + + const newName = newParts[newParts.length - 1]; + const newParentPath = newParts.length === 1 ? "/" : `/${newParts.slice(0, -1).join("/")}`; + const newParent = resolveInode(db, newParentPath); + if (newParent === null || newParent.type !== "dir") { + throw createWorkspaceError( + "ENOENT", + `parent directory missing: ${newCanonical}`, + newCanonical, + ); + } + const newParentReal = pathOf(db, newParent.inode); + if (newParentReal === null) { + throw createWorkspaceError( + "ENOENT", + `parent directory missing: ${newCanonical}`, + newCanonical, + ); + } + const newRealPath = newParentReal === "/" ? `/${newName}` : `${newParentReal}/${newName}`; + assertNotReadOnly(db, newRealPath); + + // A rename whose source and destination resolve to the very same + // dirent (same real parent and name, e.g. through a symlinked path) + // is a true no-op: leave the tree and the change stream untouched. + // This is distinct from renaming one hardlink onto another, where + // the names differ and the source link must still be removed. + if (oldParent.inode === newParent.inode && oldName === newName) return; + + const existing = db.one<{ child_inode: number; type: "file" | "dir" | "symlink" }>( + `SELECT d.child_inode AS child_inode, n.type AS type + FROM vfs_dirents d + JOIN vfs_nodes n ON n.inode = d.child_inode + WHERE d.parent_inode = ? AND d.name = ?`, + newParent.inode, + newName, + ); + + // Authoritative directory self-move guard. It tests the *resolved* + // destination parent inode against the source subtree, so it catches + // a symlinked destination that lands inside the source and allows one + // that resolves outside it. A textual prefix test on the unresolved + // path could do neither and is intentionally absent. + if ( + source.type === "dir" && + renamedSubtreeContains(db, source.inode, oldRealPath, newParent.inode) + ) { + throw createWorkspaceError( + "EINVAL", + `cannot rename a directory into itself: ${oldRealPath}`, + newCanonical, + ); + } + + if (existing !== undefined) { + assertCompatibleOverwrite(source.type, existing.type, newCanonical); + if (existing.type === "dir") { + const childCount = db.scalar( + "SELECT COUNT(*) FROM vfs_dirents WHERE parent_inode = ?", + existing.child_inode, + ); + if ((childCount ?? 0) > 0) { + throw createWorkspaceError("ENOTEMPTY", `not empty: ${newCanonical}`, newCanonical); + } + } + // Displace only the destination name. The displaced inode may + // carry other hardlinks (or be the source inode itself), so reap + // its chunks and node row only once the final link disappears. + // Order matters: displace before unlinking the source so a + // hardlink-onto-hardlink rename never momentarily drops to zero + // links and reaps the inode it is about to re-point. + unlinkDirent(db, newParent.inode, newName, existing.child_inode, existing.type); + } + + // Unlink only the source name; a hardlinked source keeps its other + // names alive. + db.run("DELETE FROM vfs_dirents WHERE parent_inode = ? AND name = ?", oldParent.inode, oldName); + db.run( + "INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", + newParent.inode, + newName, + source.inode, + ); + + const rev = incrementRev(db); + // Rename is represented on the wire as old-path tombstones plus + // live entries for the moved inode subtree, so stamp only that + // subtree with the shared rev. Parent directory mtimes are left + // unchanged on purpose; this diverges from POSIX rename(2), but + // avoids treating the old and new parents as content changes. A + // directory move stamps and tombstones its whole subtree in two + // set-based statements; a file or symlink touches one inode and + // one path. + if (source.type === "dir") { + stampRenamedSubtree(db, source.inode, oldRealPath, rev); + } else { + db.run("UPDATE vfs_nodes SET rev = ? WHERE inode = ?", rev, source.inode); + recordDelete(db, rev, oldRealPath); + } + + // Drop cached resolutions for both endpoints. A directory move + // changes every descendant's path, so both sides need a subtree + // drop; a file/symlink move only touches the two leaf paths. The + // destination drop also covers any entry displaced by an overwrite. + if (source.type === "dir") { + invalidateResolveSubtree(db, oldRealPath); + invalidateResolveSubtree(db, newRealPath); + } else { + invalidateResolveExact(db, oldRealPath); + invalidateResolveExact(db, newRealPath); + } + }); +} + +function assertCompatibleOverwrite( + sourceType: NodeType, + existingType: NodeType, + path: string, +): void { + if (sourceType === "dir" && existingType === "dir") return; + if (existingType === "dir") { + throw createWorkspaceError("EISDIR", `cannot overwrite directory: ${path}`, path); + } + if (sourceType === "dir") { + throw createWorkspaceError("ENOTDIR", `cannot overwrite non-directory: ${path}`, path); + } +} + +// Recursive walk of a directory subtree seeded at an inode and its +// path. Descends through directory dirents only, so files and symlinks +// are leaves and each hardlink name yields its own row (matching the +// per-component collection it replaces). Bound as a reusable WITH +// clause whose two placeholders are the seed inode and path; callers +// append their own projection. +const SUBTREE_CTE = `WITH RECURSIVE subtree(inode, type, path) AS ( + SELECT ?, 'dir', ? + UNION ALL + SELECT n.inode, n.type, + CASE WHEN s.path = '/' THEN '/' || d.name ELSE s.path || '/' || d.name END + FROM subtree s + JOIN vfs_dirents d ON d.parent_inode = s.inode + JOIN vfs_nodes n ON n.inode = d.child_inode + WHERE s.type = 'dir' +)`; + +function renamedSubtreeContains( + db: Database, + rootInode: number, + rootPath: string, + targetInode: number, +): boolean { + const hit = db.one<{ hit: number }>( + `${SUBTREE_CTE} SELECT 1 AS hit FROM subtree WHERE inode = ? LIMIT 1`, + rootInode, + rootPath, + targetInode, + ); + return hit !== undefined; +} + +// Stamp the shared rev on every inode in the moved subtree and record +// an old-path tombstone for each entry, in two set-based statements +// over the same walk. +function stampRenamedSubtree(db: Database, rootInode: number, rootPath: string, rev: number): void { + db.run( + `${SUBTREE_CTE} UPDATE vfs_nodes SET rev = ? WHERE inode IN (SELECT inode FROM subtree)`, + rootInode, + rootPath, + rev, + ); + db.run( + `${SUBTREE_CTE} INSERT INTO vfs_changes (rev, path, op) SELECT ?, path, 'delete' FROM subtree ORDER BY path`, + rootInode, + rootPath, + rev, + ); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/resolve.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/resolve.ts new file mode 100644 index 00000000..defaa66c --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/resolve.ts @@ -0,0 +1,264 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { ROOT_INODE } from "../schema/index.js"; +import type { Database } from "../storage.js"; +import { lookupResolveCache, storeResolveCache } from "./resolveCache.js"; + +export interface ResolvedInode { + inode: number; + type: "file" | "dir" | "symlink"; + mode: number; + mtime: number; + // Cached file size from vfs_nodes.size. Always 0 for directories + // and symlinks; for files this matches SUM(vfs_chunks.size) for + // the inode. Stat callers consume it directly instead of doing a + // separate aggregate query. + size: number; + // Populated only when type === "symlink". Higher layers (readlink, + // lstat) consume this; resolveInode follows it transparently unless + // the caller asks otherwise. + linkTarget?: string; +} + +export interface ResolveOptions { + // Default true. Pass false to land on a symlink itself — the + // lstat / readlink code paths rely on this. Loops are still + // detected when following. + followSymlinks?: boolean; +} + +interface NodeRow { + inode: number; + type: "file" | "dir" | "symlink"; + mode: number; + mtime: number; + size: number; + link_target: string | null; +} + +interface ChildRow { + child_inode: number; +} + +// Cap the total number of symlinks resolved across a single +// resolveInode() call. Matches Linux's default SYMLOOP_MAX of 40. +const MAX_SYMLINK_FOLLOWS = 40; + +// Walk vfs_dirents from ROOT_INODE down to `path`. Returns null when +// any segment is missing, when an intermediate segment is a file +// (which a real filesystem would surface as ENOTDIR — callers map +// the `null` to the appropriate POSIX code), or when a final-segment +// symlink dangles. Throws ELOOP when a cycle is detected. +// +// `path` is canonicalized internally so callers can pass user input +// directly. Pre-canonicalized paths are also accepted and incur the +// same trivial re-canonicalization cost. +export function resolveInode( + db: Database, + path: string, + options: ResolveOptions = {}, +): ResolvedInode | null { + const followFinal = options.followSymlinks !== false; + const { parts, path: canonical } = canonicalizePath(path); + + // Cache + single-statement CTE serve only cache-eligible reads: + // follow-symlinks resolutions outside a transaction. Everything else + // uses the per-component loop: + // * followSymlinks:false (lstat / readlink / the provider's + // pre-mutation captures) — not cached, and the loop is cheaper + // for these shallow one-shot resolves than the recursive CTE. + // * inside a transaction (every mutation path) — resolves are + // shallow and hot, the CTE competes with the mutation's own + // statements for the plan cache (recompiling it is far dearer + // than the loop), and the cache must not be populated + // mid-transaction anyway (rollback safety). + // Mutations still invalidate the cache; that is independent of this. + if (!followFinal || db.inTransaction) { + return resolveParts(db, parts, followFinal, 0); + } + + // Repeat reads of the same path are served from the per-Database + // cache. Only the path -> inode mapping is cached; re-read the node + // row so mode/size/mtime/type are always current. A stale mapping + // (inode reaped without invalidation) reads back null and falls + // through to a full resolve that re-populates the cache. + const hit = lookupResolveCache(db, canonical); + if (hit !== undefined) { + if (hit.kind === "negative") { + return null; + } + const node = readNode(db, hit.inode); + if (node !== null) { + return toResolved(node); + } + } + + // One recursive-CTE statement resolves the common symlink-free + // case. Any symlink on the path falls back to the per-component loop, + // which follows links and enforces ELOOP; those resolutions are not + // cached (a followed path is an alias whose invalidation can't be + // reasoned about structurally). + const cte = resolveViaCte(db, parts); + if (cte.kind === "symlink") { + return resolveParts(db, parts, followFinal, 0); + } + storeResolveCache(db, canonical, cte.node === null ? null : cte.node.inode); + return cte.node; +} + +interface CteRow { + level: number; + inode: number; + type: "file" | "dir" | "symlink"; + mode: number; + mtime: number; + size: number; + link_target: string | null; +} + +type CteResolution = + // Walk completed with no symlink on the path: `node` is the resolved + // final node, or null when a segment was missing or an intermediate + // was not a directory (both map to null, exactly like the loop). + | { kind: "resolved"; node: ResolvedInode | null } + // A symlink was encountered anywhere on the path (intermediate or + // final). The CTE can't follow links, so the caller must fall back to + // the loop for byte-identical follow / ELOOP / dangling behaviour. + | { kind: "symlink" }; + +// Single-statement path walk. Binds the canonical path segments as a +// JSON array and walks vfs_dirents -> vfs_nodes from ROOT_INODE, one +// level per segment. Descends only through directories (WHERE +// w.type = 'dir'), so a file intermediate stalls the walk (ENOTDIR) +// and a missing dirent produces no row (ENOENT) — both surface as a +// missing level-D row, matching the loop's null. Every node the walk +// touches is returned so the caller can detect any symlink and fall +// back. +function resolveViaCte(db: Database, parts: string[]): CteResolution { + const rows = db.all( + `WITH RECURSIVE + segs(level, name) AS ( + SELECT key, value FROM json_each(?) + ), + walk(level, inode, type, mode, mtime, size, link_target) AS ( + SELECT 0, n.inode, n.type, n.mode, n.mtime, n.size, n.link_target + FROM vfs_nodes n + WHERE n.inode = ? + UNION ALL + SELECT w.level + 1, n.inode, n.type, n.mode, n.mtime, n.size, n.link_target + FROM walk w + JOIN segs s ON s.level = w.level + JOIN vfs_dirents d ON d.parent_inode = w.inode AND d.name = s.name + JOIN vfs_nodes n ON n.inode = d.child_inode + WHERE w.type = 'dir' + ) + SELECT level, inode, type, mode, mtime, size, link_target + FROM walk + ORDER BY level`, + JSON.stringify(parts), + ROOT_INODE, + ); + + const depth = parts.length; + let target: CteRow | undefined; + for (const row of rows) { + // Any symlink on the walk (root is level 0 and always a dir) means + // the loop must take over to follow it. + if (row.level >= 1 && row.type === "symlink") { + return { kind: "symlink" }; + } + if (row.level === depth) { + target = row; + } + } + return { + kind: "resolved", + node: target === undefined ? null : toResolved(target), + }; +} + +function toResolved(node: NodeRow): ResolvedInode { + return { + inode: node.inode, + type: node.type, + mode: node.mode, + mtime: node.mtime, + size: node.size, + linkTarget: node.link_target ?? undefined, + }; +} + +function resolveParts( + db: Database, + parts: string[], + followFinal: boolean, + follows: number, +): ResolvedInode | null { + const root = readNode(db, ROOT_INODE); + if (root === null) { + return null; + } + + let current: NodeRow = root; + for (let i = 0; i < parts.length; i++) { + const isFinal = i === parts.length - 1; + if (current.type !== "dir") { + return null; + } + const child = db.one( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + current.inode, + parts[i], + ); + if (child === undefined) { + return null; + } + const next = readNode(db, child.child_inode); + if (next === null) { + return null; + } + // Intermediate symlinks always get followed; final-segment symlinks + // are only followed when the caller wants. A dangling intermediate + // is the same as a missing intermediate (return null). + if (next.type === "symlink" && (!isFinal || followFinal)) { + follows += 1; + if (follows > MAX_SYMLINK_FOLLOWS) { + throw createWorkspaceError("ELOOP", "too many symlinks resolving path"); + } + const target = next.link_target ?? ""; + const resolved = resolveParts(db, canonicalizePath(target).parts, true, follows); + if (resolved === null) { + return null; + } + // Replace the current dirent-resolved node with the followed + // result, then keep walking remaining segments (if any). + current = { + inode: resolved.inode, + type: resolved.type, + mode: resolved.mode, + mtime: resolved.mtime, + size: resolved.size, + link_target: resolved.linkTarget ?? null, + }; + continue; + } + current = next; + } + + return { + inode: current.inode, + type: current.type, + mode: current.mode, + mtime: current.mtime, + size: current.size, + linkTarget: current.link_target ?? undefined, + }; +} + +function readNode(db: Database, inode: number): NodeRow | null { + const row = db.one( + "SELECT inode, type, mode, mtime, size, link_target FROM vfs_nodes WHERE inode = ?", + inode, + ); + return row ?? null; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/resolveCache.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/resolveCache.ts new file mode 100644 index 00000000..0b40f054 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/resolveCache.ts @@ -0,0 +1,129 @@ +// Per-Database path -> inode resolution cache. +// +// Maps a canonical absolute path (as produced by +// canonicalizePath().path) to the inode that resolveInode(path, +// { followSymlinks: true }) lands on, or a NEGATIVE marker when the +// path does not resolve. It turns repeat stat/exists/read of the same +// path into an O(1) lookup instead of an O(depth) walk. +// +// Deliberately narrow, for correctness: +// +// * Only the path -> inode MAPPING is cached. resolveInode always +// re-reads the node row on a hit, so content changes (chmod, +// size/mtime, type) are never served stale — only structural +// mutations that move a dirent can invalidate an entry. +// +// * Only symlink-free resolutions are cached. Following a symlink +// makes the cached path an alias of the target whose invalidation +// can't be reasoned about from the path alone, so resolveInode +// stores nothing when a symlink was traversed. +// +// * Population is gated on Database.inTransaction: entries are only +// written outside a transaction, so a rolled-back mutation can +// never leave a positive/negative entry reflecting uncommitted +// state. Mutations invalidate (drop) freely — dropping is safe +// under rollback because the worst case is a recompute. +// +// The cache is per-Database (WeakMap) and bounded (LRU by Map +// insertion order, same discipline as blobCache). + +import type { Database } from "../storage.js"; + +// Sentinel value for "this path resolves to nothing" (ENOENT/ENOTDIR). +const NEGATIVE = -1; + +// Upper bound on cached paths per Database. Entries are tiny (a string +// key and a number), so this caps memory at a few MB while covering +// the working set of a busy tree. +const MAX_ENTRIES = 8192; + +// Keyed by the Database instance, so correctness assumes exactly one +// Database wraps each SqlStorage. Two Databases over the same storage +// would hold independent caches and could serve each other stale +// results; the DO owns a single Database, which upholds this. +const caches = new WeakMap>(); + +function cacheFor(db: Database): Map { + let cache = caches.get(db); + if (cache === undefined) { + cache = new Map(); + caches.set(db, cache); + } + return cache; +} + +export type ResolveCacheHit = { kind: "inode"; inode: number } | { kind: "negative" }; + +// Look up a canonical path. Returns undefined on a miss, a positive +// inode hit, or a negative (known-absent) hit. Bumps LRU recency. +export function lookupResolveCache( + db: Database, + canonicalPath: string, +): ResolveCacheHit | undefined { + const cache = cacheFor(db); + const value = cache.get(canonicalPath); + if (value === undefined) { + return undefined; + } + // Move to most-recent position for LRU eviction. + cache.delete(canonicalPath); + cache.set(canonicalPath, value); + return value === NEGATIVE ? { kind: "negative" } : { kind: "inode", inode: value }; +} + +// Cache a resolution. `inode === null` records a negative entry. No-op +// while a transaction is active so the cache never reflects +// uncommitted state (rollback safety). +export function storeResolveCache(db: Database, canonicalPath: string, inode: number | null): void { + if (db.inTransaction) { + return; + } + const cache = cacheFor(db); + cache.set(canonicalPath, inode === null ? NEGATIVE : inode); + while (cache.size > MAX_ENTRIES) { + const oldest = cache.keys().next(); + if (oldest.done === true) { + break; + } + cache.delete(oldest.value); + } +} + +// Drop the entry for exactly `canonicalPath`. Use after a mutation +// that changes a single leaf's existence without affecting anything +// beneath it: creating/removing a file, symlink, hardlink, or an +// empty directory. O(1). +export function invalidateResolveExact(db: Database, canonicalPath: string): void { + const cache = caches.get(db); + cache?.delete(canonicalPath); +} + +// Drop `canonicalPath` and every entry beneath it (keys prefixed +// `canonicalPath + "/"`). Use when a mutation changes a whole subtree's +// resolution: a recursive delete, any directory rename (every +// descendant's path changes), a structural subtree replacement, or a +// symlink create (paths *through* the new link become resolvable, so +// stale negatives beneath it must go). Root ("/") clears everything. +export function invalidateResolveSubtree(db: Database, canonicalPath: string): void { + const cache = caches.get(db); + if (cache === undefined || cache.size === 0) { + return; + } + if (canonicalPath === "/") { + cache.clear(); + return; + } + cache.delete(canonicalPath); + const prefix = `${canonicalPath}/`; + for (const key of cache.keys()) { + if (key.startsWith(prefix)) { + cache.delete(key); + } + } +} + +// Drop the entire cache for a Database. Used by tests and available as +// a blunt reset. +export function clearResolveCache(db: Database): void { + caches.get(db)?.clear(); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/rm.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/rm.ts new file mode 100644 index 00000000..cf82fb8e --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/rm.ts @@ -0,0 +1,181 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { incrementRev } from "../rev.js"; +import type { Database } from "../storage.js"; +import { recordDelete } from "../sync/changes.js"; +import { pathOf } from "../sync/paths.js"; +import { assertNotReadOnly } from "./mount-guard.js"; +import { resolveInode } from "./resolve.js"; +import { invalidateResolveExact, invalidateResolveSubtree } from "./resolveCache.js"; +import { unlinkDirent } from "./unlink.js"; + +export interface RmOptions { + recursive?: boolean; + force?: boolean; +} + +interface DirChild { + name: string; + child_inode: number; + type: "file" | "dir" | "symlink"; +} + +// Walk a directory subtree post-order so we delete leaves before +// parents. Yields each node together with the parent inode and name +// the walk already knows, so the caller can unlink the dirent by +// (parent, name) without re-resolving the parent from root. The caller +// appends one tombstone per yielded path and clears vfs_chunks for +// file inodes. +function* walkPostOrder( + db: Database, + rootInode: number, + rootPath: string, + rootParentInode: number, + rootName: string, +): Generator<{ + path: string; + inode: number; + type: "file" | "dir" | "symlink"; + parentInode: number; + name: string; +}> { + // Stack-based DFS to avoid recursion limits on deep trees. + type Frame = { + inode: number; + path: string; + type: "file" | "dir" | "symlink"; + parentInode: number; + name: string; + expanded: boolean; + }; + const stack: Frame[] = [ + { + inode: rootInode, + path: rootPath, + type: "dir", + parentInode: rootParentInode, + name: rootName, + expanded: false, + }, + ]; + + while (stack.length > 0) { + const top = stack[stack.length - 1]; + if (top.type !== "dir" || top.expanded) { + stack.pop(); + yield { + path: top.path, + inode: top.inode, + type: top.type, + parentInode: top.parentInode, + name: top.name, + }; + continue; + } + top.expanded = true; + const children = db.all( + `SELECT d.name AS name, d.child_inode AS child_inode, n.type AS type + FROM vfs_dirents d + JOIN vfs_nodes n ON n.inode = d.child_inode + WHERE d.parent_inode = ? + ORDER BY d.name`, + top.inode, + ); + for (const child of children) { + const childPath = top.path === "/" ? `/${child.name}` : `${top.path}/${child.name}`; + stack.push({ + inode: child.child_inode, + path: childPath, + type: child.type, + parentInode: top.inode, + name: child.name, + expanded: false, + }); + } + } +} + +export function rm(db: Database, path: string, options: RmOptions): void { + const { parts, path: canonical } = canonicalizePath(path); + + if (parts.length === 0) { + // The workspace root is structural; refuse to delete it even with + // recursive+force. Matches the doc's example. + throw createWorkspaceError("EPERM", `cannot remove the root directory`, canonical); + } + + // assertNotReadOnly uses the symmetric overlap predicate, so a + // recursive rm of an ancestor whose subtree contains a read-only + // mount root is caught here without walking the tree. + assertNotReadOnly(db, canonical); + + const force = options.force === true; + const recursive = options.recursive === true; + + db.transactionSync(() => { + const node = resolveInode(db, canonical, { followSymlinks: false }); + if (node === null) { + if (force) return; + throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); + } + + if (node.type === "dir" && !recursive) { + const childCount = db.scalar( + "SELECT COUNT(*) FROM vfs_dirents WHERE parent_inode = ?", + node.inode, + ); + if ((childCount ?? 0) > 0) { + throw createWorkspaceError("ENOTEMPTY", `directory not empty: ${canonical}`, canonical); + } + } + + // Resolve the entry's real path from its parent rather than from + // the inode: a hardlinked file has several names, and pathOf would + // pick an arbitrary one. Following symlinks on the parent lets a + // request through a symlinked directory land on the real container + // while still removing exactly the requested name. + const name = parts[parts.length - 1]; + const parentPath = parts.length === 1 ? "/" : `/${parts.slice(0, -1).join("/")}`; + const parent = resolveInode(db, parentPath); + if (parent === null || parent.type !== "dir") { + throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); + } + const parentReal = pathOf(db, parent.inode); + if (parentReal === null) { + throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); + } + const realPath = parentReal === "/" ? `/${name}` : `${parentReal}/${name}`; + assertNotReadOnly(db, realPath); + + const rev = incrementRev(db); + + if (node.type !== "dir" || !recursive) { + // Single entry removal — file, symlink, or empty directory. A + // file inode may have multiple dirents (hardlinks), so remove + // only the requested name and reap chunks/node after the final + // link disappears. `parent` is already resolved above, so unlink + // by (parent, name) directly rather than re-resolving. The + // tombstone is recorded at the resolved real path so sync sees + // the move-aware location. + unlinkDirent(db, parent.inode, name, node.inode, node.type); + recordDelete(db, rev, realPath); + // A single removed entry is a file, symlink, or empty directory: + // no cached descendants to worry about, so drop it exact. + invalidateResolveExact(db, realPath); + return; + } + + // Recursive directory removal. Walk leaves first so each delete + // sees an empty parent by the time we get to it. File entries may + // be hardlinked outside this subtree, so delete by path rather + // than by child inode. The walk carries each node's parent inode + // and name, so unlinkDirent needs no per-node re-resolve from root. + for (const entry of walkPostOrder(db, node.inode, realPath, parent.inode, name)) { + unlinkDirent(db, entry.parentInode, entry.name, entry.inode, entry.type); + recordDelete(db, rev, entry.path); + } + // The whole subtree under realPath is gone; one subtree drop covers + // every descendant's cached resolution. + invalidateResolveSubtree(db, realPath); + }); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/stat.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/stat.ts new file mode 100644 index 00000000..4ce2734f --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/stat.ts @@ -0,0 +1,86 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import type { Database } from "../storage.js"; +import { resolveInode } from "./resolve.js"; +import { getPendingWriteBufferByPath, getWriteBuffer } from "./writeBuffer.js"; + +export interface WorkspaceStatResult { + name: string; + // Inode of the resolved node, or 0 for a pending-create file that + // has no inode yet. Exposed so provider stat surfaces can read the + // inode from the same resolve instead of walking the path twice. + inode: number; + mode: number; + mtime: number; + size: number; + isFile: boolean; + isDirectory: boolean; + // True when the result describes a symlink itself rather than + // its target. Only lstat() can produce a true value here; stat() + // follows links and reports the final node. + isSymbolicLink: boolean; +} + +export function stat(db: Database, path: string): WorkspaceStatResult { + return statShared(db, path, true); +} + +// Like stat, but does not follow a trailing symlink. Mirrors POSIX +// lstat: the returned size for a symlink is the byte length of the +// stored target, and mode is the symlink node's own mode. +export function lstat(db: Database, path: string): WorkspaceStatResult { + return statShared(db, path, false); +} + +function statShared(db: Database, path: string, followFinal: boolean): WorkspaceStatResult { + const { name, path: canonical } = canonicalizePath(path); + // Pending-create files have no inode yet; serve the buffer state + // so callers between create and release see the file as it stands. + // Pending creates never apply to symlinks, so this is safe to run + // even on the lstat path — a hit here always corresponds to a + // file mid-open. + const pending = getPendingWriteBufferByPath(db, canonical); + if (pending !== undefined && pending.pending !== undefined) { + return { + name, + // A pending create has no inode until releaseWriteBufferSync + // commits it; report 0, which yields nlink 1 in the provider. + inode: 0, + mode: pending.mode & 0o7777, + mtime: pending.pending.mtime, + size: pending.size, + isFile: true, + isDirectory: false, + isSymbolicLink: false, + }; + } + const node = resolveInode(db, path, { followSymlinks: followFinal }); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such path: ${path}`, path); + } + + const isDirectory = node.type === "dir"; + const isFile = node.type === "file"; + const isSymbolicLink = node.type === "symlink"; + let size = 0; + if (isFile) { + // Prefer the in-memory buffer when an open file has unflushed + // writes; otherwise read the cached size off vfs_nodes that + // resolveInode just loaded for us, no extra SQL. + const buffered = getWriteBuffer(db, node.inode); + size = buffered?.dirty ? buffered.size : node.size; + } else if (isSymbolicLink) { + size = (node.linkTarget ?? "").length; + } + + return { + name, + inode: node.inode, + mode: node.mode, + mtime: node.mtime, + size, + isFile, + isDirectory, + isSymbolicLink, + }; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/symlink.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/symlink.ts new file mode 100644 index 00000000..cbd6f2e9 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/symlink.ts @@ -0,0 +1,81 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { incrementRev } from "../rev.js"; +import { ROOT_INODE } from "../schema/index.js"; +import type { Database } from "../storage.js"; +import { assertNotReadOnly } from "./mount-guard.js"; +import { invalidateResolveSubtree } from "./resolveCache.js"; + +// Create a symlink node. The target is stored as-is — it can be a +// relative or absolute path, dangling or live. resolveInode follows +// it transparently when callers walk through this entry. +export function symlink(db: Database, target: string, path: string, now: () => number): void { + const { parts, path: canonical } = canonicalizePath(path); + if (parts.length === 0) { + throw createWorkspaceError("EEXIST", "cannot symlink onto root", canonical); + } + assertNotReadOnly(db, canonical); + + db.transactionSync(() => { + // Walk to the parent dirent. Intermediate segments must be real + // directories; we don't auto-create them. + let parentInode = ROOT_INODE; + for (let i = 0; i < parts.length - 1; i++) { + const child = db.one<{ child_inode: number }>( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + parentInode, + parts[i], + ); + if (child === undefined) { + throw createWorkspaceError("ENOENT", `parent directory missing: ${canonical}`, canonical); + } + const next = db.one<{ inode: number; type: "file" | "dir" | "symlink" }>( + "SELECT inode, type FROM vfs_nodes WHERE inode = ?", + child.child_inode, + ); + if (next === undefined || next.type !== "dir") { + throw createWorkspaceError( + "ENOTDIR", + `parent path segment is not a directory: ${canonical}`, + canonical, + ); + } + parentInode = next.inode; + } + + const leafName = parts[parts.length - 1]; + const existing = db.one<{ child_inode: number }>( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + parentInode, + leafName, + ); + if (existing !== undefined) { + throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); + } + + const rev = incrementRev(db); + const mtime = now(); + // RETURNING folds the rowid read into the INSERT. + const row = db.one<{ inode: number }>( + "INSERT INTO vfs_nodes (type, mode, mtime, rev, link_target) VALUES ('symlink', ?, ?, ?, ?) RETURNING inode", + 0o777, + mtime, + rev, + target, + ); + if (row === undefined) { + throw createWorkspaceError("EIO", "failed to allocate inode"); + } + const inode = row.inode; + db.run( + "INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", + parentInode, + leafName, + inode, + ); + // Subtree, not exact: paths *through* the new link (e.g. /s/x when + // /s -> a populated dir) now resolve, so any cached negative + // beneath the link must be dropped. + invalidateResolveSubtree(db, canonical); + }); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/unlink.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/unlink.ts new file mode 100644 index 00000000..7a7d33c2 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/unlink.ts @@ -0,0 +1,34 @@ +import type { Database } from "../storage.js"; + +type NodeType = "file" | "dir" | "symlink"; + +// Remove a single (parent, name) dirent and reap the child inode's +// node and chunk rows only once its last link disappears. A file inode +// can carry several hardlink names, so the node and its chunks survive +// until the final dirent is gone. Returns true when the inode was +// reaped, false when other links keep it alive. +// +// Callers own rev bumps and tombstones; this helper touches only +// vfs_dirents, vfs_chunks, and vfs_nodes. It is the single place the +// refcount-gated reap is implemented — rm, rename, and the sync apply +// path all funnel through here so the invariant lives once. +export function unlinkDirent( + db: Database, + parentInode: number, + name: string, + childInode: number, + type: NodeType, +): boolean { + db.run("DELETE FROM vfs_dirents WHERE parent_inode = ? AND name = ?", parentInode, name); + const remaining = db.scalar( + "SELECT COUNT(*) FROM vfs_dirents WHERE child_inode = ?", + childInode, + ); + if ((remaining ?? 0) > 0) return false; + + if (type === "file") { + db.run("DELETE FROM vfs_chunks WHERE inode = ?", childInode); + } + db.run("DELETE FROM vfs_nodes WHERE inode = ?", childInode); + return true; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/writeBuffer.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/writeBuffer.ts new file mode 100644 index 00000000..72ff047a --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/writeBuffer.ts @@ -0,0 +1,138 @@ +// In-process write buffer cache. +// +// Holds per-inode mutable byte buffers between an explicit open and +// release. While a buffer is open, all reads and writes for that +// inode go through the buffer rather than the SQLite blob/chunk +// store. Release commits the bytes to chunks once per file +// and evicts the entry, so per-syscall writes no longer accumulate +// orphan blob rows in the store. +// +// The cache is keyed by Database so a fresh database (a test, a +// rebooted DO incarnation) starts with an empty cache. + +import type { Database } from "../storage.js"; + +export interface WriteBufferEntry { + // Growable backing store. byteLength is capacity; logical length + // lives in `size`. + buf: Uint8Array; + // Logical end-of-file in `buf`. + size: number; + // True once writeRange/truncate mutates the buffer. A non-dirty + // buffer is one that the caller opened but never wrote to; release + // is a no-op in that case so we do not touch the existing chunks. + dirty: boolean; + // Open handle count. Each FUSE open/create increments this; each + // release decrements. The buffer commits and evicts when the count + // reaches zero. + openCount: number; + // Mode the caller wants persisted on release. Defaults to the + // inode's existing mode at open time when the caller has none. + mode: number; + // Pending-create state. When set, no inode row exists yet; release + // will INSERT the node + dirent + chunks in one transaction. The + // synthetic inode id used to key this entry in the cache is stored + // here so release can find and remove the entry without scanning + // the cache. + pending?: { + parentInode: number; + leafName: string; + canonicalPath: string; + pendingInode: number; + mtime: number; + }; +} + +interface DatabaseCache { + byInode: Map; + byPendingPath: Map; + nextPendingInode: number; +} + +const caches = new WeakMap(); + +function cacheFor(db: Database): DatabaseCache { + let cache = caches.get(db); + if (cache === undefined) { + cache = { byInode: new Map(), byPendingPath: new Map(), nextPendingInode: -1 }; + caches.set(db, cache); + } + return cache; +} + +export function getWriteBuffer(db: Database, inode: number): WriteBufferEntry | undefined { + return caches.get(db)?.byInode.get(inode); +} + +export function getPendingWriteBufferByPath( + db: Database, + canonicalPath: string, +): WriteBufferEntry | undefined { + return caches.get(db)?.byPendingPath.get(canonicalPath); +} + +// List pending-create buffers whose parent dirent matches `parentInode`. +// Used by readdir so freshly-created-but-not-yet-released files show +// up in directory listings between open and release. +export function listPendingByParent(db: Database, parentInode: number): WriteBufferEntry[] { + const cache = caches.get(db); + if (cache === undefined) return []; + const out: WriteBufferEntry[] = []; + for (const entry of cache.byPendingPath.values()) { + if (entry.pending?.parentInode === parentInode) out.push(entry); + } + return out; +} + +export function setWriteBuffer(db: Database, inode: number, entry: WriteBufferEntry): void { + const cache = cacheFor(db); + cache.byInode.set(inode, entry); + if (entry.pending !== undefined) { + cache.byPendingPath.set(entry.pending.canonicalPath, entry); + } +} + +export function deleteWriteBuffer(db: Database, inode: number): void { + const cache = caches.get(db); + if (cache === undefined) return; + const entry = cache.byInode.get(inode); + if (entry?.pending !== undefined) { + cache.byPendingPath.delete(entry.pending.canonicalPath); + } + cache.byInode.delete(inode); +} + +// Allocate a synthetic negative inode id for a pending file. The +// real id is assigned by SQLite when release INSERTs the node row; +// the synthetic value just lets the buffer cache key entries +// before that point. +export function allocatePendingInode(db: Database): number { + const cache = cacheFor(db); + const next = cache.nextPendingInode; + cache.nextPendingInode -= 1; + return next; +} + +// Re-key a pending entry to the real inode assigned by SQLite at +// commit time, dropping the pending-path index. +export function promotePendingToInode(db: Database, pendingInode: number, realInode: number): void { + const cache = caches.get(db); + if (cache === undefined) return; + const entry = cache.byInode.get(pendingInode); + if (entry === undefined) return; + if (entry.pending !== undefined) { + cache.byPendingPath.delete(entry.pending.canonicalPath); + entry.pending = undefined; + } + cache.byInode.delete(pendingInode); + cache.byInode.set(realInode, entry); +} + +export function ensureCapacity(entry: WriteBufferEntry, needed: number): void { + if (entry.buf.byteLength >= needed) return; + let cap = Math.max(entry.buf.byteLength * 2, 64 * 1024); + while (cap < needed) cap *= 2; + const next = new Uint8Array(cap); + next.set(entry.buf.subarray(0, entry.size), 0); + entry.buf = next; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/writeFile.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/writeFile.ts new file mode 100644 index 00000000..48f6fc53 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/fs/writeFile.ts @@ -0,0 +1,1070 @@ +import { createHash } from "node:crypto"; +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { incrementRev } from "../rev.js"; +import { ROOT_INODE } from "../schema/index.js"; +import type { Database } from "../storage.js"; +import { stageBlob } from "../sync/blobs.js"; +import { buildManifest } from "../sync/manifests.js"; +import { getBlobBytes } from "./blobCache.js"; +import { assertNotReadOnly } from "./mount-guard.js"; +import { invalidateResolveExact } from "./resolveCache.js"; +import { + allocatePendingInode, + deleteWriteBuffer, + ensureCapacity as ensureBufferCapacity, + getPendingWriteBufferByPath, + getWriteBuffer, + promotePendingToInode, + setWriteBuffer, + type WriteBufferEntry, +} from "./writeBuffer.js"; + +// Fixed chunk size. Exported so tests can size inputs precisely +// without hard-coding the magic number twice. +export const CHUNK_SIZE = 512 * 1024; + +export type WriteFileContent = string | Uint8Array | ReadableStream; + +export interface WriteFileOptions { + mode?: number; + /** Fail with EEXIST when the target already exists. */ + exclusive?: boolean; +} + +export interface WriteFileRange { + start: number; + end: number; +} + +// Resolve directory-only paths (the parent of the target file). The +// final segment is handled by the caller. Returns the parent inode or +// throws ENOENT/ENOTDIR. +function resolveParent(db: Database, parts: string[], canonical: string): number { + let parentInode = ROOT_INODE; + for (let i = 0; i < parts.length - 1; i++) { + const name = parts[i]; + const child = db.one<{ child_inode: number }>( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + parentInode, + name, + ); + if (child === undefined) { + throw createWorkspaceError("ENOENT", `parent directory missing: ${canonical}`, canonical); + } + const next = db.one<{ inode: number; type: "file" | "dir" }>( + "SELECT inode, type FROM vfs_nodes WHERE inode = ?", + child.child_inode, + ); + if (next === undefined) { + throw createWorkspaceError("ENOENT", `dangling dirent: ${canonical}`, canonical); + } + if (next.type !== "dir") { + throw createWorkspaceError( + "ENOTDIR", + `parent path segment is not a directory: ${canonical}`, + canonical, + ); + } + parentInode = next.inode; + } + return parentInode; +} + +async function materialize(content: string | Uint8Array): Promise { + if (typeof content === "string") { + return new TextEncoder().encode(content); + } + return content; +} + +// sha256 with a synchronous code path so writeFile can be called both +// from async drivers (the FS API) and from sync drivers (the +// VirtualProvider). node:crypto is available natively on Node and +// polyfilled by workerd. +function sha256(bytes: Uint8Array): Uint8Array { + const hash = createHash("sha256"); + hash.update(bytes); + return new Uint8Array(hash.digest()); +} + +interface PreparedChunk { + hash: Uint8Array; + bytes: Uint8Array; + size: number; +} + +interface ChunkRef { + hash: Uint8Array; + size: number; +} + +export function chunksOf(bytes: Uint8Array): PreparedChunk[] { + const chunks: PreparedChunk[] = []; + for (let offset = 0; offset < bytes.byteLength; offset += CHUNK_SIZE) { + const end = Math.min(offset + CHUNK_SIZE, bytes.byteLength); + // subarray (not slice) avoids an extra copy; sha256() takes its own + // copy when needed. + const slice = bytes.subarray(offset, end); + const hash = sha256(slice); + chunks.push({ hash, bytes: slice, size: slice.byteLength }); + } + return chunks; +} + +export async function writeFile( + db: Database, + path: string, + content: WriteFileContent, + options: WriteFileOptions, + now: () => number, +): Promise { + if (content instanceof ReadableStream) { + await writeFileStreaming(db, path, content, options, now); + return; + } + const bytes = await materialize(content); + writeFileSync(db, path, bytes, options, now); +} + +// Streaming write path. Reads the source one source-chunk at a time, +// re-windows into fixed CHUNK_SIZE pieces, hashes each window, and +// stages it into vfs_blobs / vfs_blob_bytes as it goes. The final +// inode / dirent / vfs_chunks / manifest writes happen in a single +// short transaction once the source is drained, against a list of +// {hash, size} entries that's O(file_size / CHUNK_SIZE) bytes — not +// O(file_size). +// +// Failure mid-stream leaves blob rows behind; gc() reaps orphans on +// its next pass since no node references them. +async function writeFileStreaming( + db: Database, + path: string, + source: ReadableStream, + options: WriteFileOptions, + now: () => number, +): Promise { + const { parts, path: canonical } = canonicalizePath(path); + if (parts.length === 0) { + throw createWorkspaceError("EISDIR", "cannot write to the root directory", canonical); + } + // Reject before we stage any blob bytes so known failures do not grow + // orphan blob rows that gc() then has to reap. + assertNotReadOnly(db, canonical); + if (options.exclusive) { + const parentInode = resolveParent(db, parts, canonical); + const existing = db.one( + "SELECT 1 FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + parentInode, + parts[parts.length - 1], + ); + if (existing !== undefined) { + throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); + } + } + const mode = (options.mode ?? 0o644) & 0o7777; + const mtime = now(); + + const chunkRefs: Array<{ hash: Uint8Array; size: number }> = []; + // Carry-over buffer: bytes left over from the previous source chunk + // that didn't fill a CHUNK_SIZE window. + let carry: Uint8Array | undefined; + + const flush = (chunk: Uint8Array): void => { + const hash = sha256(chunk); + stageBlob(db, hash, chunk, mtime); + chunkRefs.push({ hash, size: chunk.byteLength }); + }; + + const reader = source.getReader(); + try { + while (true) { + const { value, done } = await reader.read(); + if (done) break; + if (value === undefined || value.byteLength === 0) continue; + let input = value; + if (carry !== undefined) { + // Splice carry-over onto the front of this source chunk so + // we can re-window cleanly. + const merged = new Uint8Array(carry.byteLength + input.byteLength); + merged.set(carry, 0); + merged.set(input, carry.byteLength); + input = merged; + carry = undefined; + } + let offset = 0; + while (input.byteLength - offset >= CHUNK_SIZE) { + // Copy the window so the staged blob doesn't alias a + // larger backing buffer. + const window = input.slice(offset, offset + CHUNK_SIZE); + flush(window); + offset += CHUNK_SIZE; + } + if (offset < input.byteLength) { + carry = input.slice(offset); + } + } + } finally { + reader.releaseLock(); + } + if (carry !== undefined && carry.byteLength > 0) { + flush(carry); + } + + // Wire up the inode against the staged blobs in one short + // transaction. From this point on the SQL is the same shape as the + // synchronous path — only the chunk-bytes step is skipped because + // stageBlob already landed them above. + db.transactionSync(() => { + const parentInode = resolveParent(db, parts, canonical); + const leafName = parts[parts.length - 1]; + const existing = db.one<{ child_inode: number }>( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + parentInode, + leafName, + ); + let inode: number; + if (existing !== undefined) { + if (options.exclusive) { + throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); + } + const node = db.one<{ type: "file" | "dir" }>( + "SELECT type FROM vfs_nodes WHERE inode = ?", + existing.child_inode, + ); + if (node?.type === "dir") { + throw createWorkspaceError("EISDIR", `path is a directory: ${canonical}`, canonical); + } + inode = existing.child_inode; + db.run("DELETE FROM vfs_chunks WHERE inode = ?", inode); + } else { + inode = insertFileNode(db, mode, mtime); + insertFileDirent(db, parentInode, leafName, inode, canonical); + } + for (let idx = 0; idx < chunkRefs.length; idx++) { + const ref = chunkRefs[idx]; + db.run( + "INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", + inode, + idx, + ref.hash, + ref.size, + ); + } + const manifestHash = buildManifest(db, chunkRefs, mtime); + const rev = incrementRev(db); + let totalSize = 0; + for (const ref of chunkRefs) totalSize += ref.size; + db.run( + "UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ?, size = ?, manifest_hash = ? WHERE inode = ?", + mode, + mtime, + rev, + totalSize, + manifestHash, + inode, + ); + }); +} + +// Allocate a fresh file inode row with the supplied mode and mtime, +// using SQLite's RETURNING so the new rowid comes back in the same +// statement instead of through a follow-up SELECT last_insert_rowid(). +// Link a freshly created file inode into its parent directory and drop +// any cached negative resolution for the new path. The single choke +// point for every new-file dirent, so the resolve cache stays correct +// on create without touching the overwrite path (which reuses the +// existing inode and dirent, so its resolution is unchanged). A new +// file is a leaf with no descendants, so exact invalidation suffices. +function insertFileDirent( + db: Database, + parentInode: number, + leafName: string, + childInode: number, + canonicalPath: string, +): void { + db.run( + "INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", + parentInode, + leafName, + childInode, + ); + invalidateResolveExact(db, canonicalPath); +} + +function insertFileNode(db: Database, mode: number, mtime: number): number { + const row = db.one<{ inode: number }>( + "INSERT INTO vfs_nodes (type, mode, mtime, rev) VALUES ('file', ?, ?, 0) RETURNING inode", + mode, + mtime, + ); + if (row === undefined) { + throw createWorkspaceError("EIO", "failed to allocate inode"); + } + return row.inode; +} + +function upsertChunkBlob(db: Database, chunk: PreparedChunk, lastSeen: number): void { + db.run( + "INSERT INTO vfs_blobs (hash, size, last_seen) VALUES (?, ?, ?) ON CONFLICT(hash) DO UPDATE SET last_seen = excluded.last_seen", + chunk.hash, + chunk.size, + lastSeen, + ); + db.run( + "INSERT INTO vfs_blob_bytes (hash, bytes) VALUES (?, ?) ON CONFLICT(hash) DO NOTHING", + chunk.hash, + chunk.bytes, + ); +} + +function replaceChunkRows( + db: Database, + inode: number, + chunks: ChunkRef[], + manifestTime: number, +): Uint8Array { + db.run("DELETE FROM vfs_chunks WHERE inode = ?", inode); + for (let idx = 0; idx < chunks.length; idx++) { + const chunk = chunks[idx]; + db.run( + "INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", + inode, + idx, + chunk.hash, + chunk.size, + ); + } + return buildManifest(db, chunks, manifestTime); +} + +function rangesOverlap(start: number, end: number, ranges: WriteFileRange[]): boolean { + for (const range of ranges) { + if (range.start < end && start < range.end) return true; + } + return false; +} + +function normalizeRanges(ranges: WriteFileRange[], size: number): WriteFileRange[] { + const normalized = ranges + .map((range) => ({ + start: Math.max(0, Math.min(size, Math.floor(range.start))), + end: Math.max(0, Math.min(size, Math.ceil(range.end))), + })) + .filter((range) => range.start < range.end) + .sort((a, b) => a.start - b.start); + + const merged: WriteFileRange[] = []; + for (const range of normalized) { + const previous = merged.at(-1); + if (previous === undefined || previous.end < range.start) { + merged.push({ ...range }); + } else { + previous.end = Math.max(previous.end, range.end); + } + } + return merged; +} + +function existingChunkRefs(db: Database, inode: number): ChunkRef[] { + return db.all("SELECT hash, size FROM vfs_chunks WHERE inode = ? ORDER BY idx", inode); +} + +function fileSizeForInode(db: Database, inode: number): number { + return db.scalar("SELECT size FROM vfs_nodes WHERE inode = ?", inode) ?? 0; +} + +function readChunkBytes(db: Database, inode: number, idx: number): Uint8Array { + const chunk = db.one<{ hash: Uint8Array }>( + "SELECT hash FROM vfs_chunks WHERE inode = ? AND idx = ?", + inode, + idx, + ); + if (chunk === undefined) return new Uint8Array(); + const bytes = getBlobBytes(db, chunk.hash); + if (bytes === undefined) { + throw createWorkspaceError("EIO", "missing blob bytes"); + } + return bytes; +} + +function resolveFileInode(db: Database, path: string): { inode: number; mode: number } { + const { path: canonical } = canonicalizePath(path); + const node = db.one<{ inode: number; type: "file" | "dir"; mode: number }>( + `SELECT n.inode AS inode, n.type AS type, n.mode AS mode + FROM vfs_nodes n + WHERE n.inode = ( + SELECT child_inode + FROM vfs_dirents + WHERE parent_inode = ? AND name = ? + )`, + ...parentAndNameForResolvedPath(db, path), + ); + if (node === undefined) { + throw createWorkspaceError("ENOENT", `no such file: ${canonical}`, canonical); + } + if (node.type !== "file") { + throw createWorkspaceError("EISDIR", `path is a directory: ${canonical}`, canonical); + } + return { inode: node.inode, mode: node.mode }; +} + +function parentAndNameForResolvedPath(db: Database, path: string): [number, string] { + const { parts, path: canonical } = canonicalizePath(path); + if (parts.length === 0) { + throw createWorkspaceError("EISDIR", "cannot write to the root directory", canonical); + } + return [resolveParent(db, parts, canonical), parts[parts.length - 1]]; +} + +// Update an inode's chunk-backed representation in place. Iterates over +// the full chunk grid but only touches `vfs_chunks` rows whose contents +// or size actually changed, so untouched chunk rows keep their +// rowids and the surrounding rows do not churn. The manifest is +// invalidated rather than recomputed; sync rebuilds it lazily. +function applyChunkedInodeUpdate( + db: Database, + inode: number, + size: number, + mode: number, + mtime: number, + isTouched: (idx: number, start: number, end: number) => boolean, + buildChunkBytes: (idx: number, start: number, end: number, existing: Uint8Array) => Uint8Array, +): void { + const oldChunks = existingChunkRefs(db, inode); + const chunkCount = Math.ceil(size / CHUNK_SIZE); + const oldChunkCount = oldChunks.length; + + for (let idx = 0; idx < chunkCount; idx++) { + const start = idx * CHUNK_SIZE; + const end = Math.min(start + CHUNK_SIZE, size); + const intendedSize = end - start; + const old = oldChunks[idx]; + const touched = isTouched(idx, start, end); + // Stable chunk: existed before with the same logical size and the + // caller did not flag it as touched. Skip without issuing SQL so + // its rowid stays put. + if (old !== undefined && old.size === intendedSize && !touched) continue; + + const existingBytes = old !== undefined ? readChunkBytes(db, inode, idx) : new Uint8Array(); + const chunkBytes = buildChunkBytes(idx, start, end, existingBytes); + if (chunkBytes.byteLength !== intendedSize) { + throw createWorkspaceError("EIO", "chunk builder returned wrong size"); + } + const chunk = { hash: sha256(chunkBytes), bytes: chunkBytes, size: chunkBytes.byteLength }; + upsertChunkBlob(db, chunk, mtime); + db.run( + "INSERT OR REPLACE INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", + inode, + idx, + chunk.hash, + chunk.size, + ); + } + + // Drop any old chunks past the new end of file (shrink case). + if (oldChunkCount > chunkCount) { + db.run("DELETE FROM vfs_chunks WHERE inode = ? AND idx >= ?", inode, chunkCount); + } + + const rev = incrementRev(db); + db.run( + "UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ?, size = ?, manifest_hash = NULL WHERE inode = ?", + mode, + mtime, + rev, + size, + inode, + ); +} + +export function createFileSync( + db: Database, + path: string, + options: WriteFileOptions, + now: () => number, +): void { + const { path: canonical } = canonicalizePath(path); + assertNotReadOnly(db, canonical); + const [parentInode, leafName] = parentAndNameForResolvedPath(db, path); + const mode = (options.mode ?? 0o644) & 0o7777; + const mtime = now(); + + db.transactionSync(() => { + const existing = db.one<{ child_inode: number }>( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + parentInode, + leafName, + ); + if (existing !== undefined) { + throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); + } + const rev = incrementRev(db); + // INSERT with RETURNING folds the last_insert_rowid lookup into + // the same statement, and computing rev up front lets us write + // the node row with its final stamp in one shot. + const row = db.one<{ inode: number }>( + "INSERT INTO vfs_nodes (type, mode, mtime, rev, manifest_hash) VALUES ('file', ?, ?, ?, NULL) RETURNING inode", + mode, + mtime, + rev, + ); + if (row === undefined) throw createWorkspaceError("EIO", "failed to allocate inode"); + insertFileDirent(db, parentInode, leafName, row.inode, canonical); + }); +} + +// Open a write buffer for an existing file. Subsequent writes, +// truncates, and reads against the same Database operate on the +// buffer instead of the SQLite chunk/blob store. Release commits +// the bytes back to chunks. +export function openWriteBufferSync(db: Database, path: string): void { + const { path: canonical } = canonicalizePath(path); + const pending = getPendingWriteBufferByPath(db, canonical); + if (pending !== undefined) { + pending.openCount += 1; + return; + } + const { inode, mode } = resolveFileInode(db, path); + const existing = getWriteBuffer(db, inode); + if (existing !== undefined) { + existing.openCount += 1; + return; + } + setWriteBuffer(db, inode, { + buf: new Uint8Array(0), + size: 0, + dirty: false, + openCount: 1, + mode, + }); +} + +// Create a new file lazily: stash a pending-create write buffer +// keyed by path, without touching SQL until release. createFileSync +// + openWriteBufferSync + writes + releaseWriteBufferSync would +// otherwise spend two transactions per file (one INSERT round and +// one chunk-commit round); this collapses them into a single +// INSERT-and-chunks transaction at release time. +// +// Throws EEXIST if a path already resolves to a live node or to +// another pending buffer. +export function openWriteBufferForCreateSync( + db: Database, + path: string, + options: WriteFileOptions, + now: () => number, +): void { + const { path: canonical } = canonicalizePath(path); + assertNotReadOnly(db, canonical); + if (getPendingWriteBufferByPath(db, canonical) !== undefined) { + throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); + } + const [parentInode, leafName] = parentAndNameForResolvedPath(db, path); + const existing = db.one<{ child_inode: number }>( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + parentInode, + leafName, + ); + if (existing !== undefined) { + throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); + } + const mode = (options.mode ?? 0o644) & 0o7777; + const mtime = now(); + const pendingInode = allocatePendingInode(db); + setWriteBuffer(db, pendingInode, { + buf: new Uint8Array(0), + size: 0, + dirty: true, + openCount: 1, + mode, + pending: { parentInode, leafName, canonicalPath: canonical, pendingInode, mtime }, + }); +} + +// Release one open of an inode's write buffer. When the open count +// reaches zero, commit the buffered bytes to chunk rows and drop +// the entry. The committed mode is the buffer's mode at release +// time so an intermediate chmod survives. Pending-create entries +// emit their INSERT + dirent + chunks in the same transaction. +export function releaseWriteBufferSync(db: Database, path: string, now: () => number): void { + const { path: canonical } = canonicalizePath(path); + const pending = getPendingWriteBufferByPath(db, canonical); + if (pending !== undefined) { + releasePendingBuffer(db, pending, now); + return; + } + const node = resolveFileInode(db, path); + const entry = getWriteBuffer(db, node.inode); + if (entry === undefined) return; + entry.openCount -= 1; + if (entry.openCount > 0) return; + + if (!entry.dirty) { + deleteWriteBuffer(db, node.inode); + return; + } + + const mtime = now(); + const mode = entry.mode & 0o7777; + const buffered = entry.buf.subarray(0, entry.size); + + db.transactionSync(() => { + if (entry.size === 0) { + // An empty file owns no chunk rows; clear any old ones the + // buffer would otherwise have replaced and bump metadata. + db.run("DELETE FROM vfs_chunks WHERE inode = ?", node.inode); + const rev = incrementRev(db); + db.run( + "UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ?, size = 0, manifest_hash = NULL WHERE inode = ?", + mode, + mtime, + rev, + node.inode, + ); + return; + } + applyChunkedInodeUpdate( + db, + node.inode, + entry.size, + mode, + mtime, + (_idx, start, end) => start < entry.size && end > 0, + (_idx, start, end) => buffered.subarray(start, Math.min(end, entry.size)), + ); + }); + + deleteWriteBuffer(db, node.inode); +} + +// Commit a pending-create buffer to SQLite. Returns the real inode +// allocated by the INSERT, or throws. Promotes the cache entry's key +// from the synthetic pending id to the real inode so subsequent +// reads/writes through the inode-keyed cache still see the same +// buffer. Caller owns the lifecycle of the now-promoted entry. +function commitPendingBuffer(db: Database, entry: WriteBufferEntry, now: () => number): number { + if (entry.pending === undefined) { + throw createWorkspaceError("EIO", "commitPendingBuffer called on non-pending entry"); + } + const { parentInode, leafName, canonicalPath, pendingInode } = entry.pending; + const mtime = now(); + const mode = entry.mode & 0o7777; + const buffered = entry.buf.subarray(0, entry.size); + + let realInode = 0; + try { + db.transactionSync(() => { + // Re-check at commit time: a non-buffered writeFile or another + // out-of-band path could have landed between open and release. + const collision = db.one<{ child_inode: number }>( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + parentInode, + leafName, + ); + if (collision !== undefined) { + throw createWorkspaceError( + "EEXIST", + `path exists at commit time: ${canonicalPath}`, + canonicalPath, + ); + } + const rev = incrementRev(db); + const row = db.one<{ inode: number }>( + "INSERT INTO vfs_nodes (type, mode, mtime, rev, size, manifest_hash) VALUES ('file', ?, ?, ?, ?, NULL) RETURNING inode", + mode, + mtime, + rev, + entry.size, + ); + if (row === undefined) { + throw createWorkspaceError("EIO", "failed to allocate inode"); + } + insertFileDirent(db, parentInode, leafName, row.inode, canonicalPath); + if (entry.size > 0) { + const inode = row.inode; + const chunkCount = Math.ceil(entry.size / CHUNK_SIZE); + for (let idx = 0; idx < chunkCount; idx++) { + const start = idx * CHUNK_SIZE; + const end = Math.min(start + CHUNK_SIZE, entry.size); + const chunkBytes = buffered.subarray(start, end); + const chunk = { + hash: sha256(chunkBytes), + bytes: chunkBytes, + size: chunkBytes.byteLength, + }; + upsertChunkBlob(db, chunk, mtime); + db.run( + "INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", + inode, + idx, + chunk.hash, + chunk.size, + ); + } + } + realInode = row.inode; + }); + } catch (error) { + // Transaction rolled back; drop the buffer so the next caller + // starts clean. + deleteWriteBuffer(db, pendingInode); + throw error; + } + promotePendingToInode(db, pendingInode, realInode); + return realInode; +} + +/** + * @internal + * Bridges a pending-create write buffer into the SQL world ahead of a + * dirent-mutating provider operation (link, rename, unlink). Leaves + * the open count untouched so a still-open handle keeps writing into + * the now-promoted buffer. Returns true when a pending buffer was + * committed. External callers should never invoke this directly. + */ +export function flushPendingByPath(db: Database, path: string, now: () => number): boolean { + const { path: canonical } = canonicalizePath(path); + const entry = getPendingWriteBufferByPath(db, canonical); + if (entry === undefined || entry.pending === undefined) return false; + commitPendingBuffer(db, entry, now); + return true; +} + +function releasePendingBuffer(db: Database, entry: WriteBufferEntry, now: () => number): void { + if (entry.pending === undefined) return; + entry.openCount -= 1; + if (entry.openCount > 0) return; + + const inode = commitPendingBuffer(db, entry, now); + // File is closed; drop the now-promoted entry. A subsequent open + // hits the SQL path and gets a fresh buffer if needed. + deleteWriteBuffer(db, inode); +} + +// Hydrate a freshly-opened buffer with the inode's current bytes +// the first time we mutate it. Avoids paying the read cost when the +// caller opens a file just to truncate or overwrite it. +function hydrateBufferIfNeeded(db: Database, inode: number, entry: WriteBufferEntry): void { + if (entry.dirty) return; + const existingSize = fileSizeForInode(db, inode); + if (existingSize === 0) { + entry.dirty = true; + return; + } + ensureBufferCapacity(entry, existingSize); + let copied = 0; + for (let idx = 0; copied < existingSize; idx++) { + const chunk = readChunkBytes(db, inode, idx); + if (chunk.byteLength === 0) break; + entry.buf.set(chunk, copied); + copied += chunk.byteLength; + } + entry.size = existingSize; + entry.dirty = true; +} + +export function writeRangeSync( + db: Database, + path: string, + bytes: Uint8Array, + offset: number, + options: WriteFileOptions, + now: () => number, +): number { + const { path: canonical } = canonicalizePath(path); + assertNotReadOnly(db, canonical); + if (!Number.isInteger(offset) || offset < 0) { + throw createWorkspaceError("EINVAL", `invalid write offset: ${offset}`, canonical); + } + if (bytes.byteLength === 0) return 0; + const mtime = now(); + + // Pending-create files don't have an inode yet; route the write + // straight into the path-keyed buffer. + const pending = getPendingWriteBufferByPath(db, canonical); + if (pending !== undefined) { + const writeEnd = offset + bytes.byteLength; + ensureBufferCapacity(pending, writeEnd); + if (offset > pending.size) { + pending.buf.fill(0, pending.size, offset); + } + pending.buf.set(bytes, offset); + if (writeEnd > pending.size) pending.size = writeEnd; + pending.mode = (options.mode ?? pending.mode) & 0o7777; + pending.dirty = true; + return bytes.byteLength; + } + + const { inode, mode: existingMode } = resolveFileInode(db, path); + const mode = (options.mode ?? existingMode) & 0o7777; + const buffered = getWriteBuffer(db, inode); + + // Buffered path: mutate the in-memory bytes and defer storage + // writes until release. Reads through the same Database see the + // buffer's current bytes via readRangeSync's buffer check. + if (buffered !== undefined) { + hydrateBufferIfNeeded(db, inode, buffered); + const writeEnd = offset + bytes.byteLength; + ensureBufferCapacity(buffered, writeEnd); + if (offset > buffered.size) { + buffered.buf.fill(0, buffered.size, offset); + } + buffered.buf.set(bytes, offset); + if (writeEnd > buffered.size) buffered.size = writeEnd; + buffered.mode = mode; + buffered.dirty = true; + return bytes.byteLength; + } + + db.transactionSync(() => { + const oldSize = fileSizeForInode(db, inode); + const writeEnd = offset + bytes.byteLength; + const nextSize = Math.max(oldSize, writeEnd); + + applyChunkedInodeUpdate( + db, + inode, + nextSize, + mode, + mtime, + (_idx, start, end) => offset < end && start < writeEnd, + (_idx, start, end, existing) => { + const chunkBytes = new Uint8Array(end - start); + chunkBytes.set(existing.subarray(0, Math.min(existing.byteLength, chunkBytes.byteLength))); + if (offset < end && start < writeEnd) { + const copyStart = Math.max(start, offset); + const copyEnd = Math.min(end, writeEnd); + chunkBytes.set(bytes.subarray(copyStart - offset, copyEnd - offset), copyStart - start); + } + return chunkBytes; + }, + ); + }); + + return bytes.byteLength; +} + +export function truncateFileSync( + db: Database, + path: string, + size: number, + now: () => number, +): void { + const { path: canonical } = canonicalizePath(path); + assertNotReadOnly(db, canonical); + if (!Number.isInteger(size) || size < 0) { + throw createWorkspaceError("EINVAL", `invalid truncate size: ${size}`, canonical); + } + const mtime = now(); + + // Pending-create files truncate in-place on the path-keyed buffer. + const pending = getPendingWriteBufferByPath(db, canonical); + if (pending !== undefined) { + if (size > pending.size) { + ensureBufferCapacity(pending, size); + pending.buf.fill(0, pending.size, size); + } + pending.size = size; + pending.dirty = true; + return; + } + + const { inode, mode } = resolveFileInode(db, path); + const buffered = getWriteBuffer(db, inode); + + if (buffered !== undefined) { + hydrateBufferIfNeeded(db, inode, buffered); + if (size > buffered.size) { + ensureBufferCapacity(buffered, size); + buffered.buf.fill(0, buffered.size, size); + } + buffered.size = size; + buffered.dirty = true; + return; + } + + db.transactionSync(() => { + const oldSize = fileSizeForInode(db, inode); + if (oldSize === size) return; + + if (size === 0) { + db.run("DELETE FROM vfs_chunks WHERE inode = ?", inode); + const rev = incrementRev(db); + db.run( + "UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ?, size = 0, manifest_hash = NULL WHERE inode = ?", + mode, + mtime, + rev, + inode, + ); + return; + } + + applyChunkedInodeUpdate( + db, + inode, + size, + mode, + mtime, + () => false, + (_idx, start, end, existing) => { + const chunkBytes = new Uint8Array(end - start); + chunkBytes.set(existing.subarray(0, Math.min(existing.byteLength, chunkBytes.byteLength))); + return chunkBytes; + }, + ); + }); +} + +// Synchronous entry point used by the VirtualProvider. Identical SQL +// to the async path; differs only in that the bytes have already been +// materialized. +export function writeFileSync( + db: Database, + path: string, + bytes: Uint8Array, + options: WriteFileOptions, + now: () => number, +): void { + const { parts, path: canonical } = canonicalizePath(path); + if (parts.length === 0) { + throw createWorkspaceError("EISDIR", "cannot write to the root directory", canonical); + } + assertNotReadOnly(db, canonical); + const mode = (options.mode ?? 0o644) & 0o7777; + const mtime = now(); + + db.transactionSync(() => { + const parentInode = resolveParent(db, parts, canonical); + const leafName = parts[parts.length - 1]; + const existing = db.one<{ child_inode: number }>( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + parentInode, + leafName, + ); + + let inode: number; + if (existing !== undefined) { + if (options.exclusive) { + throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); + } + const node = db.one<{ type: "file" | "dir" }>( + "SELECT type FROM vfs_nodes WHERE inode = ?", + existing.child_inode, + ); + if (node?.type === "dir") { + throw createWorkspaceError("EISDIR", `path is a directory: ${canonical}`, canonical); + } + inode = existing.child_inode; + // Replace the existing representation. Orphaned blobs (if any) + // are cleaned up by a later gc() pass. + db.run("DELETE FROM vfs_chunks WHERE inode = ?", inode); + } else { + inode = insertFileNode(db, mode, mtime); + insertFileDirent(db, parentInode, leafName, inode, canonical); + } + + const rev = incrementRev(db); + const chunks = chunksOf(bytes); + // Upsert blobs and write the new chunk list. + for (let idx = 0; idx < chunks.length; idx++) { + const chunk = chunks[idx]; + upsertChunkBlob(db, chunk, mtime); + db.run( + "INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", + inode, + idx, + chunk.hash, + chunk.size, + ); + } + + const manifestHash = buildManifest(db, chunks, mtime); + db.run( + "UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ?, size = ?, manifest_hash = ? WHERE inode = ?", + mode, + mtime, + rev, + bytes.byteLength, + manifestHash, + inode, + ); + }); +} + +export function writeFileRangesSync( + db: Database, + path: string, + bytes: Uint8Array, + dirtyRanges: WriteFileRange[], + options: WriteFileOptions, + now: () => number, +): void { + const { parts, path: canonical } = canonicalizePath(path); + if (parts.length === 0) { + throw createWorkspaceError("EISDIR", "cannot write to the root directory", canonical); + } + assertNotReadOnly(db, canonical); + const mode = (options.mode ?? 0o644) & 0o7777; + const ranges = normalizeRanges(dirtyRanges, bytes.byteLength); + const mtime = now(); + db.transactionSync(() => { + const parentInode = resolveParent(db, parts, canonical); + const leafName = parts[parts.length - 1]; + const existing = db.one<{ child_inode: number }>( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + parentInode, + leafName, + ); + + let inode: number; + let oldChunks: ChunkRef[] = []; + if (existing !== undefined) { + const node = db.one<{ type: "file" | "dir" }>( + "SELECT type FROM vfs_nodes WHERE inode = ?", + existing.child_inode, + ); + if (node?.type === "dir") { + throw createWorkspaceError("EISDIR", `path is a directory: ${canonical}`, canonical); + } + inode = existing.child_inode; + oldChunks = existingChunkRefs(db, inode); + } else { + inode = insertFileNode(db, mode, mtime); + insertFileDirent(db, parentInode, leafName, inode, canonical); + } + + const rev = incrementRev(db); + const nextChunks: ChunkRef[] = []; + const chunkCount = Math.ceil(bytes.byteLength / CHUNK_SIZE); + for (let idx = 0; idx < chunkCount; idx++) { + const start = idx * CHUNK_SIZE; + const end = Math.min(start + CHUNK_SIZE, bytes.byteLength); + const size = end - start; + const oldChunk = oldChunks[idx]; + if (oldChunk !== undefined && oldChunk.size === size && !rangesOverlap(start, end, ranges)) { + nextChunks.push(oldChunk); + continue; + } + const chunk = { + hash: sha256(bytes.subarray(start, end)), + bytes: bytes.subarray(start, end), + size, + }; + upsertChunkBlob(db, chunk, mtime); + nextChunks.push({ hash: chunk.hash, size: chunk.size }); + } + + const manifestHash = replaceChunkRows(db, inode, nextChunks, mtime); + db.run( + "UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ?, size = ?, manifest_hash = ? WHERE inode = ?", + mode, + mtime, + rev, + bytes.byteLength, + manifestHash, + inode, + ); + }); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/path.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/path.ts new file mode 100644 index 00000000..777a39a0 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/path.ts @@ -0,0 +1,52 @@ +import { invalidPath } from "./errors.js"; + +export interface CanonicalPath { + path: string; + parts: string[]; + name: string; + parentPath: string | undefined; +} + +export function canonicalizePath(path: string): CanonicalPath { + if (path.length === 0) { + throw invalidPath(path, "empty"); + } + + if (!path.startsWith("/")) { + throw invalidPath(path, "must be absolute"); + } + + if (path.includes("\0")) { + throw invalidPath(path, "contains NUL byte"); + } + + const parts: string[] = []; + for (const part of path.split("/")) { + if (part === "" || part === ".") { + continue; + } + + if (part === "..") { + if (parts.length === 0) { + throw invalidPath(path, "escapes root"); + } + parts.pop(); + continue; + } + + parts.push(part); + } + + const canonical = parts.length === 0 ? "/" : `/${parts.join("/")}`; + const name = parts.length === 0 ? "" : parts[parts.length - 1]; + const parentParts = parts.slice(0, -1); + const parentPath = + parts.length === 0 ? undefined : parentParts.length === 0 ? "/" : `/${parentParts.join("/")}`; + + return { + path: canonical, + parts, + name, + parentPath, + }; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/rev.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/rev.ts new file mode 100644 index 00000000..df7f6d3c --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/rev.ts @@ -0,0 +1,21 @@ +import type { Database } from "./storage.js"; + +// Atomic monotonic rev counter. Every FS mutation (mkdir, writeFile, +// rm, ...) calls incrementRev once per transaction and stamps the returned +// value into vfs_nodes.rev. The sync layer reads vfs_meta.rev as +// currentRev and consumes vfs_changes.rev for tombstones. +// +// Must be called inside a transactionSync — the UPDATE and SELECT +// otherwise race with concurrent mutations. The DO single-writer model +// makes that unlikely in practice, but the contract is "wrap me". +export function incrementRev(db: Database): number { + // RETURNING folds the read into the same statement so each mutation + // pays one round-trip instead of two. SQLite has supported it since + // 3.35; both node:sqlite and Cloudflare DO SqlStorage are on newer + // versions. + const row = db.one<{ v: number }>("UPDATE vfs_meta SET v = v + 1 WHERE k = 'rev' RETURNING v"); + if (row === undefined) { + throw new Error("vfs_meta.rev row missing; was initializeSchema run?"); + } + return row.v; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/schema/core.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/schema/core.ts new file mode 100644 index 00000000..bc77dd62 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/schema/core.ts @@ -0,0 +1,81 @@ +// Filesystem-side tables. These hold the inode graph and the +// content-addressed blob store. See docs/03_filesystem_schema.md. + +// Bumped to 2 when `_vfs_mounts.mode` landed (read-only mount +// enforcement at the data layer). Bumped to 3 when `vfs_nodes` +// gained a cached `size` column so stat() doesn't have to SUM +// chunks on every call. Bumped to 4 when `_vfs_watermark` gained +// a `backend` column so a single workspace can host more than +// one backend with independent sync cursors. Bumped to 5 when +// `vfs_dirents` and `vfs_chunks` became WITHOUT ROWID: their +// composite-PK lookups now read straight from the PK b-tree leaf +// with no rowid indirection, and `child_inode` lives in the +// dirents leaf so the (parent, name) resolve read is covering +// (no separate index needed). See `schema/migrations.ts` for the +// migration list; `sync.ts` carries the fresh-install DDL. +export const SCHEMA_VERSION = 5; +export const ROOT_INODE = 1; + +export const CORE_STATEMENTS = [ + `CREATE TABLE IF NOT EXISTS vfs_meta ( + k TEXT PRIMARY KEY, + v INTEGER NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS vfs_nodes ( + inode INTEGER PRIMARY KEY AUTOINCREMENT, + type TEXT NOT NULL CHECK(type IN ('file','dir','symlink')), + mode INTEGER NOT NULL DEFAULT 493, + mtime INTEGER NOT NULL, + rev INTEGER NOT NULL DEFAULT 0, + mount_root TEXT, + stub_size INTEGER, + manifest_hash BLOB, + link_target TEXT, + size INTEGER NOT NULL DEFAULT 0 + )`, + // WITHOUT ROWID: the row lives in the (parent_inode, name) PK + // b-tree leaf, so resolving a path segment reads child_inode + // directly from the leaf — no autoindex -> rowid hop, and no + // separate covering index. Legal here because the PK is composite + // and the table has no AUTOINCREMENT. Existing databases are + // rebuilt by the v4 -> v5 migration in schema/migrations.ts; keep + // this DDL and that migrator's CREATE in lockstep. + `CREATE TABLE IF NOT EXISTS vfs_dirents ( + parent_inode INTEGER NOT NULL, + name TEXT NOT NULL, + child_inode INTEGER NOT NULL, + PRIMARY KEY (parent_inode, name) + ) WITHOUT ROWID`, + `CREATE INDEX IF NOT EXISTS vfs_dirents_by_child ON vfs_dirents(child_inode)`, + `CREATE INDEX IF NOT EXISTS vfs_nodes_by_rev ON vfs_nodes(rev)`, + // gc/manifests checks every manifest row against vfs_nodes via a + // correlated NOT EXISTS (manifest_hash = ?). Without this index + // gc full-scans vfs_nodes per candidate manifest — O(N×M). + // Partial because the column is null on every dir and symlink + // node, and on files until they get their first content write. + `CREATE INDEX IF NOT EXISTS vfs_nodes_by_manifest_hash + ON vfs_nodes(manifest_hash) WHERE manifest_hash IS NOT NULL`, + `CREATE TABLE IF NOT EXISTS vfs_blobs ( + hash BLOB PRIMARY KEY, + size INTEGER NOT NULL, + last_seen INTEGER NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS vfs_blob_bytes ( + hash BLOB PRIMARY KEY REFERENCES vfs_blobs(hash) ON DELETE CASCADE, + bytes BLOB NOT NULL + )`, + // WITHOUT ROWID: clustered on (inode, idx) so a file's chunks are + // stored and scanned in index order straight from the PK leaf. + // Legal here — composite PK, no AUTOINCREMENT. The bytes live in + // vfs_blob_bytes (content-addressed), so these rows stay small, + // which is what WITHOUT ROWID wants. Rebuilt for existing DBs by + // the v4 -> v5 migration; keep in lockstep with that migrator. + `CREATE TABLE IF NOT EXISTS vfs_chunks ( + inode INTEGER NOT NULL, + idx INTEGER NOT NULL, + hash BLOB NOT NULL, + size INTEGER NOT NULL, + PRIMARY KEY (inode, idx) + ) WITHOUT ROWID`, + `CREATE INDEX IF NOT EXISTS vfs_chunks_by_hash ON vfs_chunks(hash)`, +] as const; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/schema/index.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/schema/index.ts new file mode 100644 index 00000000..e0147e96 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/schema/index.ts @@ -0,0 +1,82 @@ +import { createWorkspaceError } from "../errors.js"; +import type { Database } from "../storage.js"; +import { CORE_STATEMENTS, ROOT_INODE, SCHEMA_VERSION } from "./core.js"; +import { runMigrations } from "./migrations.js"; +import { SYNC_STATEMENTS } from "./sync.js"; + +export { ROOT_INODE, SCHEMA_VERSION } from "./core.js"; + +interface MetaRow { + v: number; +} + +export function initializeSchema(db: Database, now: () => number): void { + db.transactionSync(() => { + // 1. Baseline DDL. Every statement is "CREATE TABLE IF NOT + // EXISTS" / "CREATE INDEX IF NOT EXISTS" so this is a no-op + // on already-initialized databases. Fresh databases come out + // of this step at the latest column shape (SCHEMA_VERSION). + for (const statement of CORE_STATEMENTS) { + db.run(statement); + } + for (const statement of SYNC_STATEMENTS) { + db.run(statement); + } + + // 2. Read the on-disk schema version. Absent → 0 (very first + // boot of this database). The baseline above just created + // every table at the latest shape, so a 0 → SCHEMA_VERSION + // jump has nothing to migrate. + const storedVersion = db.one( + "SELECT v FROM vfs_meta WHERE k = ?", + "schema_version", + )?.v; + const onDiskVersion = storedVersion ?? 0; + + if (onDiskVersion > SCHEMA_VERSION) { + throw createWorkspaceError( + "EIO", + `Unsupported workspace filesystem schema version ${onDiskVersion}`, + ); + } + + // 3. Migrate. Skip when the database is fresh (0) — the + // baseline DDL already shipped the latest shape. Otherwise + // dispatch each registered migrator until we hit the + // target. + if (onDiskVersion > 0 && onDiskVersion < SCHEMA_VERSION) { + runMigrations(db, onDiskVersion, SCHEMA_VERSION); + } + + // 4. Stamp the version and seed the boot rows. Both shapes + // (insert-if-missing, then update) keep this idempotent so + // repeat calls do nothing. + db.run("INSERT OR IGNORE INTO vfs_meta (k, v) VALUES (?, ?)", "schema_version", SCHEMA_VERSION); + db.run("UPDATE vfs_meta SET v = ? WHERE k = ?", SCHEMA_VERSION, "schema_version"); + db.run("INSERT OR IGNORE INTO vfs_meta (k, v) VALUES (?, ?)", "rev", 1); + db.run( + "INSERT OR IGNORE INTO _vfs_watermark (k, backend, v) VALUES (?, 'default', ?)", + "pushRev", + 0, + ); + db.run( + "INSERT OR IGNORE INTO _vfs_watermark (k, backend, v) VALUES (?, 'default', ?)", + "fetchRev", + 0, + ); + db.run( + "INSERT OR IGNORE INTO _vfs_fetch_cursor (k, backend, path) VALUES (?, 'default', ?)", + "fetch", + null, + ); + + db.run( + `INSERT OR IGNORE INTO vfs_nodes + (inode, type, mode, mtime, rev) + VALUES (?, 'dir', ?, ?, 0)`, + ROOT_INODE, + 0o755, + now(), + ); + }); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/schema/migrations.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/schema/migrations.ts new file mode 100644 index 00000000..d2425ada --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/schema/migrations.ts @@ -0,0 +1,169 @@ +// Schema migration runner. +// +// The schema's "CREATE TABLE IF NOT EXISTS" baseline handles fresh +// databases. When a schema column changes shape — added, dropped, +// renamed, retyped — IF NOT EXISTS does nothing and the older +// rows stay incompatible. Migrations close that gap. +// +// Shape: an ordered list of `(from, to, migrator)` tuples. The +// runner reads `vfs_meta.schema_version` (defaulting to 0 when the +// row is absent), picks every migration whose `from === current`, +// runs it, advances `current`, and repeats until `current >= +// SCHEMA_VERSION`. The whole pass runs inside the caller's +// transactionSync so a partial migration rolls back. +// +// Each migrator is a `(db: Database) => void` and may assume the +// previous version's schema is in place. Migrators land schema +// changes only; they don't touch user data unless the column shape +// requires it. + +import type { Database } from "../storage.js"; + +export interface Migration { + readonly from: number; + readonly to: number; + readonly migrator: (db: Database) => void; +} + +// v1 → v2 — add `_vfs_mounts.mode` so dofs can enforce read-only +// mounts at the data layer. Existing rows default to 'read-only'; +// the workspace re-stamps them with the registered mount's mode on +// the next index pass. +// +// The CHECK constraint is duplicated in `sync.ts`'s fresh-install +// DDL; both paths must keep the same allowed set. +function v1_to_v2_add_mounts_mode(db: Database): void { + db.run( + `ALTER TABLE _vfs_mounts + ADD COLUMN mode TEXT NOT NULL DEFAULT 'read-only' + CHECK(mode IN ('read-only', 'read-write'))`, + ); +} + +// v2 → v3 — denormalise file size onto vfs_nodes so stat doesn't +// have to SUM the chunk rows on every call. The column is +// backfilled from existing vfs_chunks; later writes maintain it. +function v2_to_v3_add_size_column(db: Database): void { + const hasColumn = db + .all<{ name: string }>("PRAGMA table_info(vfs_nodes)") + .some((column) => column.name === "size"); + if (!hasColumn) { + db.run("ALTER TABLE vfs_nodes ADD COLUMN size INTEGER NOT NULL DEFAULT 0"); + } + db.run( + `UPDATE vfs_nodes + SET size = COALESCE( + (SELECT SUM(size) FROM vfs_chunks WHERE vfs_chunks.inode = vfs_nodes.inode), + 0 + ) + WHERE type = 'file'`, + ); +} + +// v3 → v4 — add a `backend` column to `_vfs_watermark` so a +// workspace can host more than one backend with independent sync +// cursors. SQLite's ALTER TABLE can't change a primary key; copy +// existing rows into a fresh table with the composite +// (k, backend) primary key, then swap the tables. +// +// Existing rows land under the `default` backend id, which the +// dofs sync helpers also use as the fallback when a caller +// doesn't pass an id. Pre-multi-backend workspaces keep their +// pushRev / fetchRev cursors intact through the upgrade. +function v3_to_v4_watermark_backend_column(db: Database): void { + db.run(`ALTER TABLE _vfs_watermark RENAME TO _vfs_watermark_v3`); + db.run( + `CREATE TABLE _vfs_watermark ( + k TEXT NOT NULL, + backend TEXT NOT NULL DEFAULT 'default', + v INTEGER NOT NULL, + PRIMARY KEY (k, backend) + )`, + ); + db.run( + `INSERT INTO _vfs_watermark (k, backend, v) + SELECT k, 'default', v FROM _vfs_watermark_v3`, + ); + db.run(`DROP TABLE _vfs_watermark_v3`); +} + +// v4 → v5 — rebuild `vfs_dirents` and `vfs_chunks` as WITHOUT ROWID. +// SQLite can't convert a table to WITHOUT ROWID in place, so for each +// table: rename it aside, create the WITHOUT ROWID replacement, copy +// the rows, drop the old table. +// +// Both targets are FK-inert (neither is an FK parent or child; the +// schema's only foreign key is vfs_blob_bytes -> vfs_blobs) and have +// composite primary keys with no AUTOINCREMENT, so WITHOUT ROWID is +// legal and sqlite_sequence is untouched. `vfs_blob_bytes` is left +// alone on purpose — it holds the large blob payloads and the FK. +// +// A RENAME carries the table's secondary index along to the temp +// name, and the following DROP takes the index with it. The baseline +// `CREATE INDEX IF NOT EXISTS` in initializeSchema already ran, before +// migrations, and does not re-run — so this migrator must recreate +// vfs_dirents_by_child and vfs_chunks_by_hash itself, or upgraded +// databases silently lose them. Keep the CREATE bodies in lockstep +// with the fresh-install DDL in core.ts. +function v4_to_v5_without_rowid(db: Database): void { + // vfs_dirents + db.run(`ALTER TABLE vfs_dirents RENAME TO vfs_dirents_v4`); + db.run( + `CREATE TABLE vfs_dirents ( + parent_inode INTEGER NOT NULL, + name TEXT NOT NULL, + child_inode INTEGER NOT NULL, + PRIMARY KEY (parent_inode, name) + ) WITHOUT ROWID`, + ); + db.run( + `INSERT INTO vfs_dirents (parent_inode, name, child_inode) + SELECT parent_inode, name, child_inode FROM vfs_dirents_v4`, + ); + db.run(`DROP TABLE vfs_dirents_v4`); + db.run(`CREATE INDEX vfs_dirents_by_child ON vfs_dirents(child_inode)`); + + // vfs_chunks + db.run(`ALTER TABLE vfs_chunks RENAME TO vfs_chunks_v4`); + db.run( + `CREATE TABLE vfs_chunks ( + inode INTEGER NOT NULL, + idx INTEGER NOT NULL, + hash BLOB NOT NULL, + size INTEGER NOT NULL, + PRIMARY KEY (inode, idx) + ) WITHOUT ROWID`, + ); + db.run( + `INSERT INTO vfs_chunks (inode, idx, hash, size) + SELECT inode, idx, hash, size FROM vfs_chunks_v4`, + ); + db.run(`DROP TABLE vfs_chunks_v4`); + db.run(`CREATE INDEX vfs_chunks_by_hash ON vfs_chunks(hash)`); +} + +export const MIGRATIONS: readonly Migration[] = [ + { from: 1, to: 2, migrator: v1_to_v2_add_mounts_mode }, + { from: 2, to: 3, migrator: v2_to_v3_add_size_column }, + { from: 3, to: 4, migrator: v3_to_v4_watermark_backend_column }, + { from: 4, to: 5, migrator: v4_to_v5_without_rowid }, +] as const; + +// Apply every migration whose `from` matches the current version, +// in order, until we reach the target. The caller has already +// wrapped this in a transactionSync; failures here roll the whole +// initializeSchema call back. +export function runMigrations(db: Database, current: number, target: number): number { + let version = current; + while (version < target) { + const next = MIGRATIONS.find((m) => m.from === version); + if (next === undefined) { + // No migration registered for this jump. This is a bug — the + // version was bumped without a matching migration. + throw new Error(`dofs schema: no migration registered for v${version} -> v${target}`); + } + next.migrator(db); + version = next.to; + } + return version; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/schema/sync.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/schema/sync.ts new file mode 100644 index 00000000..fdbac101 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/schema/sync.ts @@ -0,0 +1,59 @@ +// Sync-protocol tables. Populated by the sync module; the FS module +// only writes to vfs_changes (via sync/changes.ts) on rm. The rest +// of these tables stay empty until the sync task is implemented. + +export const SYNC_STATEMENTS = [ + `CREATE TABLE IF NOT EXISTS vfs_manifests ( + hash BLOB PRIMARY KEY, + size INTEGER NOT NULL, + encoded BLOB NOT NULL, + last_seen INTEGER NOT NULL DEFAULT 0 + )`, + `CREATE TABLE IF NOT EXISTS vfs_changes ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + rev INTEGER NOT NULL, + path TEXT NOT NULL, + op TEXT NOT NULL CHECK(op IN ('delete')) + )`, + `CREATE INDEX IF NOT EXISTS vfs_changes_by_rev ON vfs_changes(rev)`, + // changes.ts looks up the latest op for a path via + // `WHERE path = ? ORDER BY id DESC LIMIT 1`. Without the index + // SQLite falls back to a full scan; with (path, id DESC) the + // lookup is O(log n) and the ORDER BY drains straight from the + // index. Used on every recordDelete and on every push-tick that + // processes tombstones. + `CREATE INDEX IF NOT EXISTS vfs_changes_by_path ON vfs_changes(path, id DESC)`, + // Watermarks are keyed by (k, backend) so a workspace hosting + // multiple backends keeps each backend's sync cursors + // independent. The `backend` column was added at schema v3; + // `schema/migrations.ts` owns the ALTER for existing + // databases. Fresh installs land the composite key directly. + `CREATE TABLE IF NOT EXISTS _vfs_watermark ( + k TEXT NOT NULL, + backend TEXT NOT NULL DEFAULT 'default', + v INTEGER NOT NULL, + PRIMARY KEY (k, backend) + )`, + // The fetch cursor's same-rev `path` component, keyed by + // (k, backend) so each backend resumes a partially-drained rev + // independently. The rev component lives in _vfs_watermark under + // 'fetchRev'; this table only holds the in-rev path. `backend` + // mirrors _vfs_watermark and defaults to 'default'. + `CREATE TABLE IF NOT EXISTS _vfs_fetch_cursor ( + k TEXT NOT NULL CHECK(k = 'fetch'), + backend TEXT NOT NULL DEFAULT 'default', + path TEXT, + PRIMARY KEY (k, backend) + )`, + // The `mode` column was added at schema v2; `schema/migrations.ts` + // owns the ALTER for existing databases. Keep the CHECK + // constraint here aligned with the migration's CHECK so fresh + // installs and upgrades enforce the same allowed set. + `CREATE TABLE IF NOT EXISTS _vfs_mounts ( + root TEXT PRIMARY KEY, + kind TEXT NOT NULL, + indexed INTEGER NOT NULL DEFAULT 0, + mode TEXT NOT NULL DEFAULT 'read-only' + CHECK(mode IN ('read-only', 'read-write')) + )`, +] as const; diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/storage.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/storage.ts new file mode 100644 index 00000000..becce5a7 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/storage.ts @@ -0,0 +1,115 @@ +import type { DurableObjectStorageLike, SQLStorageLike } from "./types.js"; + +export class Database { + readonly sql: SQLStorageLike; + readonly transactionSync: (closure: () => T) => T; + // Depth counter so reentrant transactionSync() calls work. The + // outer call uses the storage adapter's transactionSync (or + // BEGIN/COMMIT under the hood); nested calls use SAVEPOINTs + // through sql.exec directly. SQLite forbids a real BEGIN inside + // an active transaction. + #txDepth = 0; + + constructor(storage: DurableObjectStorageLike) { + this.sql = storage.sql; + this.transactionSync = (closure: () => T): T => { + if (this.#txDepth > 0) { + // Reentrant call: use a savepoint. SQLite's RELEASE on a + // savepoint inside an outer transaction commits the inner + // work without ending the outer one. + const sp = `_t${this.#txDepth}`; + this.sql.exec(`SAVEPOINT ${sp}`); + this.#txDepth++; + try { + const result = closure(); + this.sql.exec(`RELEASE ${sp}`); + return result; + } catch (error) { + this.sql.exec(`ROLLBACK TO ${sp}`); + this.sql.exec(`RELEASE ${sp}`); + throw error; + } finally { + this.#txDepth--; + } + } + // Outer call: hand off to the storage adapter so the DO + // runtime's transaction semantics apply. + this.#txDepth++; + try { + if (storage.transactionSync !== undefined) { + return storage.transactionSync(closure); + } + if (storage.transaction !== undefined) { + const result = storage.transaction(closure); + if ( + result !== undefined && + result !== null && + typeof result === "object" && + "then" in result + ) { + throw new Error("Durable Object storage adapter requires synchronous transactions"); + } + return result; + } + return closure(); + } finally { + this.#txDepth--; + } + }; + } + + // True while a transactionSync closure is on the stack. The resolve + // cache uses this to refuse populating entries mid-transaction, so a + // rolled-back mutation can never leave the cache reflecting + // uncommitted state. (Invalidation still runs freely inside a + // transaction — dropping an entry is always safe.) + // + // Invariant: #txDepth only tracks transactionSync. A raw + // BEGIN/SAVEPOINT issued through run() would open a transaction this + // flag can't see, letting the cache populate mid-transaction and + // survive a rollback — so transactionSync is the only sanctioned way + // to open one. + get inTransaction(): boolean { + return this.#txDepth > 0; + } + + run(query: string, ...bindings: unknown[]): void { + this.sql.exec(query, ...bindings); + } + + all(query: string, ...bindings: unknown[]): Row[] { + const rows = this.sql.exec(query, ...bindings).toArray(); + return rows.map((row) => normalizeRow(row as Record)) as Row[]; + } + + one(query: string, ...bindings: unknown[]): Row | undefined { + return this.all(query, ...bindings)[0]; + } + + scalar(query: string, ...bindings: unknown[]): T | undefined { + const row = this.one>(query, ...bindings); + if (row === undefined) { + return undefined; + } + + const [value] = Object.values(row); + return value; + } +} + +// Cloudflare's DO SqlStorage returns BLOB columns as ArrayBuffer, +// whereas node:sqlite returns Uint8Array. Normalise to Uint8Array so +// the rest of the code only has to handle one shape. +function normalizeRow(row: Record): Record { + // node:sqlite hands back rows with a null prototype; the DO SQL + // flavour returns ArrayBuffer for BLOB columns. Re-key into a plain + // {} so consumers get Object.prototype-shaped rows (capnweb's + // serializer keys off Object.prototype to detect "object") and + // convert any ArrayBuffer to Uint8Array in the same pass. + const out: Record = {}; + for (const key of Object.keys(row)) { + const value = row[key]; + out[key] = value instanceof ArrayBuffer ? new Uint8Array(value) : value; + } + return out; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/sync/blobs.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/sync/blobs.ts new file mode 100644 index 00000000..d51fabe5 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/sync/blobs.ts @@ -0,0 +1,32 @@ +import { clearBlobCache } from "../fs/blobCache.js"; +import type { Database } from "../storage.js"; + +// Stage a chunk directly into vfs_blobs + vfs_blob_bytes without +// creating a node or a manifest. The receiver-side push path uses +// this to land bytes the sender shipped via pushObjects so a +// subsequent applyChanges call can find them by hash. +// +// Idempotent: a second call with the same hash refreshes +// last_seen so the bytes don't get reaped by an interleaved gc. +// Conflict updates also repair incomplete or size-mismatched rows +// left by an interrupted or corrupt write. +// +// Callers are expected to have verified that hash === sha256(bytes) +// before calling. The function trusts the caller; a mismatched +// pair would silently land under the wrong key. +export function stageBlob(db: Database, hash: Uint8Array, bytes: Uint8Array, now: number): void { + db.transactionSync(() => { + db.run( + "INSERT INTO vfs_blobs (hash, size, last_seen) VALUES (?, ?, ?) ON CONFLICT(hash) DO UPDATE SET size = excluded.size, last_seen = excluded.last_seen", + hash, + bytes.byteLength, + now, + ); + db.run( + "INSERT INTO vfs_blob_bytes (hash, bytes) VALUES (?, ?) ON CONFLICT(hash) DO UPDATE SET bytes = excluded.bytes", + hash, + bytes, + ); + }); + clearBlobCache(db); +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/sync/changes.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/sync/changes.ts new file mode 100644 index 00000000..2cef2326 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/sync/changes.ts @@ -0,0 +1,105 @@ +import { resolveInode } from "../fs/resolve.js"; +import { canonicalizePath } from "../path.js"; +import type { Database } from "../storage.js"; + +// Record a tombstone for a deleted path so the next push to the +// container learns the path is gone. Called by fs/rm inside the same +// transaction that bumped rev and removed the inode rows; the caller +// passes the post-bump rev value. +export function recordDelete(db: Database, rev: number, path: string): void { + db.run("INSERT INTO vfs_changes (rev, path, op) VALUES (?, ?, 'delete')", rev, path); +} + +// One row of the sync wire. The DO pushes these to the container +// and the container fetches them back. Bytes are never inline: +// file entries carry chunk hashes and the receiver does its own +// hasObjects probe + fetchObjects pull for the bytes it lacks. +// +// `rev` is the sender's currentRev at the moment this entry was +// stamped — vfs_nodes.rev for live mutations, vfs_changes.rev for +// tombstones. The puller uses it as a per-entry cursor so it can +// advance fetchRev per committed batch instead of waiting for the +// whole stream to drain. +export type ChangeEntry = + | { + kind: "file"; + rev: number; + path: string; + mode: number; + mtime: number; + size: number; + chunks: { hash: Uint8Array; size: number }[]; + } + | { kind: "dir"; rev: number; path: string; mode: number; mtime: number } + | { + kind: "symlink"; + rev: number; + path: string; + target: string; + mode: number; + mtime: number; + } + | { kind: "delete"; rev: number; path: string }; + +// Read the current state of `path` and turn it into a wire entry. +// Returns null when the path was never touched (no live inode and no +// tombstone in vfs_changes). Live inodes win over tombstones, which +// handles the delete-then-recreate case correctly. +// +// Symlinks are returned as symlink entries; we never follow them on +// the sync wire. Callers that want "the file the link points at" +// resolve it themselves after applying the symlink entry. +export function materialiseChange(db: Database, path: string): ChangeEntry | null { + const canonical = canonicalizePath(path).path; + const live = resolveInode(db, canonical, { followSymlinks: false }); + if (live !== null) { + // Read the rev stamped on this inode. Used as the per-entry + // cursor on the sync wire; coalesceChanges yields entries in + // ascending rev order so the puller can checkpoint per batch. + const revRow = db.one<{ rev: number }>("SELECT rev FROM vfs_nodes WHERE inode = ?", live.inode); + const rev = revRow?.rev ?? 0; + if (live.type === "dir") { + return { kind: "dir", rev, path: canonical, mode: live.mode, mtime: live.mtime }; + } + if (live.type === "symlink") { + return { + kind: "symlink", + rev, + path: canonical, + target: live.linkTarget ?? "", + mode: live.mode, + mtime: live.mtime, + }; + } + // file: collect chunk rows in index order. Each row carries hash + // and size so the receiver can probe hasObjects without a + // separate manifest lookup. An empty file has zero chunk rows + // and reports size 0. + const chunks = db.all<{ hash: Uint8Array; size: number }>( + "SELECT hash, size FROM vfs_chunks WHERE inode = ? ORDER BY idx", + live.inode, + ); + let size = 0; + for (const c of chunks) size += c.size; + return { + kind: "file", + rev, + path: canonical, + mode: live.mode, + mtime: live.mtime, + size, + chunks, + }; + } + // No live inode — check for a tombstone. The last row wins if the + // path was deleted and never recreated; an indexed scan by path is + // cheap because vfs_changes is bounded by the watermark window. + const tomb = db.one<{ rev: number; op: string }>( + "SELECT rev, op FROM vfs_changes WHERE path = ? ORDER BY id DESC LIMIT 1", + canonical, + ); + if (tomb?.op === "delete") { + return { kind: "delete", rev: tomb.rev, path: canonical }; + } + return null; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/sync/manifests.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/sync/manifests.ts new file mode 100644 index 00000000..98dfcfbc --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/sync/manifests.ts @@ -0,0 +1,74 @@ +import { createHash } from "node:crypto"; + +import type { Database } from "../storage.js"; + +// A manifest names the ordered chunk list for a single file. Two +// files whose bytes chunk identically share one manifest row, which +// is what lets the sync wire say "this file is the same as the one +// I just sent you" by hash alone. +// +// Encoding is JSON for now — readable, debuggable, and structurally +// identical to casync's `.caidx`. A future commit can swap the +// encoding to the `.caidx` byte layout without a schema change. + +export interface ManifestChunk { + hash: Uint8Array; + size: number; +} + +export const MANIFEST_VERSION = 1; + +interface EncodedManifest { + version: number; + chunks: { hash: string; size: number }[]; +} + +function toHex(bytes: Uint8Array): string { + let out = ""; + for (let i = 0; i < bytes.byteLength; i++) { + out += bytes[i].toString(16).padStart(2, "0"); + } + return out; +} + +function sha256(bytes: Uint8Array): Uint8Array { + return new Uint8Array(createHash("sha256").update(bytes).digest()); +} + +// Serialize a chunk list into the canonical manifest bytes. The +// hash is taken over these bytes and the same bytes are stored, so +// producing them once keeps the two in step. +function encodeManifest(chunks: ManifestChunk[]): Uint8Array { + const encoded: EncodedManifest = { + version: MANIFEST_VERSION, + chunks: chunks.map((c) => ({ hash: toHex(c.hash), size: c.size })), + }; + return new TextEncoder().encode(JSON.stringify(encoded)); +} + +// Compute the manifest hash for a chunk list without touching the +// DB. Used by the apply path to short-circuit when an upstream +// entry already matches the local node — the manifest hash is +// content-addressed so identical chunks always produce the same +// hash. +export function computeManifestHash(chunks: ManifestChunk[]): Uint8Array { + return sha256(encodeManifest(chunks)); +} + +// Build a manifest row for the given chunk list. Idempotent: a +// second call with the same chunks no-ops on the UNIQUE(hash). The +// returned hash is what the caller writes onto +// `vfs_nodes.manifest_hash`. +export function buildManifest(db: Database, chunks: ManifestChunk[], now: number): Uint8Array { + const bytes = encodeManifest(chunks); + const hash = sha256(bytes); + const size = chunks.reduce((acc, c) => acc + c.size, 0); + db.run( + "INSERT INTO vfs_manifests (hash, size, encoded, last_seen) VALUES (?, ?, ?, ?) ON CONFLICT(hash) DO UPDATE SET last_seen = excluded.last_seen", + hash, + size, + bytes, + now, + ); + return hash; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/sync/paths.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/sync/paths.ts new file mode 100644 index 00000000..d68a1e14 --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/sync/paths.ts @@ -0,0 +1,46 @@ +import { ROOT_INODE } from "../schema/index.js"; +import type { Database } from "../storage.js"; + +// Walk vfs_dirents from `inode` up to ROOT_INODE, gathering the path +// segments along the way. Returns null when the inode is unreachable. +export function pathOf(db: Database, inode: number): string | null { + if (inode === ROOT_INODE) return "/"; + const segments: string[] = []; + let current = inode; + // Bound the walk: a million levels deep is well past any real FS; + // anything beyond that is corruption and should not loop forever. + for (let i = 0; i < 1_000_000; i++) { + const row = db.one<{ parent_inode: number; name: string }>( + "SELECT parent_inode, name FROM vfs_dirents WHERE child_inode = ?", + current, + ); + if (row === undefined) return null; + segments.push(row.name); + if (row.parent_inode === ROOT_INODE) { + segments.reverse(); + return `/${segments.join("/")}`; + } + current = row.parent_inode; + } + return null; +} + +// Every path that currently names `inode`. A file may carry several +// hardlink names; pathOf collapses them to one arbitrary name, which +// is wrong for the change stream — every name has to reach the wire so +// the receiver materialises each. Directories cannot be hardlinked, so +// each parent walk is unambiguous. +export function pathsOf(db: Database, inode: number): string[] { + if (inode === ROOT_INODE) return ["/"]; + const dirents = db.all<{ parent_inode: number; name: string }>( + "SELECT parent_inode, name FROM vfs_dirents WHERE child_inode = ?", + inode, + ); + const paths: string[] = []; + for (const { parent_inode, name } of dirents) { + const parent = pathOf(db, parent_inode); + if (parent === null) continue; + paths.push(parent === "/" ? `/${name}` : `${parent}/${name}`); + } + return paths; +} diff --git a/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/types.ts b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/types.ts new file mode 100644 index 00000000..2832820e --- /dev/null +++ b/packages/workflow/vendor/cloudflare-computer-dofs/upstream/src/types.ts @@ -0,0 +1,16 @@ +export interface SQLCursorLike> { + toArray(): Row[]; +} + +export interface SQLStorageLike { + exec>( + query: string, + ...bindings: unknown[] + ): SQLCursorLike; +} + +export interface DurableObjectStorageLike { + sql: SQLStorageLike; + transaction?(closure: () => T | Promise): T | Promise; + transactionSync?(closure: () => T): T; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b9b0951f..48ff3aad 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -462,6 +462,9 @@ importers: effection: specifier: 4.1.0 version: 4.1.0 + zod: + specifier: ^4.3.6 + version: 4.4.3 packages: diff --git a/scripts/lib/verify.ts b/scripts/lib/verify.ts index 2e4a797d..72d1bd5b 100644 --- a/scripts/lib/verify.ts +++ b/scripts/lib/verify.ts @@ -58,6 +58,7 @@ export interface CommandSpec { /** The battery, in the order every report uses. */ export const BATTERY: readonly CommandSpec[] = [ + { id: "vendor", program: "deno", args: ["task", "vendor:verify"] }, { id: "lint", program: "deno", args: ["task", "lint"] }, { id: "check", program: "deno", args: ["task", "check"] }, { id: "test", program: "deno", args: ["task", "test"] }, diff --git a/scripts/runtime-test-exclusions.ts b/scripts/runtime-test-exclusions.ts index e02ba6e9..311a2f58 100644 --- a/scripts/runtime-test-exclusions.ts +++ b/scripts/runtime-test-exclusions.ts @@ -36,6 +36,12 @@ const DENO_ONLY_TOOLING: RuntimeExclusion[] = [ "builds the npm package with dnt, which only runs under Deno; the test calls Deno.readTextFileSync", issue: DERIVED_SCOPE, }, + { + path: "scripts/tests/cloudflare-dofs-vendor.test.ts", + reason: + "runs the no-network vendored-source verifier under the Deno executable against altered temporary snapshots", + issue: "https://github.com/taras/executable.md/issues/365", + }, { path: "scripts/tests/build-web-client.test.ts", reason: @@ -90,6 +96,12 @@ const DENO_ONLY_TOOLING: RuntimeExclusion[] = [ "the same Deno storage adapter, plus a restart proof that relaunches the run under the Deno executable; `node:sqlite` is behind --experimental-sqlite on Node 22", issue: DERIVED_SCOPE, }, + { + path: "packages/workflow/tests/workspace-filesystem.test.ts", + reason: + "exercises the Deno workflow storage adapter through node:sqlite and includes a real Deno child-process SIGKILL recovery proof", + issue: "https://github.com/taras/executable.md/issues/365", + }, ]; /** diff --git a/scripts/tests/cloudflare-dofs-vendor.test.ts b/scripts/tests/cloudflare-dofs-vendor.test.ts new file mode 100644 index 00000000..ea4200e2 --- /dev/null +++ b/scripts/tests/cloudflare-dofs-vendor.test.ts @@ -0,0 +1,66 @@ +import { cpSync, mkdtempSync, rmSync, unlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; +import { describe, it } from "@executablemd/test-support/bdd"; +import { expect } from "@executablemd/test-support/expect"; +import { exec } from "@executablemd/runtime"; +import { ensure } from "effection"; + +const REPOSITORY = fileURLToPath(new URL("../../", import.meta.url)); +const SNAPSHOT = join(REPOSITORY, "packages/workflow/vendor/cloudflare-computer-dofs"); +const VERIFY = join(REPOSITORY, "scripts/verify-cloudflare-dofs.ts"); + +function* refused(edit: (copy: string) => void) { + const temporary = mkdtempSync(join(tmpdir(), "xmd-dofs-drift-")); + yield* ensure(() => { + rmSync(temporary, { recursive: true, force: true }); + }); + const copy = join(temporary, "snapshot"); + cpSync(SNAPSHOT, copy, { recursive: true }); + edit(copy); + return yield* exec({ + command: [ + process.execPath, + "run", + "--allow-read", + "--allow-write=/tmp", + "--allow-env", + "--allow-run", + "--cached-only", + "--frozen", + VERIFY, + copy, + ], + cwd: REPOSITORY, + }); +} + +describe("Cloudflare Computer DOFS vendored snapshot", () => { + it("rejects changed source or generated output, missing, and extra files", function* () { + const changed = yield* refused((copy) => { + writeFileSync(join(copy, "upstream/src/path.ts"), "changed\n"); + }); + expect(changed.exitCode).not.toBe(0); + expect(changed.stderr).toContain("vendored file changed"); + + const generated = yield* refused((copy) => { + writeFileSync(join(copy, "generated/path.js"), "changed\n"); + }); + expect(generated.exitCode).not.toBe(0); + expect(generated.stderr).toContain("vendored file changed"); + + const missing = yield* refused((copy) => { + unlinkSync(join(copy, "upstream/src/path.ts")); + }); + expect(missing.exitCode).not.toBe(0); + expect(missing.stderr).toContain("vendored inventory differs"); + + const extra = yield* refused((copy) => { + writeFileSync(join(copy, "unrecorded.ts"), "export {};\n"); + }); + expect(extra.exitCode).not.toBe(0); + expect(extra.stderr).toContain("vendored inventory differs"); + }); +}); diff --git a/scripts/tests/verify-coordinator.test.ts b/scripts/tests/verify-coordinator.test.ts index e16e7a25..9fecf07f 100644 --- a/scripts/tests/verify-coordinator.test.ts +++ b/scripts/tests/verify-coordinator.test.ts @@ -260,7 +260,8 @@ describe("verify", () => { describe("line", () => { it("writes a root command as a reader would type it", function* () { - expect(line(BATTERY[0]!)).toEqual("deno task lint"); + const lint = BATTERY.find((command) => command.id === "lint")!; + expect(line(lint)).toEqual("deno task lint"); }); it("writes a site command with its directory", function* () { diff --git a/scripts/verify-cloudflare-dofs.ts b/scripts/verify-cloudflare-dofs.ts new file mode 100644 index 00000000..3f1dee35 --- /dev/null +++ b/scripts/verify-cloudflare-dofs.ts @@ -0,0 +1,182 @@ +import { createHash } from "node:crypto"; +import { dirname, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { z } from "zod"; + +const repositoryRoot = dirname(fileURLToPath(import.meta.url)); +const root = + Deno.args[0] === undefined + ? join(repositoryRoot, "../packages/workflow/vendor/cloudflare-computer-dofs") + : resolve(Deno.args[0]); +const manifestPath = join(root, "MANIFEST.json"); + +const expectedRepository = "https://github.com/cloudflare/computer"; +const expectedCommit = "63d363632e558f7e077794988d36ed75017c2a62"; + +const manifestSchema = z.object({ + format: z.literal(1), + repository: z.literal(expectedRepository), + commit: z.literal(expectedCommit), + compiler: z.string().min(1), + files: z.array( + z.object({ + path: z.string().min(1), + kind: z.enum(["license", "provenance", "upstream", "generated"]), + sha256: z.string().regex(/^[0-9a-f]{64}$/), + }), + ), +}); + +type Manifest = z.infer; + +function digest(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +function walk(directory: string): string[] { + const files: string[] = []; + for (const entry of Deno.readDirSync(directory)) { + const path = join(directory, entry.name); + if (entry.isDirectory) { + files.push(...walk(path)); + } else if (entry.isFile) { + files.push(relative(root, path)); + } else { + throw new Error(`vendored entry is not a regular file: ${relative(root, path)}`); + } + } + return files.sort(); +} + +function loadManifest(): Manifest { + let parsed: unknown; + try { + parsed = JSON.parse(Deno.readTextFileSync(manifestPath)); + } catch (error) { + throw new Error("Cloudflare DOFS manifest is unreadable", { cause: error }); + } + return manifestSchema.parse(parsed); +} + +function verifyInventory(manifest: Manifest): void { + const recorded = manifest.files.map((file) => file.path); + const duplicates = recorded.filter((path, index) => recorded.indexOf(path) !== index); + if (duplicates.length > 0) { + throw new Error(`duplicate vendored paths: ${[...new Set(duplicates)].join(", ")}`); + } + + const expected = [...recorded, "MANIFEST.json"].sort(); + const actual = walk(root); + if (JSON.stringify(actual) !== JSON.stringify(expected)) { + throw new Error( + `vendored inventory differs\nexpected: ${expected.join("\n")}\nactual: ${actual.join("\n")}`, + ); + } + + for (const file of manifest.files) { + const actualDigest = digest(Deno.readFileSync(join(root, file.path))); + if (actualDigest !== file.sha256) { + throw new Error( + `vendored file changed: ${file.path}\nexpected ${file.sha256}\nactual ${actualDigest}`, + ); + } + } +} + +function sourceFiles(manifest: Manifest): string[] { + return manifest.files + .filter((file) => file.kind === "upstream") + .map((file) => join(root, file.path)); +} + +function generatedFiles(manifest: Manifest): string[] { + return manifest.files + .filter((file) => file.kind === "generated") + .map((file) => file.path.slice("generated/".length)) + .sort(); +} + +function emit(manifest: Manifest, output: string): void { + const tsc = join(repositoryRoot, "../node_modules/typescript/bin/tsc"); + const command = new Deno.Command(Deno.execPath(), { + cwd: join(repositoryRoot, ".."), + args: [ + "run", + "--allow-read", + `--allow-write=${output}`, + "--allow-env", + tsc, + "--target", + "ES2022", + "--module", + "ES2022", + "--moduleResolution", + "bundler", + "--lib", + "ES2023,WebWorker", + "--strict", + "--skipLibCheck", + "--noEmit", + "false", + "--declaration", + "true", + "--rootDir", + join(root, "upstream/src"), + "--outDir", + output, + ...sourceFiles(manifest), + ], + stdout: "piped", + stderr: "piped", + }); + const result = command.outputSync(); + if (!result.success) { + const message = new TextDecoder().decode(result.stderr); + throw new Error(`vendored generated output did not compile\n${message}`); + } +} + +function verifyGenerated(manifest: Manifest): void { + const temporary = Deno.makeTempDirSync({ dir: "/tmp", prefix: "xmd-dofs-vendor-" }); + try { + emit(manifest, temporary); + const actual = walkGenerated(temporary); + const expected = generatedFiles(manifest); + if (JSON.stringify(actual) !== JSON.stringify(expected)) { + throw new Error( + `generated inventory drifted\nexpected: ${expected.join("\n")}\nactual: ${actual.join("\n")}`, + ); + } + for (const path of expected) { + const committed = Deno.readFileSync(join(root, "generated", path)); + const reproduced = Deno.readFileSync(join(temporary, path)); + if (digest(committed) !== digest(reproduced)) { + throw new Error(`generated output drifted: generated/${path}`); + } + } + } finally { + Deno.removeSync(temporary, { recursive: true }); + } +} + +function walkGenerated(directory: string): string[] { + const files: string[] = []; + for (const entry of Deno.readDirSync(directory)) { + const path = join(directory, entry.name); + if (entry.isDirectory) { + for (const child of walkGenerated(path)) { + files.push(join(entry.name, child)); + } + } else if (entry.isFile) { + files.push(entry.name); + } + } + return files.sort(); +} + +const manifest = loadManifest(); +verifyInventory(manifest); +verifyGenerated(manifest); +console.log( + `verified Cloudflare Computer DOFS ${expectedCommit}: ${manifest.files.length} recorded files`, +); diff --git a/specs/workflow-spec.md b/specs/workflow-spec.md index db66eb35..fb425d21 100644 --- a/specs/workflow-spec.md +++ b/specs/workflow-spec.md @@ -178,8 +178,10 @@ after an interruption. The Deno host installs its own with entrypoint is the only place SQLite, run-id hashing, filesystem paths and host behavior appear. Shared modules import none of them and detect no runtime. -A handle belongs to the scope that asked for it. Its connection closes through -ordinary teardown, and every later call answers with a closed-handle failure +A handle belongs to the scope that asked for it. It is a lease over the +provider's authoritative per-path connection and becomes unusable through +ordinary teardown. The provider closes that connection with its own scope; a +later call through the expired lease answers with a closed-handle failure rather than reopening the file. ### 9.1 What identifies a run @@ -263,6 +265,8 @@ collision or tampering, reported as its own failure and left unchanged. - Replaceable retrieval metadata, with a revision counting replacements since it was last cleared. - The filtered journal. +- The retained Workspace roots, current-root pointer, and DOFS filesystem + content described by the Workspace specification. ### 9.5 The journal @@ -286,6 +290,14 @@ Storage performs no filtering of its own — a second policy in a second place i a second thing to keep in agreement with the first — and a gate that rejects or is cancelled leaves no row at all. +Every version-1 event row names a retained Workspace root. Ordinary +non-Workspace appends use the current root. A Workspace mutation binds an +adapter-private destination only while its already-filtered Yield is being +published; that destination validates the database path, connection generation, +transaction identity and open state before it delegates to the caller-owned +transaction journal. Missing, foreign, completed, fabricated and stale +destinations cannot enlist. + ### 9.6 One connection, one operation Operations on one handle are serialized, and each runs inside a transaction. A @@ -317,6 +329,14 @@ share them, so a second handle waits while the first holds the database instead of entering SQLite and stopping the host. Contention between processes remains SQLite's own. +The Deno provider registry owns one physical connection, one DOFS database +wrapper, one Workspace filesystem, one cooperative connection queue and one +savepoint-name allocator for each run path. DOFS uses the same physical +connection and caller-owned transaction as the journal. Its synchronous nested +transactions are savepoints; a Workspace mutation also owns one +operation-spanning savepoint whose child scope tears down before release or +rollback. + A transaction opened inside another on the same database is refused rather than nested, and so is an ordinary operation called from inside a body — that call would otherwise wait for a transaction its own scope is holding open. @@ -351,6 +371,15 @@ A database is initialized only when it is pristine — no application id, no schema version and not one object anybody created. A file carrying a version but no tables, or tables belonging to something else, is not empty. +Version 1 has one frozen structural manifest containing the workflow tables, +the pinned Cloudflare DOFS schema-version-5 tables and indexes, immutable +Workspace root and reachability tables, the current-root singleton, and the +non-null root reference on every journal event. Initialization creates the +DOFS objects inside the same immediate transaction through a savepoint, creates +the canonical root-only empty Workspace, and selects it before commit. Existing +files are validation-only; the unsupported pre-release metadata-only version-1 +shape is refused without repair or migration and must be deleted and recreated. + Rows are held to what they mean and not only to their column types: a timestamp is an instant, an identity is not the empty string, and props are an object. @@ -363,12 +392,13 @@ An incompatible or damaged database is described and left exactly as it was found. Nothing initializes, migrates, truncates, deletes or replaces one, and a lookup that finds nothing creates no file. -Version 1 reads and writes version 1. An older version with no implemented -migration, and every newer version, are refused without the file being touched. +Version 1 reads and writes this complete version-1 shape. An older version with +no implemented migration, every newer version, and an intermediate pre-release +version-1 shape are refused without the file being touched. ## 10. Intentionally excluded Public `xmd workflow` lifecycle commands; lifecycle transition policy, executor -leases and stale-owner recovery; Workspace filesystem storage and its -transactions; history checkpoints and forks; workflow-owned worktrees; and -deterministic Git and GitHub effects. +leases and stale-owner recovery; public `` and history commands; history +forks; workflow-owned worktrees; Worker Shell; and deterministic Git and GitHub +effects. diff --git a/specs/workflow-workspace-spec.md b/specs/workflow-workspace-spec.md index dc39507b..3cfe0aa2 100644 --- a/specs/workflow-workspace-spec.md +++ b/specs/workflow-workspace-spec.md @@ -694,6 +694,17 @@ A crash commits all three or none. Nested child effects finish before the parent's effect transaction begins. Direct filesystem operations and declarative Git operations use this boundary. +The retained-filesystem foundation implements this boundary for +provider-level Workspace effects. Its durable-effect coordinator runs a live +mutation inside an operation savepoint and invokes the existing publication +continuation while the outer transaction remains open. Successful teardown +publishes a complete immutable root and its exact retained DOFS reachability +set before the filtered Yield. A known filesystem-domain failure rolls the +savepoint back and publishes one failed Yield against the prior root. Storage, +schema, corruption, routing, filtering, serialization and journal failures +roll back the outer transaction and publish nothing. Cancellation is never +converted into a failed effect. + ### 10.2 External effects Prompt, Push and PullRequest cannot place provider-owned state in SQLite. Each @@ -815,6 +826,38 @@ is outside the initial local capability set; Worker Shell follows §10.3. A late Cloudflare-hosted or workerd-backed provider may install the same Workspace and lifecycle contracts; documents do not choose that topology. +The provider registry owns one physical SQLite connection, one DOFS database +wrapper, one Workspace filesystem, one cooperative queue and one unique +savepoint allocator per run path. Scope-owned run-database handles are leases; +no second long-lived DOFS wrapper observes the same path. DOFS synchronous +transactions use nested savepoints on the exact connection and caller-owned +transaction used by the journal. + +The local database keeps XMD schema version 1, DOFS internal schema version 5 +and Workspace-root format version 1 as separate contracts. One frozen +`sqlite_schema` manifest covers every table, index and constraint. Only a +pristine database initializes: Cloudflare initialization runs through a +savepoint inside XMD's immediate transaction, then XMD creates a canonical +root-only empty Workspace and selects it before commit. Existing databases are +validation-only. The intermediate metadata-only version-1 shape is an +unsupported pre-release artifact and is refused byte-for-byte unchanged. + +Root format 1 is fixed-key-order canonical UTF-8 JSON containing `/` and every +reachable canonical absolute POSIX path in UTF-8 byte order. Names retain their +original code points and invalid or noncanonical names are refused. Entries +record kind, mode and observable mtime; files also record size, DOFS manifest +hash and deterministic path-order hardlink group; symlinks record their target. +The lowercase root ID is SHA-256 over +`"xmd-workspace-root\0v1\0"` followed by the canonical manifest bytes. + +An immutable root owns normalized references to exactly its transitive DOFS +manifests and blobs. Foreign keys prevent their deletion while the root remains +retained, so historical roots reuse content rather than copying bytes. Garbage +collection is disabled at this stage. An adapter-private materializer can +rebuild a complete valid DOFS frontier from a retained root inside a +caller-owned transaction and savepoint, then proves the rebuilt frontier +snapshots to the same root ID. No public history or fork command exposes it yet. + SQLite is a host implementation detail. The CLI deliberately exposes no remote host-selection option yet, while retaining a control surface that can be delegated without changing the document language. @@ -825,13 +868,14 @@ delegated without changing the document language. | --- | --- | | workflow-run and expansion identity | built by #289 / PR #341 | | retained run record and filtered journal | built by #291 | -| caller-owned storage transaction | built by #291; Workspace mutations join it in #365 | -| provider-backed retained Workspace | defined here; unbuilt (#218) | +| caller-owned storage transaction | built by #291; provider-level Workspace mutations join it atomically in #365 | +| retained Workspace filesystem foundation | built by #365; immutable roots, retained DOFS content, restoration and atomic filtered publication | +| public Workspace composition and `` | defined here; unbuilt (#218 / #366) | | Repository, Worktree and transactional Git components | defined here; unbuilt | | lifecycle start/resume/status/history/fork/delete | defined here; unbuilt | | read-only Agent materialization | defined here; proof required | | generated-XMD constrained evaluator | behavior defined; public name/schema open | -| Deno-local DOFS persistence | POC proven by #349 / PR #350 | +| Deno-local DOFS persistence | production retained-filesystem foundation built by #365 from the pinned upstream source; #349 / PR #350 remain evidence only | | scoped Deno Worker Shell | containment proven by #351 / PR #353 and transactions by #357 / PR #362; production integration unbuilt | | Worker JavaScript | deferred | | bundled workerd local host | omitted; POC #347 / PR #348 retained as provider evidence |