diff --git a/architecture.md b/architecture.md index 5cc28fbd..9d56ce73 100644 --- a/architecture.md +++ b/architecture.md @@ -550,6 +550,23 @@ supplies a provider-neutral failure activator so a provider can retain that first infrastructure failure by identity; the default coordinator does not use it and keeps its existing behavior. +Losing the host is not one of those outcomes, because nothing runs to handle +it. No cleanup, commit or rollback happens after the process dies: the +operating system closes its connection and releases the locks it held, and the +interrupted transaction is left for the next connection to recover. The +mutation, the immutable root, the current-root pointer and the routed event +have all been written by then, and recovery exposes the last committed state +and none of them. + +That is the same boundary every other reader already observes rather than a +second rule for crashes. A second connection sees the last committed state for +as long as the writer's transaction is uncommitted, so a crash publishes +nothing that was not visible before it. What a later process finds is that +committed state — the filesystem, the current root, the retained roots and +references, and the ordered journal with its event identities and root +associations — from which it performs no recorded effect again and can +materialize any retained root. + The Deno journal adapter routes an append ordinarily when no destination is bound. A publication may instead bind one exact transaction destination for its own lexical scope after the existing secret gate. The route validates the diff --git a/deno.json b/deno.json index 7854ef4b..bf91d5d8 100644 --- a/deno.json +++ b/deno.json @@ -42,6 +42,7 @@ "@std/testing/bdd": "jsr:@std/testing@^1/bdd", "oxlint": "npm:oxlint@1.74.0", "oxlint-tsgolint": "npm:oxlint-tsgolint@0.25.0", + "typescript": "npm:typescript@^5.0.0", "unist-util-select": "npm:unist-util-select@^5", "zod": "npm:zod@^4.3.6", "mdast-util-to-string": "npm:mdast-util-to-string@^4" diff --git a/deno.lock b/deno.lock index d822f1d8..7d09e342 100644 --- a/deno.lock +++ b/deno.lock @@ -3974,6 +3974,7 @@ "npm:mdast-util-to-string@4", "npm:oxlint-tsgolint@0.25.0", "npm:oxlint@1.74.0", + "npm:typescript@5", "npm:unist-util-select@5", "npm:zod@^4.3.6" ], diff --git a/packages/workflow/src/run.ts b/packages/workflow/src/run.ts index ef37c9e6..caef5dcb 100644 --- a/packages/workflow/src/run.ts +++ b/packages/workflow/src/run.ts @@ -28,7 +28,6 @@ import { createContext } from "effection"; import type { Context, Operation } from "effection"; -import { randomUUID } from "node:crypto"; import { createDurableOperation } from "@executablemd/durable-streams"; import { ReplayGuard } from "@executablemd/durable-streams"; import type { EffectDescription, Json, Workflow, Yield } from "@executablemd/durable-streams"; @@ -77,7 +76,9 @@ function* record(description: EffectDescription, base: string): Workflow + * deno run -A workspace-crash-child.ts inspect + * ``` + * + * `crash` opens the run, performs one real Workspace effect, and stops inside + * the still-open transaction with the mutation, the immutable root, the + * current-root pointer and the routed journal row all written and none of them + * committed. It reports what that connection can see and then waits to be + * killed. + * + * That stopping point is the connection registry's construction-time routed + * append hook, which only a caller that builds the registry can install. So + * `crash` assembles the same adapter modules the Deno provider installs, at the + * path the provider derives, rather than calling `useWorkflowRunStorage`. + * `inspect` has no such need and uses the provider itself. + */ + +import process from "node:process"; +import { durableRun, guardDurableStream, type Workflow } from "@executablemd/durable-streams"; +import { ensure, main, type Operation, suspend } from "effection"; +import { WorkflowRunStorage } from "../../mod.ts"; +import { useWorkflowRunStorage, workflowRunPath } from "../../deno.ts"; +import { createWorkflowRunConnections } from "../../src/deno/connections.ts"; +import { openWorkflowRunDatabase, readRunRow } from "../../src/deno/database.ts"; +import { useJournalRouting } from "../../src/deno/journal-route.ts"; +import { readTransaction } from "../../src/deno/reading.ts"; +import { verifySchema } from "../../src/deno/schema.ts"; +import { + createWorkspaceProofEffect, + useWorkspaceEffects, + withWorkspaceEffects, +} from "../../src/deno/workspace/effect.ts"; +import type { DenoWorkspaceFilesystem } from "../../src/deno/workspace/filesystem.ts"; +import { currentWorkspaceRoot } from "../../src/deno/workspace/root.ts"; +import { + setPrivateWorkspaceClock, + transactWorkspaceRoots, + usePrivateWorkspace, +} from "../../src/deno/workspace/private.ts"; +import { + BASELINE_EFFECT, + count, + CRASH_CONTENT, + CRASH_EFFECT, + CRASH_PATH, + readTree, + report, +} from "./workspace-process.ts"; + +const CLOCK = 1_750_000_100_000; + +function* crash(root: string, runId: string): Operation { + const path = workflowRunPath(root, runId); + let filesystem: DenoWorkspaceFilesystem | undefined; + let gateCalls = 0; + let baselineExecutions = 0; + + const connections = createWorkflowRunConnections(() => {}, { + *afterRoutedJournalAppend(_database, event): Operation { + if (event.type !== "yield" || filesystem === undefined) { + return; + } + // Every read below is on the connection that opened the transaction, so + // it sees that transaction's own uncommitted writes. Nothing else can. + const sqlite = connection.database; + const currentRoot = currentWorkspaceRoot(sqlite, path); + const journalRow = sqlite + .prepare( + `SELECT event_id, workspace_root_id FROM journal_events + WHERE record LIKE ? ORDER BY sequence DESC LIMIT 1`, + ) + .get(`%"name":"${CRASH_EFFECT}"%`); + report({ + ready: true, + content: yield* filesystem.readTextFile(CRASH_PATH), + currentRoot, + retainedRoots: count( + sqlite.prepare("SELECT COUNT(*) AS count FROM workspace_roots").get()?.["count"], + ), + currentRootRetained: count( + sqlite + .prepare("SELECT COUNT(*) AS count FROM workspace_roots WHERE root_id = ?") + .get(currentRoot)?.["count"], + ), + journalEventId: journalRow?.["event_id"], + journalRootId: journalRow?.["workspace_root_id"], + journalRows: count( + sqlite.prepare("SELECT COUNT(*) AS count FROM journal_events").get()?.["count"], + ), + gateCalls, + baselineExecutions, + }); + // Deno leaves when its event loop is empty, and a suspended Effection + // task is not on it. A timer nothing clears is what keeps this process + // and its open transaction alive until the signal arrives. + setInterval(() => {}, 1_000); + // The transaction stays open from here until the operating system takes + // this process away, which is what the process is for. + yield* suspend(); + }, + }); + yield* ensure(() => connections.close()); + + const connection = connections.at(path); + readTransaction(connection.database, () => { + verifySchema(connection.database, path, connection.dofs); + }); + const record = readRunRow(connection.database, path); + + yield* useJournalRouting(connections); + yield* usePrivateWorkspace(connections); + yield* useWorkspaceEffects(connections); + const database = yield* openWorkflowRunDatabase({ connection, connections, record }); + yield* setPrivateWorkspaceClock(database, () => CLOCK); + + const guarded = guardDurableStream(database.journal, function* (event) { + if (event.type === "yield") { + gateCalls += 1; + } + }); + + function* workflow(): Workflow { + // The run this process resumes already holds this effect's result, so it + // replays. Executing it would mean the crash effect below is not the + // first live work of the process, and the count says which happened. + yield createWorkspaceProofEffect( + database, + { type: "workspace-proof", name: BASELINE_EFFECT }, + // deno-lint-ignore require-yield + function* () { + baselineExecutions += 1; + return null; + }, + ); + yield createWorkspaceProofEffect( + database, + { type: "workspace-proof", name: CRASH_EFFECT }, + function* (selected) { + filesystem = selected; + yield* selected.writeFile(CRASH_PATH, CRASH_CONTENT, 0o640); + return null; + }, + ); + } + + yield* withWorkspaceEffects(database, durableRun(workflow, { stream: guarded })); + report({ ready: false, reason: "the crash effect committed" }); +} + +function* inspect(root: string, runId: string): Operation { + yield* useWorkflowRunStorage({ root }); + const opened = yield* WorkflowRunStorage.operations.lookup(runId); + if (!opened.ok) { + throw opened.error; + } + const database = opened.value; + + const entries = yield* database.readJournalEntries(); + if (!entries.ok) { + throw entries.error; + } + + const observed = yield* transactWorkspaceRoots(database, function* (workspace) { + return { + currentRoot: yield* workspace.currentRoot(), + tree: yield* readTree(workspace.filesystem, "/"), + }; + }); + if (!observed.ok) { + throw observed.error; + } + + report({ + ...observed.value, + events: entries.value.map((entry) => ({ + eventId: entry.eventId, + name: entry.event.type === "yield" ? entry.event.description.name : undefined, + })), + }); +} + +main(function* () { + // `process.argv` rather than `Deno.args`: this file is Deno-only to run, and + // still has to typecheck under the Node project like every other source. + const [mode, root, runId] = process.argv.slice(2); + if (mode === "crash") { + yield* crash(root, runId); + return; + } + if (mode === "inspect") { + yield* inspect(root, runId); + return; + } + throw new Error(`the Workspace crash helper has no ${mode} mode`); +}); diff --git a/packages/workflow/tests/support/workspace-process.ts b/packages/workflow/tests/support/workspace-process.ts new file mode 100644 index 00000000..3674a716 --- /dev/null +++ b/packages/workflow/tests/support/workspace-process.ts @@ -0,0 +1,67 @@ +/** + * What the crash and restart processes and the test that drives them agree on. + * + * The helpers below are imported by a child process and by the suite that + * launches it, so this module starts nothing: a child's `main()` lives in the + * child's own file, and importing a constant from it would run the child + * inside the test. + */ + +import type { Operation } from "effection"; +import type { DenoWorkspaceFilesystem } from "../../src/deno/workspace/filesystem.ts"; + +/** The mutation the killed process performs, and never publishes. */ +export const CRASH_PATH = "/crash.txt"; +export const CRASH_CONTENT = "bytes that must never be published"; +export const CRASH_EFFECT = "crash-before-commit"; + +/** The baseline the crash runs against, committed before the child starts. */ +export const BASELINE_PATH = "/baseline.txt"; +export const BASELINE_CONTENT = "committed baseline bytes"; +export const NESTED_PATH = "/kept/nested.txt"; +export const NESTED_CONTENT = "nested baseline bytes"; +export const BASELINE_EFFECT = "baseline"; +export const BASELINE_CLOCK = 1_750_000_000_000; + +/** The two committed effects of the restart proof, and what the first retains. */ +export const SEED_CLOCK = 10_000; +export const REVISE_CLOCK = 20_000; +export const HISTORICAL_PATH = "/tree/file.txt"; +export const HISTORICAL_CONTENT = "historical bytes"; + +/** A SQLite count, which arrives as a `bigint` from a read that asked for one. */ +export function count(value: unknown): number { + return typeof value === "bigint" ? Number(value) : Number(value); +} + +/** Everything the Workspace holds, as one comparable value. */ +export function* readTree( + filesystem: DenoWorkspaceFilesystem, + directory: string, +): Operation> { + const tree: Record = {}; + for (const entry of yield* filesystem.readdir(directory)) { + const path = directory === "/" ? `/${entry.name}` : `${directory}/${entry.name}`; + const stat = yield* filesystem.lstat(path); + if (entry.kind === "directory") { + tree[path] = { kind: "directory", mode: stat.mode, mtime: stat.mtime }; + Object.assign(tree, yield* readTree(filesystem, path)); + } else if (entry.kind === "symlink") { + tree[path] = { kind: "symlink", target: yield* filesystem.readlink(path) }; + } else { + tree[path] = { + kind: "file", + mode: stat.mode, + mtime: stat.mtime, + size: stat.size, + content: yield* filesystem.readTextFile(path), + }; + } + } + return tree; +} + +/** One line of JSON on standard output is the whole protocol with the parent. */ +export function report(value: unknown): void { + console.log(JSON.stringify(value)); +} diff --git a/packages/workflow/tests/support/workspace-restart-child.ts b/packages/workflow/tests/support/workspace-restart-child.ts new file mode 100644 index 00000000..c3a94e04 --- /dev/null +++ b/packages/workflow/tests/support/workspace-restart-child.ts @@ -0,0 +1,213 @@ +/** + * A committed Workspace, and a second process that has never seen it. + * + * `commit` performs two real Workspace effects through the production Deno + * provider and exits normally, so its connection, its DOFS wrapper and every + * cache they held are gone before anything else looks. `restore` reopens the + * same run and answers three questions the first process cannot: whether the + * committed filesystem, root and journal come back; whether replay performs + * the recorded effects a second time; and whether an event's historical root + * still materializes exactly. + * + * ```sh + * deno run -A workspace-restart-child.ts commit + * deno run -A workspace-restart-child.ts read + * deno run -A workspace-restart-child.ts restore + * ``` + * + * Each effect appends its name to the marker file, so "did this run again" is + * observable from outside the process rather than inferred from its output. + */ + +import { appendFileSync } from "node:fs"; +import process from "node:process"; +import { durableRun, type Workflow } from "@executablemd/durable-streams"; +import { main, type Operation } from "effection"; +import { WorkflowRunStorage, type WorkflowRunDatabase } from "../../mod.ts"; +import { useWorkflowRunStorage } from "../../deno.ts"; +import { + createWorkspaceProofEffect, + withWorkspaceEffects, +} from "../../src/deno/workspace/effect.ts"; +import { + setPrivateWorkspaceClock, + transactWorkspaceRoots, +} from "../../src/deno/workspace/private.ts"; +import { + HISTORICAL_CONTENT, + HISTORICAL_PATH, + readTree, + report, + REVISE_CLOCK, + SEED_CLOCK, +} from "./workspace-process.ts"; + +const DEFINITION = { + version: 1, + kind: "git", + objectFormat: "sha1", + objectId: "9fceb02d0ae598e95dc970b74767f19372d61af8", + rootDocumentPath: "workflows/release.md", +} as const; + +/** + * Two effects, so there is a history to select from. + * + * The first builds every topology the root format describes: a directory with + * its own mode, a file with its own mode, a hardlink and a symbolic link. The + * second takes all of it apart again — overwrite, mode change, rename, two + * deletions, a different symbolic link and a new directory. Nothing the first + * effect wrote survives at the path it wrote it to, so restoring its root + * cannot be satisfied by whatever happens to be live. + */ +function workflow(database: WorkflowRunDatabase, marker: string, clock: { now: number }) { + return function* (): Workflow { + yield createWorkspaceProofEffect( + database, + { type: "workspace-proof", name: "seed" }, + function* (filesystem) { + appendFileSync(marker, "seed\n"); + yield* filesystem.mkdir("/tree", { mode: 0o750 }); + yield* filesystem.writeFile(HISTORICAL_PATH, HISTORICAL_CONTENT, 0o640); + yield* filesystem.link(HISTORICAL_PATH, "/tree/hardlink.txt"); + yield* filesystem.symlink("file.txt", "/tree/current.txt"); + return null; + }, + ); + yield createWorkspaceProofEffect( + database, + { type: "workspace-proof", name: "revise" }, + function* (filesystem) { + appendFileSync(marker, "revise\n"); + clock.now = REVISE_CLOCK; + yield* filesystem.writeFile(HISTORICAL_PATH, "later bytes"); + yield* filesystem.chmod(HISTORICAL_PATH, 0o600); + yield* filesystem.rename(HISTORICAL_PATH, "/renamed.txt"); + yield* filesystem.remove("/tree/hardlink.txt"); + yield* filesystem.remove("/tree/current.txt"); + yield* filesystem.symlink("/renamed.txt", "/latest.txt"); + yield* filesystem.mkdir("/later", { mode: 0o700 }); + return null; + }, + ); + }; +} + +function* openRun(root: string, runId: string, create: boolean): Operation { + yield* useWorkflowRunStorage({ root }); + const opened = create + ? yield* WorkflowRunStorage.operations.create({ + runId, + definition: DEFINITION, + base: "main", + props: { channel: "stable" }, + }) + : yield* WorkflowRunStorage.operations.lookup(runId); + if (!opened.ok) { + throw opened.error; + } + return opened.value; +} + +function* observe(database: WorkflowRunDatabase): Operation> { + const entries = yield* database.readJournalEntries(); + if (!entries.ok) { + throw entries.error; + } + const state = yield* transactWorkspaceRoots(database, function* (workspace) { + return { + currentRoot: yield* workspace.currentRoot(), + tree: yield* readTree(workspace.filesystem, "/"), + }; + }); + if (!state.ok) { + throw state.error; + } + return { + ...state.value, + events: entries.value.map((entry) => ({ + eventId: entry.eventId, + name: entry.event.type === "yield" ? entry.event.description.name : undefined, + })), + }; +} + +function* run( + root: string, + runId: string, + marker: string, + create: boolean, +): Operation { + const database = yield* openRun(root, runId, create); + const clock = { now: SEED_CLOCK }; + yield* setPrivateWorkspaceClock(database, () => clock.now); + yield* withWorkspaceEffects( + database, + durableRun(workflow(database, marker, clock), { stream: database.journal }), + ); + return database; +} + +function* commit(root: string, runId: string, marker: string): Operation { + report(yield* observe(yield* run(root, runId, marker, true))); +} + +/** + * The same workflow, in a process that never ran it. + * + * Every effect is already in the journal, so replay must restore each result + * without reaching the coordinator — and the marker file is what says whether + * it did. + */ +function* read(root: string, runId: string, marker: string): Operation { + report(yield* observe(yield* run(root, runId, marker, false))); +} + +function* restore(root: string, runId: string, marker: string, rootId: string): Operation { + const database = yield* run(root, runId, marker, false); + const committed = yield* observe(database); + + const restored = yield* transactWorkspaceRoots(database, function* (workspace) { + let absent: string | undefined; + try { + yield* workspace.filesystem.readTextFile(HISTORICAL_PATH); + } catch (error) { + absent = error instanceof Error ? error.name : "unknown"; + } + + const selected = yield* workspace.restore(rootId, { publish: true }); + const tree = yield* readTree(workspace.filesystem, "/"); + const resnapshot = yield* workspace.capture({ publish: true }); + return { + absent, + selectedRoot: selected.rootId, + manifestHashes: selected.manifestHashes, + blobHashes: selected.blobHashes, + resnapshotRoot: resnapshot.rootId, + currentRoot: yield* workspace.currentRoot(), + tree, + }; + }); + if (!restored.ok) { + throw restored.error; + } + + report({ committed, restored: restored.value }); +} + +main(function* () { + const [mode, root, runId, marker, rootId] = process.argv.slice(2); + if (mode === "commit") { + yield* commit(root, runId, marker); + return; + } + if (mode === "read") { + yield* read(root, runId, marker); + return; + } + if (mode === "restore") { + yield* restore(root, runId, marker, rootId); + return; + } + throw new Error(`the Workspace restart helper has no ${mode} mode`); +}); diff --git a/packages/workflow/tests/workspace-crash-recovery.test.ts b/packages/workflow/tests/workspace-crash-recovery.test.ts new file mode 100644 index 00000000..0d9c2a2a --- /dev/null +++ b/packages/workflow/tests/workspace-crash-recovery.test.ts @@ -0,0 +1,424 @@ +/** + * Tier WAC, continued — what survives a process, and what must not. + * + * The rest of the atomic Workspace suite ends its transactions in this + * process: it commits, fails, or is cancelled, and Effection tears the scope + * down. None of that is a crash. At the kill point the transaction is open; + * `SIGKILL` then runs no application cleanup at all, so nothing commits and + * nothing rolls back. The operating system closes the connection and releases + * its locks, and the next connection to open the database recovers the + * interrupted transaction to the last committed state. Whether the mutation, + * the immutable root, the current-root pointer and the routed journal row + * reappear is decided there rather than by any code here. + * + * So the proofs below are made of real processes. One is killed with SIGKILL + * while it holds all four of those writes uncommitted; another, which has + * never seen it, reopens the database and must find the baseline exactly. Two + * more commit a Workspace history and then reconstruct an older event's root + * from a cold start. + * + * The handshake is a line of JSON on standard output, never a sleep: the + * parent kills the child at a point the child has said it has reached. + */ + +import { readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import process from "node:process"; +import { DatabaseSync } from "node:sqlite"; +import { fileURLToPath } from "node:url"; +import { describe, it } from "@executablemd/test-support/bdd"; +import { expect } from "@executablemd/test-support/expect"; +import { exec as execProcess } from "@effectionx/process"; +import { exec } from "@executablemd/runtime"; +import { guardDurableStream } from "@executablemd/durable-streams"; +import { call, type Operation, race, scoped, spawn, withResolvers } from "effection"; +import { setPrivateWorkspaceClock, transactWorkspaceRoots } from "../src/deno/workspace/private.ts"; +import { createRun, runPath, useStorageRoot, withStorage } from "./support/storage.ts"; +import { + BASELINE_CLOCK, + BASELINE_CONTENT, + BASELINE_EFFECT, + BASELINE_PATH, + count, + CRASH_CONTENT, + CRASH_EFFECT, + CRASH_PATH, + HISTORICAL_CONTENT, + HISTORICAL_PATH, + NESTED_CONTENT, + NESTED_PATH, + REVISE_CLOCK, + SEED_CLOCK, +} from "./support/workspace-process.ts"; + +const REPOSITORY = fileURLToPath(new URL("../../..", import.meta.url)); +const CRASH_CHILD = fileURLToPath(new URL("./support/workspace-crash-child.ts", import.meta.url)); +const RESTART_CHILD = fileURLToPath( + new URL("./support/workspace-restart-child.ts", import.meta.url), +); + +interface ReportedEvent { + readonly eventId: string; + readonly name?: string; +} + +/** + * Event names in order, with the run's terminating Close named too. + * + * `toEqual` treats a trailing `undefined` as absent, so a Close reported as an + * unnamed event would let a comparison of names pass without it. + */ +function names(events: readonly ReportedEvent[]): string[] { + return events.map((event) => event.name ?? "close"); +} + +interface ChildResult { + readonly code: number; + readonly out: string; + readonly err: string; +} + +/** One whole child process, reaped before the operation returns. */ +function* runChild(script: string, args: string[]): Operation { + const result = yield* exec({ + command: [process.execPath, "run", "--allow-all", "--frozen", script, ...args], + cwd: REPOSITORY, + }); + return { code: result.exitCode, out: result.stdout, err: result.stderr }; +} + +function announced(result: ChildResult): Record { + if (result.code !== 0) { + throw new Error(`the child exited ${result.code}: ${result.err}`); + } + return JSON.parse(result.out); +} + +/** + * Everything a second connection can see of the database right now. + * + * Raw SQLite rather than another DOFS wrapper: the question is only what has + * been committed, and a second authoritative wrapper on the same path is the + * thing the provider exists to prevent. + */ +function committed(path: string): Record { + const sqlite = new DatabaseSync(path, { readOnly: true }); + try { + sqlite.exec("PRAGMA busy_timeout = 10000"); + const total = (table: string): number => + count(sqlite.prepare(`SELECT COUNT(*) AS count FROM ${table}`).get()?.["count"]); + return { + currentRoot: sqlite.prepare("SELECT current_root_id FROM workspace_state").get()?.[ + "current_root_id" + ], + roots: total("workspace_roots"), + manifestRefs: total("workspace_root_manifest_refs"), + blobRefs: total("workspace_root_blob_refs"), + manifests: total("vfs_manifests"), + blobs: total("vfs_blobs"), + names: sqlite + .prepare("SELECT name FROM vfs_dirents ORDER BY name") + .all() + .map((row) => row["name"]), + journal: sqlite + .prepare("SELECT event_id, workspace_root_id, record FROM journal_events ORDER BY sequence") + .all() + .map((row) => ({ + eventId: row["event_id"], + rootId: row["workspace_root_id"], + name: JSON.parse(String(row["record"])).description?.name, + })), + }; + } finally { + sqlite.close(); + } +} + +/** The retained roots an event never named, so restoration cannot borrow them. */ +function manifestsOf(path: string, rootId: string): string[] { + const sqlite = new DatabaseSync(path, { readOnly: true }); + try { + return sqlite + .prepare( + "SELECT hex(manifest_hash) AS hash FROM workspace_root_manifest_refs WHERE root_id = ?", + ) + .all(rootId) + .map((row) => String(row["hash"]).toLowerCase()); + } finally { + sqlite.close(); + } +} + +/** + * A committed, non-empty Workspace and one filtered journal event. + * + * Recorded as a run that has not finished, rather than as a completed + * `durableRun`: the crash process resumes this run, and a journal that already + * holds a Close is a run with nothing left to execute. So the mutation and its + * root are published through the private Workspace transaction, and the one + * event is appended through a secret gate — the same two writes a coordinated + * effect commits, in a shape the next process continues from. + */ +function* baseline(root: string, runId: string): Operation { + yield* withStorage(root, function* () { + const database = yield* createRun({ runId }); + yield* setPrivateWorkspaceClock(database, () => BASELINE_CLOCK); + const captured = yield* transactWorkspaceRoots(database, function* (workspace) { + yield* workspace.filesystem.mkdir("/kept", { mode: 0o750 }); + yield* workspace.filesystem.writeFile(BASELINE_PATH, BASELINE_CONTENT, 0o640); + yield* workspace.filesystem.writeFile(NESTED_PATH, NESTED_CONTENT, 0o600); + return yield* workspace.capture({ publish: true }); + }); + if (!captured.ok) { + throw captured.error; + } + const guarded = guardDurableStream(database.journal, function* () {}); + yield* guarded.append({ + type: "yield", + coroutineId: "root", + description: { type: "workspace-proof", name: BASELINE_EFFECT }, + result: { status: "ok", value: null }, + }); + }); +} + +describe("Tier WAC — Workspace effects across a process boundary", () => { + it("WAC24: a real SIGKILL before commit leaves the baseline and nothing else", function* () { + const root = yield* useStorageRoot(); + const runId = "crash-before-commit"; + yield* baseline(root, runId); + + // Taken after the provider scope closed, so nothing this process holds is + // keeping the database open when the child starts. + const path = runPath(root, runId); + const before = committed(path); + const baselineJournal = before["journal"] as (ReportedEvent & { rootId: string })[]; + expect(names(baselineJournal)).toEqual([BASELINE_EFFECT]); + expect(before["roots"]).toBe(2); + expect(baselineJournal[0].rootId).toBe(before["currentRoot"]); + + const during = yield* scoped(function* () { + const child = yield* execProcess(process.execPath, { + arguments: ["run", "--allow-all", "--frozen", CRASH_CHILD, "crash", root, runId], + cwd: REPOSITORY, + }); + const ready = withResolvers>(); + const decoder = new TextDecoder(); + let out = ""; + let err = ""; + yield* spawn(function* () { + const subscription = yield* child.stdout; + let next = yield* subscription.next(); + while (!next.done) { + out += decoder.decode(next.value, { stream: true }); + const end = out.indexOf("\n"); + if (end >= 0) { + ready.resolve(JSON.parse(out.slice(0, end))); + } + next = yield* subscription.next(); + } + }); + yield* spawn(function* () { + const subscription = yield* child.stderr; + let next = yield* subscription.next(); + while (!next.done) { + err += decoder.decode(next.value, { stream: true }); + next = yield* subscription.next(); + } + }); + + const announcement = yield* race([ + ready.operation, + call(function* (): Operation> { + const status = yield* child.join(); + throw new Error( + `the crash child ended before its handshake (${JSON.stringify(status)}): ${err}`, + ); + }), + ]); + + // The child holds its transaction open here, so what a second connection + // can see is the discriminating observation: it must still be exactly + // the baseline, with no crash mutation, root, pointer change or event. + const outside = committed(path); + process.kill(child.pid, "SIGKILL"); + const status = yield* child.join(); + return { announcement, outside, status }; + }); + + // The child died from the signal rather than from cleanup of its own. + expect(during.status.signal).toBe("SIGKILL"); + expect(during.status.code ?? null).toBeNull(); + + // Inside the crashed transaction all four writes existed. + expect(during.announcement).toEqual({ + ready: true, + content: CRASH_CONTENT, + currentRoot: expect.any(String), + retainedRoots: 3, + currentRootRetained: 1, + journalEventId: expect.any(String), + journalRootId: during.announcement["currentRoot"], + journalRows: baselineJournal.length + 1, + gateCalls: 1, + baselineExecutions: 0, + }); + expect(during.announcement["currentRoot"]).not.toBe(before["currentRoot"]); + + // None of them was visible to anyone else while the child was alive. The + // two readings are of one database at one moment, through the connection + // that made the writes and through a connection that did not. + expect(during.outside["currentRoot"]).not.toBe(during.announcement["currentRoot"]); + expect(during.outside["roots"]).toBe(2); + expect(during.outside).toEqual(before); + + // A different process, through a newly installed production provider, + // finds the baseline and only the baseline. + const inspected = announced(yield* runChild(CRASH_CHILD, ["inspect", root, runId])); + expect(inspected["currentRoot"]).toBe(before["currentRoot"]); + expect(inspected["tree"]).toEqual({ + "/baseline.txt": { + kind: "file", + mode: 0o640, + mtime: BASELINE_CLOCK, + size: BASELINE_CONTENT.length, + content: BASELINE_CONTENT, + }, + "/kept": { kind: "directory", mode: 0o750, mtime: BASELINE_CLOCK }, + [NESTED_PATH]: { + kind: "file", + mode: 0o600, + mtime: BASELINE_CLOCK, + size: NESTED_CONTENT.length, + content: NESTED_CONTENT, + }, + }); + const inspectedEvents = inspected["events"] as ReportedEvent[]; + expect(names(inspectedEvents)).toEqual([BASELINE_EFFECT]); + expect(inspectedEvents.map((event) => event.eventId)).toEqual( + baselineJournal.map((event) => event.eventId), + ); + expect(JSON.stringify(inspected["tree"])).not.toContain(CRASH_PATH); + + // Recovery restored the file itself, not only what the adapter reads. + expect(committed(path)).toEqual(before); + }); + + it("WAC25: a second process restores the committed Workspace and re-executes nothing", function* () { + const root = yield* useStorageRoot(); + const runId = "workspace-restart"; + const marker = join(root, "restart-marker.txt"); + writeFileSync(marker, ""); + + const first = announced(yield* runChild(RESTART_CHILD, ["commit", root, runId, marker])); + expect(readFileSync(marker, "utf8")).toBe("seed\nrevise\n"); + + const path = runPath(root, runId); + const stored = committed(path); + const second = announced(yield* runChild(RESTART_CHILD, ["read", root, runId, marker])); + + // Nothing ran twice: the second process restored both recorded results. + expect(readFileSync(marker, "utf8")).toBe("seed\nrevise\n"); + expect(second["currentRoot"]).toBe(first["currentRoot"]); + expect(second["tree"]).toEqual(first["tree"]); + expect(second["tree"]).toEqual({ + "/tree": { kind: "directory", mode: 0o750, mtime: SEED_CLOCK }, + "/renamed.txt": { + kind: "file", + mode: 0o600, + mtime: REVISE_CLOCK, + size: 11, + content: "later bytes", + }, + "/latest.txt": { kind: "symlink", target: "/renamed.txt" }, + "/later": { kind: "directory", mode: 0o700, mtime: REVISE_CLOCK }, + }); + expect(second["events"]).toEqual(first["events"]); + expect(names(second["events"] as ReportedEvent[])).toEqual(["seed", "revise", "close"]); + + // Each event keeps the root it was committed against, and the later one + // is the current root a cold process reads. + const journal = stored["journal"] as (ReportedEvent & { rootId: string })[]; + expect(names(journal)).toEqual(["seed", "revise", "close"]); + expect(journal.map((row) => row.eventId)).toEqual( + (first["events"] as ReportedEvent[]).map((event) => event.eventId), + ); + expect(journal[0].rootId).not.toBe(journal[1].rootId); + expect(journal[1].rootId).toBe(first["currentRoot"]); + expect(journal[2].rootId).toBe(first["currentRoot"]); + expect(committed(path)).toEqual(stored); + }); + + it("WAC26: an older event's root reconstructs exactly in a fresh process", function* () { + const root = yield* useStorageRoot(); + const runId = "workspace-history"; + const marker = join(root, "history-marker.txt"); + writeFileSync(marker, ""); + + const first = announced(yield* runChild(RESTART_CHILD, ["commit", root, runId, marker])); + const path = runPath(root, runId); + const stored = committed(path); + const journal = stored["journal"] as (ReportedEvent & { rootId: string })[]; + expect(names(journal)).toEqual(["seed", "revise", "close"]); + const historical = journal[0].rootId; + const later = journal[1].rootId; + expect(historical).not.toBe(later); + + // The retained content the older root needs is no longer reachable from + // the live frontier, so restoration has to come from what that root + // retains rather than from anything still live. + const retired = manifestsOf(path, historical).filter( + (hash) => !manifestsOf(path, later).includes(hash), + ); + expect(retired.length).toBeGreaterThan(0); + + const replayed = announced( + yield* runChild(RESTART_CHILD, ["restore", root, runId, marker, historical]), + ); + expect(readFileSync(marker, "utf8")).toBe("seed\nrevise\n"); + + const restored = replayed["restored"] as Record; + // The negative lookup happened first and was answered from the live + // frontier, so the successful read after restoration is the authoritative + // negative cache having been invalidated rather than never consulted. + expect(restored["absent"]).toBe("WorkspaceFsError"); + expect(restored["selectedRoot"]).toBe(historical); + // Two paths reading the same bytes are not yet one file. The canonical + // manifest names the hardlink group, so a rebuild that gave them separate + // inodes would resnapshot to a different identity than the one selected. + expect(restored["resnapshotRoot"]).toBe(historical); + expect(restored["currentRoot"]).toBe(historical); + expect(restored["tree"]).toEqual({ + "/tree": { kind: "directory", mode: 0o750, mtime: SEED_CLOCK }, + [HISTORICAL_PATH]: { + kind: "file", + mode: 0o640, + mtime: SEED_CLOCK, + size: HISTORICAL_CONTENT.length, + content: HISTORICAL_CONTENT, + }, + "/tree/hardlink.txt": { + kind: "file", + mode: 0o640, + mtime: SEED_CLOCK, + size: HISTORICAL_CONTENT.length, + content: HISTORICAL_CONTENT, + }, + "/tree/current.txt": { kind: "symlink", target: "file.txt" }, + }); + expect(restored["manifestHashes"]).toEqual(expect.arrayContaining(retired)); + + // The committed observations the same process made before restoring are + // still the ones the first process published. + const committedAgain = replayed["committed"] as Record; + expect(committedAgain["currentRoot"]).toBe(first["currentRoot"]); + expect(committedAgain["events"]).toEqual(first["events"]); + + // Restoration published a root that was already retained; it created no + // new one and lost no history. + const after = committed(path); + expect(after["roots"]).toBe(stored["roots"]); + expect(after["currentRoot"]).toBe(historical); + expect((after["journal"] as unknown[]).length).toBe(journal.length); + }); +}); diff --git a/packages/workflow/tests/workspace-effect.test.ts b/packages/workflow/tests/workspace-effect.test.ts index b2572266..c0bcafd5 100644 --- a/packages/workflow/tests/workspace-effect.test.ts +++ b/packages/workflow/tests/workspace-effect.test.ts @@ -1,7 +1,11 @@ +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; import { describe, it } from "@executablemd/test-support/bdd"; import { expect } from "@executablemd/test-support/expect"; import { createApi } from "@effectionx/context-api"; import { readTextFile } from "@effectionx/fs"; +import { glob } from "@executablemd/runtime"; +import ts from "typescript"; import { type Operation, scoped } from "effection"; import { claimDurablePublicationIdentity, @@ -73,6 +77,323 @@ function successfulProvider(observe?: (authority: WorkspaceCoordinationAuthority }; } +const REPOSITORY = fileURLToPath(new URL("../../..", import.meta.url)); + +/** + * Storage and adapter type names that mean a host reached this surface. + * + * Distinctive enough to be read as text. Runtime globals are not here — `Bun` + * is inside `Bundle` and `Deno` inside `Denominator`, so those are recognized + * as identifiers instead. + */ +const FORBIDDEN = [ + "DatabaseSync", + "SQLite", + "sqlite", + "Cloudflare", + "DOFS", + "dofs", + "savepoint", + "Savepoint", + "SAVEPOINT", + "RunConnection", + "WorkflowRunConnections", + "WorkflowRunTransactionToken", + "ConnectionGeneration", + "TransactionIdentity", +]; + +/** + * The names this repository gives a runtime-specific entry point. + * + * Code Rule 12 puts host behavior behind runtime-named modules — + * `packages/cli/src/{deno,node,bun,compiled}.ts` are the CLI's — so the name of + * the module is what says a host owns it. `deno` is not the only one, and an + * adapter rule that knows only `deno` is a rule about the adapter someone + * happened to write first. + */ +const RUNTIMES = ["deno", "node", "bun", "compiled", "cloudflare", "workerd"]; + +/** + * Globals only one host provides. + * + * `crypto`, `TextEncoder` and the rest of the cross-runtime Web surface are + * not here: naming a standard is not naming a host. + */ +const HOST_GLOBALS = [ + "process", + "Deno", + "Bun", + "Buffer", + "globalThis", + "navigator", + "__dirname", + "__filename", +]; + +/** + * A module specifier only one host can resolve. + * + * Named by shape rather than one at a time: a list of the host modules anyone + * thought of is a list of the ones that had already been noticed, and the + * import that crosses this boundary next is the one nobody wrote down. + * + * Segments are compared whole. `nodes/`, `bundle.ts` and `vendors/` contain a + * runtime's name without being one, and rejecting them would make the rule + * about spelling rather than about hosts. + */ +function hostModule(specifier: string): boolean { + if (/^(node|bun|deno|cloudflare|workerd):/.test(specifier)) { + return true; + } + if (specifier === "@effectionx/process") { + return true; + } + const segments = specifier.split("/"); + const last = segments[segments.length - 1].replace(/\.[cm]?[jt]sx?$/, ""); + return ( + segments.includes("vendor") || + segments.some((segment) => RUNTIMES.includes(segment)) || + RUNTIMES.includes(last) + ); +} + +/** + * Source with its comments removed. + * + * These modules describe in prose that they name no host, and a search of the + * whole file would find that description rather than a boundary crossing. + */ +function code(source: string): string { + let output = ""; + let index = 0; + while (index < source.length) { + const character = source[index]; + const following = source[index + 1]; + if (character === "/" && following === "/") { + while (index < source.length && source[index] !== "\n") { + index += 1; + } + continue; + } + if (character === "/" && following === "*") { + index += 2; + while (index < source.length && !(source[index] === "*" && source[index + 1] === "/")) { + index += 1; + } + index += 2; + continue; + } + if (character === '"' || character === "'" || character === "`") { + output += character; + index += 1; + while (index < source.length && source[index] !== character) { + if (source[index] === "\\") { + output += source[index]; + index += 1; + } + output += source[index]; + index += 1; + } + output += character; + index += 1; + continue; + } + output += character; + index += 1; + } + return output; +} + +/** + * What a module this file loads cannot be shown to be. + * + * A specifier the file computes names whatever it is handed, so no inspection + * of this surface can say it is not a host module. It is refused rather than + * skipped: a boundary that admits what it cannot read is not a boundary. + */ +const COMPUTED = "a computed module specifier"; + +/** + * Every module this source loads, read from the syntax rather than the text. + * + * Parsed, because module loading is not a pattern: a specifier can be a + * template literal, can escape its own characters, can be an expression, and + * the same characters can appear in a string that loads nothing. Each of those + * is a different answer, and only a parse tells them apart. + */ +function moduleSpecifiers(file: ts.SourceFile): string[] { + const found: string[] = []; + + function record(node: ts.Node | undefined): void { + if ( + node !== undefined && + (ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node)) + ) { + // The parser has already decoded escapes, so `node:crypto` and + // `node:crypto` arrive here as the same specifier. + found.push(node.text); + return; + } + found.push(COMPUTED); + } + + function visit(node: ts.Node): void { + if (ts.isImportDeclaration(node) || ts.isExportDeclaration(node)) { + if (node.moduleSpecifier !== undefined) { + record(node.moduleSpecifier); + } + } else if (ts.isImportEqualsDeclaration(node)) { + if (ts.isExternalModuleReference(node.moduleReference)) { + record(node.moduleReference.expression); + } + } else if (ts.isImportTypeNode(node)) { + record(ts.isLiteralTypeNode(node.argument) ? node.argument.literal : node.argument); + } else if (ts.isCallExpression(node)) { + const callee = node.expression; + if ( + callee.kind === ts.SyntaxKind.ImportKeyword || + (ts.isIdentifier(callee) && callee.text === "require") + ) { + record(node.arguments[0]); + } + } + ts.forEachChild(node, visit); + } + + visit(file); + return found; +} + +/** + * The scanned source, and a checker that knows what its names mean. + * + * `noLib` and `noResolve` are the point rather than an economy: nothing + * outside this file is loaded, so a name resolves only to what the file itself + * declares. Anything left unresolved is ambient — supplied by a host at + * runtime — which is exactly the question being asked. + */ +function parse(source: string): { file: ts.SourceFile; checker: ts.TypeChecker } { + const path = "/scanned.ts"; + const file = ts.createSourceFile(path, source, ts.ScriptTarget.Latest, true, ts.ScriptKind.TS); + const program = ts.createProgram({ + rootNames: [path], + options: { noLib: true, noResolve: true, target: ts.ScriptTarget.Latest }, + host: { + getSourceFile: (name) => (name === path ? file : undefined), + getDefaultLibFileName: () => "", + writeFile: () => {}, + getCurrentDirectory: () => "/", + getCanonicalFileName: (name) => name, + useCaseSensitiveFileNames: () => true, + getNewLine: () => "\n", + fileExists: (name) => name === path, + readFile: (name) => (name === path ? source : undefined), + }, + }); + return { file, checker: program.getTypeChecker() }; +} + +/** + * The slots in which the grammar writes a name rather than a reference. + * + * TypeScript spells the distinction structurally: an `IdentifierName` fills a + * `name`, `propertyName` or `label` slot of the node that owns it, and a + * qualified name's `right` is the same thing in type position. Everywhere else + * an identifier is an `IdentifierReference`. + */ +const LABEL_SLOTS = ["name", "propertyName", "label"]; + +/** + * Whether this identifier refers to a binding at all. + * + * Not a scope question — the checker answers those. This asks the grammar + * instead of listing the node kinds someone remembered: the member in + * `x.process`, the label in `break process`, the imported member in + * `{ Deno as portable }`, the key in `{ process: local }`, a named tuple + * element, an import attribute and every declaration's own name all fill a + * name slot, and none of them reads the name it spells. + * + * A shorthand property is the one name slot that is also a read, because + * `{ process }` declares a property and reads a binding with one identifier. + */ +function refers(node: ts.Identifier): boolean { + const parent = node.parent; + if (parent === undefined) { + return true; + } + if (ts.isShorthandPropertyAssignment(parent) && parent.name === node) { + return true; + } + if (ts.isQualifiedName(parent) && parent.right === node) { + return false; + } + return !LABEL_SLOTS.some((slot) => Reflect.get(parent, slot) === node); +} + +/** + * Host globals this source actually reads. + * + * A name is the host's only when nothing in this file declares it, and the + * checker is what knows that. Value scopes and type scopes, `var` hoisting, + * `import =`, `namespace`, mapped-type and `infer` type parameters, accessors + * and shadowing are the language's rules, not a list kept here — every one of + * them was a false positive while this was a list. + */ +function hostGlobals(parsed: { file: ts.SourceFile; checker: ts.TypeChecker }): string[] { + const found: string[] = []; + + /** + * The binding this identifier reads. + * + * `{ process }` writes one name in two roles: the property the literal + * declares and the value it reads. The ordinary symbol is the property — it + * is declared right there, so asking for it would answer that every host + * global is locally declared the moment it is put in an object. The value + * symbol is the one the shorthand refers to. + */ + function binding(node: ts.Identifier): ts.Symbol | undefined { + const parent = node.parent; + if (parent !== undefined && ts.isShorthandPropertyAssignment(parent) && parent.name === node) { + return parsed.checker.getShorthandAssignmentValueSymbol(parent); + } + return parsed.checker.getSymbolAtLocation(node); + } + + function visit(node: ts.Node): void { + if (ts.isIdentifier(node) && HOST_GLOBALS.includes(node.text) && refers(node)) { + const declared = (binding(node)?.declarations ?? []).some( + (declaration) => declaration.getSourceFile() === parsed.file, + ); + if (!declared && !found.includes(node.text)) { + found.push(node.text); + } + } + ts.forEachChild(node, visit); + } + + visit(parsed.file); + return found; +} + +function forbiddenNames(source: string): string[] { + const parsed = parse(source); + const scanned = code(source); + const crossings = FORBIDDEN.filter((name) => scanned.includes(name)); + for (const global of hostGlobals(parsed)) { + if (!crossings.includes(global)) { + crossings.push(global); + } + } + for (const specifier of moduleSpecifiers(parsed.file)) { + const refused = specifier === COMPUTED || hostModule(specifier); + if (refused && !crossings.includes(specifier)) { + crossings.push(specifier); + } + } + return crossings; +} + describe("Tier DLC — Workspace coordination selection", () => { it("DLC10: a missing Workspace provider fails before execution or publication", function* () { const stream = new InMemoryStream(); @@ -149,26 +470,227 @@ describe("Tier DLC — Workspace coordination selection", () => { expect(yieldEvents(stream.snapshot())).toHaveLength(1); }); - it("DLC13: shared Workspace coordination source stays runtime-neutral", function* () { - const sources = [ - yield* readTextFile(new URL("../src/workspace/api.ts", import.meta.url)), - yield* readTextFile(new URL("../src/workspace/effect.ts", import.meta.url)), - ]; - const forbidden = [ - "node:sqlite", + it("DLC13: the whole shared coordination surface stays runtime-neutral", function* () { + // The scanner has to be able to fail, and it has to read code rather than + // prose: these modules explain in their own comments that they name no + // host, and a substring search would find the explanation. + expect(code(`const value = "DOFS"; // Cloudflare\n/* SQLite */`)).toBe( + `const value = "DOFS"; \n`, + ); + expect(forbiddenNames(`import { DatabaseSync } from "node:sqlite";`)).toEqual([ "DatabaseSync", - "SQLite", - "Cloudflare", - "DOFS", - "savepoint", - "ConnectionGeneration", - "TransactionIdentity", - ]; - for (const source of sources) { - for (const name of forbidden) { - expect(source.includes(name)).toBe(false); + "sqlite", + "node:sqlite", + ]); + expect(forbiddenNames("// the Deno adapter owns DOFS and its savepoints")).toEqual([]); + + // A host module is a crossing by its shape, not because someone listed it. + // `node:crypto` names nothing else on this list, and shared source did + // import it while an earlier version of this test reported a clean + // boundary. + expect(forbiddenNames(`import { randomUUID } from "node:crypto";`)).toEqual(["node:crypto"]); + expect(forbiddenNames(`export { x } from "node:os";`)).toEqual(["node:os"]); + expect(forbiddenNames(`import "../deno.ts";`)).toEqual(["../deno.ts"]); + expect(forbiddenNames(`import type { X } from "node:fs";`)).toEqual(["node:fs"]); + expect(forbiddenNames(`type X = import("node:fs").Stats;`)).toEqual(["node:fs"]); + expect(forbiddenNames(`import fs = require("node:fs");`)).toEqual(["node:fs"]); + expect(forbiddenNames(`const m = await import("bun:sqlite");`)).toEqual([ + "sqlite", + "bun:sqlite", + ]); + + // How the specifier is spelled is not how it is found: a template literal + // and an escaped character load the same module as the plain string. + expect(forbiddenNames("const m = await import(`node:crypto`);")).toEqual(["node:crypto"]); + expect(forbiddenNames(`const m = await import("node\\u003acrypto");`)).toEqual(["node:crypto"]); + + // A destination this surface computes cannot be shown not to be a host + // module, so it is refused rather than skipped. + expect(forbiddenNames(`const target = "node:crypto";\nawait import(target);`)).toEqual([ + COMPUTED, + ]); + expect(forbiddenNames("await import(`${scheme}:crypto`);")).toEqual([COMPUTED]); + + // And text that only looks like module loading loads nothing. + expect(forbiddenNames("// run ids used to come from node:crypto")).toEqual([]); + expect(forbiddenNames(`const note = "node:crypto";`)).toEqual([]); + expect(forbiddenNames('const note = `import "node:crypto"`;')).toEqual([]); + expect(forbiddenNames(`const note = 'export { x } from "node:os"';`)).toEqual([]); + expect(forbiddenNames(`const hint = 'import from "@executablemd/workflow/deno"';`)).toEqual([]); + + // A host global is a crossing wherever the module came from. + expect(forbiddenNames("const pid = process.pid;")).toEqual(["process"]); + expect(forbiddenNames("export const p = process;")).toEqual(["process"]); + expect(forbiddenNames("const { env } = process;")).toEqual(["process"]); + expect(forbiddenNames("const home = Deno.cwd();")).toEqual(["Deno"]); + expect(forbiddenNames("const v = Bun.version;")).toEqual(["Bun"]); + expect(forbiddenNames("const b = Buffer.from([]);")).toEqual(["Buffer"]); + expect(forbiddenNames("const here = __dirname;")).toEqual(["__dirname"]); + expect(forbiddenNames("type F = Deno.FsFile;")).toEqual(["Deno"]); + + // A name is the host's only when nothing declared it. A parameter, an + // import and a local are all something else that happens to be spelled + // the same way. + expect( + forbiddenNames("function inspect(process: { pid: number }) { return process.pid; }"), + ).toEqual([]); + expect(forbiddenNames("const Buffer = 1;\nconst b = Buffer;")).toEqual([]); + expect(forbiddenNames(`import { Deno } from "./host.ts";\nconst c = Deno.cwd();`)).toEqual([]); + expect(forbiddenNames(`import Buffer from "./bytes.ts";\nconst b = Buffer.from([]);`)).toEqual( + [], + ); + expect(forbiddenNames("const { process } = deps;\nconst pid = process.pid;")).toEqual([]); + expect(forbiddenNames("try { run(); } catch (process) { report(process); }")).toEqual([]); + expect(forbiddenNames("[1].forEach((process) => report(process));")).toEqual([]); + + // Binding semantics, not a list of node shapes. An aliased member is a + // label on both sides, a type parameter binds its own scope, and `var` + // belongs to the function however deeply it is nested. + expect(forbiddenNames("const { process: local } = deps;")).toEqual([]); + expect(forbiddenNames(`import { Deno as portable } from "./host.ts";`)).toEqual([]); + expect(forbiddenNames(`export { process as runner } from "./host.ts";`)).toEqual([]); + expect(forbiddenNames("function read(value: Deno): Deno { return value; }")).toEqual([]); + expect(forbiddenNames("interface Holder { value: Buffer }")).toEqual([]); + expect( + forbiddenNames( + "function read() {\n if (ready) var process = portable;\n return process.pid;\n}", + ), + ).toEqual([]); + expect( + forbiddenNames( + "function read() {\n for (var Buffer of list) use(Buffer);\n return Buffer;\n}", + ), + ).toEqual([]); + + // The language's own resolution, not a catalogue of declaration shapes: + // `import =`, a namespace, a mapped-type parameter, an `infer` parameter, + // a statement label and an accessor member each bind or label the name + // without any rule about them being written here. + expect(forbiddenNames(`import Deno = require("./portable.ts");\nDeno.cwd();`)).toEqual([]); + expect( + forbiddenNames('namespace Deno {\n export const cwd = () => "";\n}\nDeno.cwd();'), + ).toEqual([]); + expect(forbiddenNames("type Rename = { [process in keyof T]: T[process] };")).toEqual([]); + expect(forbiddenNames("type Value = T extends infer Buffer ? Buffer : never;")).toEqual([]); + expect(forbiddenNames("process: for (;;) {\n break process;\n}")).toEqual([]); + expect(forbiddenNames("class Queue {\n get process() {\n return 1;\n }\n}")).toEqual([]); + + // A shorthand property writes one name in two roles. The property it + // declares is not the binding it reads, and reading an ambient global is + // a crossing however briefly the value is held. + expect(forbiddenNames("const environment = { process };")).toEqual(["process"]); + expect(forbiddenNames("const runtimes = { Deno, Bun };")).toEqual(["Deno", "Bun"]); + expect(forbiddenNames("const process = 1;\nconst environment = { process };")).toEqual([]); + expect(forbiddenNames("function hold(Buffer: number) {\n return { Buffer };\n}")).toEqual([]); + expect(forbiddenNames("const environment = { process: local };")).toEqual([]); + + // A name slot is a name slot wherever the grammar puts one: a named tuple + // element and an import attribute key are labels, and the specifier beside + // the attribute is still read as a module. + expect(forbiddenNames("type Pair = [process: string, Deno?: number];")).toEqual([]); + expect( + forbiddenNames('import data from "./portable.json" with {\n process: "portable",\n};'), + ).toEqual([]); + expect(forbiddenNames("type Pair = [value: typeof process];")).toEqual(["process"]); + expect(forbiddenNames('import data from "node:fs" with {\n process: "portable",\n};')).toEqual( + ["node:fs"], + ); + expect(forbiddenNames("enum Kind {\n process,\n}")).toEqual([]); + expect(forbiddenNames("interface Host {\n process: number;\n}")).toEqual([]); + + // The same forms still end at their own boundary. + expect( + forbiddenNames("function read(value: Deno): Deno { return value; }\nDeno.cwd();"), + ).toEqual(["Deno"]); + expect( + forbiddenNames("function read() {\n var process = portable;\n}\nconst pid = process.pid;"), + ).toEqual(["process"]); + + // Shadowing is lexical, so it ends where its scope does. + expect( + forbiddenNames( + "function inner(process: unknown) { return process; }\nconst pid = process.pid;", + ), + ).toEqual(["process"]); + expect( + forbiddenNames("{\n const Deno = 1;\n use(Deno);\n}\nconst home = Deno.cwd();"), + ).toEqual(["Deno"]); + + // Every runtime this repository names an entry point after, not only the + // one whose adapter exists today. + expect(forbiddenNames(`import "./node.ts";`)).toEqual(["./node.ts"]); + expect(forbiddenNames(`import "./bun.ts";`)).toEqual(["./bun.ts"]); + expect(forbiddenNames(`import "./compiled.ts";`)).toEqual(["./compiled.ts"]); + expect(forbiddenNames(`import "../src/node/journal.ts";`)).toEqual(["../src/node/journal.ts"]); + expect(forbiddenNames(`import "../src/cloudflare/storage.ts";`)).toEqual([ + "../src/cloudflare/storage.ts", + ]); + expect(forbiddenNames(`import "../../vendor/store/mod.ts";`)).toEqual([ + "../../vendor/store/mod.ts", + ]); + + // Positive controls: a word is not a host because a host's name is inside + // it, and neither is a path segment. + expect(forbiddenNames("const preprocessor = 1;")).toEqual([]); + expect(forbiddenNames("const bundle = 1;\nclass Denominator {}")).toEqual([]); + expect(forbiddenNames("const x = { process: 1 };\nconst y = x.process;")).toEqual([]); + expect(forbiddenNames("queue.process(job);")).toEqual([]); + expect(forbiddenNames(`import "./nodes.ts";`)).toEqual([]); + expect(forbiddenNames(`import "./bundle.ts";`)).toEqual([]); + expect(forbiddenNames(`import "../vendors/helper.ts";`)).toEqual([]); + expect(forbiddenNames(`import "@executablemd/runtime";`)).toEqual([]); + expect(forbiddenNames("const id = crypto.randomUUID();")).toEqual([]); + + const found = (yield* glob({ + root: REPOSITORY, + patterns: [ + "packages/workflow/mod.ts", + "packages/workflow/src/**/*.ts", + "packages/durable-streams/*.ts", + ], + // Whole packages rather than named modules, so a coordination module + // added later is covered without this list being remembered. The single + // exception carries its reason: the HTTP stream is a client for a remote + // durable stream and reaches the platform's own `fetch`. + exclude: ["packages/workflow/src/deno/**", "packages/durable-streams/http-stream.ts"], + })) + .map((entry) => entry.path) + .sort(); + + // A pattern that matched nothing would report a clean boundary, so the + // surface every Workspace effect actually crosses is named here. + expect(found).toEqual( + expect.arrayContaining([ + "packages/durable-streams/durability.ts", + "packages/durable-streams/effect.ts", + "packages/durable-streams/guard.ts", + "packages/durable-streams/live-coordinator.ts", + "packages/durable-streams/types.ts", + "packages/workflow/mod.ts", + "packages/workflow/src/storage/api.ts", + "packages/workflow/src/workspace/api.ts", + "packages/workflow/src/workspace/effect.ts", + ]), + ); + expect(found.some((path) => path.includes("/src/deno/"))).toBe(false); + + const crossings: Record = {}; + const unread: string[] = []; + for (const path of found) { + const source = yield* readTextFile(join(REPOSITORY, path)); + // A parse that failed would report every file as clean. A module whose + // text imports something must yield a specifier, or this scan is reading + // nothing and saying so approvingly. + if (/^import\s/m.test(source) && moduleSpecifiers(parse(source).file).length === 0) { + unread.push(path); + } + const names = forbiddenNames(source); + if (names.length > 0) { + crossings[path] = names; } } + expect(unread).toEqual([]); + expect(crossings).toEqual({}); }); it("DLC15: live Workspace invocation authority is one-shot", function* () { diff --git a/scripts/runtime-test-exclusions.ts b/scripts/runtime-test-exclusions.ts index 8f17b6a2..a1f35802 100644 --- a/scripts/runtime-test-exclusions.ts +++ b/scripts/runtime-test-exclusions.ts @@ -132,6 +132,12 @@ const DENO_ONLY_TOOLING: RuntimeExclusion[] = [ "creates and imports a physical temporary copy of the Workspace modules through Deno.makeTempDir and the Deno module loader", issue: "https://github.com/taras/executable.md/issues/365", }, + { + path: "packages/workflow/tests/workspace-crash-recovery.test.ts", + reason: + "kills real Deno child processes with SIGKILL and reads the recovered node:sqlite WorkflowRun database they leave behind; the children run under the Deno executable and node:sqlite remains behind --experimental-sqlite on Node 22", + issue: "https://github.com/taras/executable.md/issues/365", + }, ]; /** diff --git a/specs/executable-mdx-spec.md b/specs/executable-mdx-spec.md index 81c4616d..b953be4a 100644 --- a/specs/executable-mdx-spec.md +++ b/specs/executable-mdx-spec.md @@ -7018,7 +7018,7 @@ Defined in [Workflow runs](./workflow-spec.md) §9.5–§9.6. | DLC10 | Fail-closed Workspace | A missing Workspace provider activates fail-stop before execution or publication and persists no Yield or Close | | DLC11 | Workspace isolation | Replaceable context carries only provider routing; the selected provider directly invokes the credentialed execution-owned capability, while unrelated durable operations stay on the default coordinator | | DLC12 | Workspace replay | Replayed Workspace operations require no live provider | -| DLC13 | Runtime-neutral boundary | Shared Workspace coordination source exposes no runtime or storage implementation type | +| DLC13 | Runtime-neutral boundary | No module of the shared coordination surface names a storage, connection, savepoint or transaction-token type outside its own prose, reads a host global, or loads a module only one host can resolve. Host globals (`process`, `Deno`, `Bun`, `Buffer`, `globalThis`, `navigator`, `__dirname`, `__filename`) are recognized as parsed identifier references, so a word containing one and a property of that name are not crossings, while cross-runtime Web APIs such as `crypto` are never crossings. Module loading is read from parsed syntax — static imports and re-exports, `import type`, type-position `import()`, dynamic `import()`, `import =` and `require()` — with quoted and no-substitution template specifiers decoded, so comments and strings that merely contain import syntax load nothing. A destination the surface computes cannot be shown not to be a host module and is refused. Host schemes (`node:`, `bun:`, `deno:`, `cloudflare:`, `workerd:`), whole path segments naming any runtime this repository builds an entry point for (`deno`, `node`, `bun`, `compiled`, `cloudflare`, `workerd`), vendored sources and host process modules are classified by shape rather than by an enumerated list | | DLC14 | Provider infrastructure failure | A selected coordinator activates one first failure by identity and fences later execution and publication | | DLC15 | One-shot Workspace invocation | A provider can use the execution-owned invocation authority only during its original call; retained execution, publication and failure operations are refused after completion | | DLC16 | Loaded-copy Workspace selection | A provider installed by one physical package copy coordinates one operation created by another copy exactly once without sharing authority through context or a module registry; substituted selection and retained authority remain fail-closed | @@ -7060,6 +7060,9 @@ workspaces](./workflow-workspace-spec.md) §13. | WAC21 | Selection refusal | A substituted or foreign provider selection is rejected before transaction or savepoint work and leaves no mutation, retained root, pointer change, Yield or Close | | WAC22 | Minimum-priority publication isolation | A minimum-priority same-named handler cannot observe or acknowledge publication; the real mutation, retained root, current pointer and filtered Yield commit together | | WAC23 | Minimum-priority failure isolation | A minimum-priority same-named handler cannot replace infrastructure-failure activation; the exact first failure rolls back mutation, roots and journal and fences later work | +| WAC24 | Real host crash | A `SIGKILL` while the mutation, immutable root, current-root pointer and routed journal row are written and uncommitted leaves a second connection seeing only the baseline, and a fresh process recovers the baseline filesystem, root, retained counts and journal exactly | +| WAC25 | Committed restart | A second process reopens a run whose Workspace effects committed in a process that has ended, observes the same filesystem, current root, ordered events, event identities and event-to-root associations, and performs no recorded effect again | +| WAC26 | Historical reconstruction | A fresh process selects an older event's root through the adapter-private materializer, invalidates the authoritative negative resolution, rebuilds its exact topology, bytes, modes, hardlinks and symbolic links from that root's retained DOFS content, and resnapshots to the selected identity | ### Tier WTX — WorkflowRun savepoints and transaction authority diff --git a/specs/workflow-spec.md b/specs/workflow-spec.md index 93ed93c6..f72d75bf 100644 --- a/specs/workflow-spec.md +++ b/specs/workflow-spec.md @@ -509,6 +509,25 @@ That identity fences later coordinators, executors and appends. An already active durability failure takes precedence. Cancellation at any phase publishes nothing. +Losing the host is not one of those phases. Nothing runs after the process +dies: no cleanup, no commit, no rollback. The operating system closes the +connection and releases the locks it held, and the next connection to open the +database recovers it. The mutation, the immutable root, the current-root +pointer and the routed journal row have all been written inside the +caller-owned transaction by then; recovery exposes the last committed state and +none of them. + +A second connection sees that same last committed state for as long as the +writer's transaction is uncommitted, so a crash publishes nothing that was not +already visible before it. + +A later process therefore opens the last committed state and nothing else: the +same live filesystem, the same current root, the same retained roots, manifest +and blob references, and the same ordered journal with the same event +identities and Workspace-root associations. Recorded effects replay rather than +execute again, and the private restoration materializer reconstructs any +retained root from that state without the process that wrote it. + The provider-neutral coordinator receives the failure-activation continuation needed for this boundary. The default live coordinator ignores it and preserves ordinary success/failure publication. Replay bypasses coordination, and only an diff --git a/specs/workflow-workspace-spec.md b/specs/workflow-workspace-spec.md index e046613f..d5ece6cb 100644 --- a/specs/workflow-workspace-spec.md +++ b/specs/workflow-workspace-spec.md @@ -690,7 +690,12 @@ BEGIN COMMIT ``` -A crash commits all three or none. Nested child effects finish before the +A crash commits all three or none. Nothing runs after a killed host dies — no +cleanup, no commit, no rollback — so the operating system closes its connection +and releases its locks, and the next connection recovers the database to the +last committed state, exposing neither the mutation, the root, the pointer +change nor the result. Another connection never sees them while that +transaction is uncommitted either. Nested child effects finish before the parent's effect transaction begins. Direct filesystem operations and declarative Git operations use this boundary. @@ -841,6 +846,14 @@ fail-stop fence for infrastructure failures. This foundation is not a public filesystem effect: ``, `API.Files`, workflow start/resume and history commands do not route to it in this slice. +That boundary holds across processes as well as within one. A host killed +between the mutation and the commit publishes nothing, and a process that +opens the database afterwards finds the last committed filesystem, current +root, retained roots and ordered event-to-root associations. Reopening replays +recorded effects instead of performing them, and the adapter-private +materializer reconstructs an older event's root from that root's retained DOFS +manifests and blobs. + SQLite is a host implementation detail. The CLI deliberately exposes no remote host-selection option yet, while retaining a control surface that can be delegated without changing the document language.