Skip to content

Commit 32ce912

Browse files
os-zhuangCopilot
andcommitted
feat(metadata-fs): FileSystemRepository with chokidar watch + JSONL log (ADR-0008 M0 PR-4)
New package @objectstack/metadata-fs: - repository.ts: FileSystemRepository implementing the M0 contract against a <root>/<type>/<name>.json + JSONL log layout. Atomic writes (tmpfile + rename), heads/seq recovered from log on start, chokidar watch translates external edits to events with source='fs', 200ms self-write suppression window. - layout.ts: path helpers + parseItemPath(). - jsonl-log.ts: append-only log reader/writer with corrupt-line skip. - sync.ts: KeyedMutex (per-key serialization) + event broker. - watch-iterable.ts: manual AsyncIterator (reusable from in-memory pkg). - 17 contract tests + 5 fs-specific tests pass. metadata-core: - Add singleBranch option to contract suite (FS is one branch per repo). - tsup splitting: true so testing.js shares one ConflictError class identity with index.js (was double-bundled, breaking instanceof). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 4150fe4 commit 32ce912

17 files changed

Lines changed: 1189 additions & 1 deletion

.changeset/metadata-fs-init.md

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
---
2+
'@objectstack/metadata-fs': minor
3+
'@objectstack/metadata-core': patch
4+
---
5+
6+
Add `@objectstack/metadata-fs` — Node-only `FileSystemRepository`
7+
implementation of the M0 Repository contract.
8+
9+
Layout:
10+
11+
```
12+
<root>/
13+
<type>/<name>.json # canonical body (atomic rename writes)
14+
.objectstack/.log/<branch>.jsonl # append-only change log
15+
```
16+
17+
Features:
18+
19+
- All 17 contract tests pass (`singleBranch: true`).
20+
- Per-key serialization via `KeyedMutex`.
21+
- Atomic writes via tmpfile + rename.
22+
- Heads and `seq` recovered from the JSONL log on `start()` — survives
23+
process restart.
24+
- chokidar watcher translates external edits (e.g. VSCode saves) into
25+
`MetadataEvent`s with `source: 'fs'`.
26+
- Self-write suppression: 200ms window prevents the watcher from
27+
re-emitting events for files we wrote ourselves.
28+
- Manual `AsyncIterator` for `watch()` to mirror the in-memory pattern.
29+
30+
Also (`metadata-core`):
31+
32+
- Add `singleBranch` option to `runRepositoryContractTests` so
33+
single-branch backends (like the FS one) skip the cross-branch test.
34+
- Switch tsup `splitting: true` so `index.js` and `testing.js` share a
35+
single `ConflictError` class identity (was double-bundled before).
36+
37+
See ADR-0008 §10 PR-4.

packages/metadata-core/src/contract-suite.ts

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,12 @@ import { ConflictError } from './errors.js';
2727
export interface ContractSuiteOptions {
2828
/** If the implementation supports `version`-pinned reads, set true. */
2929
supportsVersionedReads?: boolean;
30+
/**
31+
* Set true for backends that are scoped to a single branch (e.g. the
32+
* FileSystemRepository is one branch per instance). The cross-branch
33+
* test is skipped.
34+
*/
35+
singleBranch?: boolean;
3036
}
3137

3238
const refOf = (overrides: Partial<MetaRef> = {}): MetaRef => ({
@@ -171,6 +177,7 @@ export function runRepositoryContractTests(
171177
});
172178

173179
it('different branches have independent sequences', async () => {
180+
if (opts.singleBranch) return;
174181
const repo = await factory();
175182
const mainRef = refOf({ branch: 'main' });
176183
const devRef = refOf({ branch: 'dev' });

packages/metadata-core/tsup.config.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import { defineConfig } from 'tsup';
44

55
export default defineConfig({
66
entry: ['src/index.ts', 'src/testing.ts'],
7-
splitting: false,
7+
splitting: true,
88
sourcemap: true,
99
clean: true,
1010
dts: true,

packages/metadata-fs/README.md

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
# @objectstack/metadata-fs
2+
3+
`FileSystemRepository` — Node-only implementation of the
4+
`MetadataRepository` contract defined in `@objectstack/metadata-core`.
5+
6+
## Layout on disk
7+
8+
```
9+
<root>/
10+
<type>/
11+
<name>.json # canonical body of the item
12+
.objectstack/
13+
.log/
14+
<branch>.jsonl # append-only change log (one JSON object per line)
15+
```
16+
17+
For example:
18+
19+
```
20+
metadata/
21+
view/
22+
case_grid.json
23+
case_timeline.json
24+
object/
25+
case.json
26+
.objectstack/.log/main.jsonl
27+
```
28+
29+
## Usage
30+
31+
```ts
32+
import { FileSystemRepository } from '@objectstack/metadata-fs';
33+
34+
const repo = new FileSystemRepository({
35+
root: './metadata',
36+
org: 'system',
37+
project: 'crm',
38+
branch: 'main',
39+
});
40+
await repo.start(); // scan + open watcher
41+
42+
const view = await repo.get({
43+
org: 'system', project: 'crm', branch: 'main',
44+
type: 'view', name: 'case_grid',
45+
});
46+
47+
for await (const evt of repo.watch({})) {
48+
console.log('changed', evt.ref, evt.hash);
49+
}
50+
```
51+
52+
See ADR-0008 §10 PR-4.

packages/metadata-fs/package.json

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
{
2+
"name": "@objectstack/metadata-fs",
3+
"version": "0.1.0",
4+
"license": "Apache-2.0",
5+
"description": "FileSystemRepository: Node-only Repository implementation backed by JSON files and a JSONL change log (ADR-0008).",
6+
"type": "module",
7+
"main": "dist/index.js",
8+
"types": "dist/index.d.ts",
9+
"exports": {
10+
".": {
11+
"types": "./dist/index.d.ts",
12+
"import": "./dist/index.js",
13+
"require": "./dist/index.cjs"
14+
}
15+
},
16+
"files": [
17+
"dist",
18+
"README.md"
19+
],
20+
"scripts": {
21+
"build": "tsup",
22+
"dev": "tsc --watch",
23+
"clean": "rm -rf dist",
24+
"test": "vitest run",
25+
"test:watch": "vitest"
26+
},
27+
"keywords": [
28+
"objectstack",
29+
"metadata",
30+
"filesystem",
31+
"repository"
32+
],
33+
"dependencies": {
34+
"@objectstack/metadata-core": "workspace:*",
35+
"chokidar": "^5.0.0"
36+
},
37+
"devDependencies": {
38+
"@types/node": "^25.9.1",
39+
"tsup": "^8.5.1",
40+
"typescript": "^6.0.3",
41+
"vitest": "^4.1.7"
42+
},
43+
"author": "ObjectStack",
44+
"repository": {
45+
"type": "git",
46+
"url": "https://github.com/objectstack-ai/framework.git",
47+
"directory": "packages/metadata-fs"
48+
},
49+
"homepage": "https://objectstack.ai/docs",
50+
"bugs": "https://github.com/objectstack-ai/framework/issues",
51+
"publishConfig": {
52+
"access": "public"
53+
},
54+
"engines": {
55+
"node": ">=18.0.0"
56+
}
57+
}

packages/metadata-fs/src/index.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
2+
3+
export * from './repository.js';
4+
export { JsonlLog } from './jsonl-log.js';
5+
export type { FsLayout } from './layout.js';
Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
2+
3+
/**
4+
* Append-only JSONL change log writer / reader. Each line is a single
5+
* `MetadataEvent` serialized via `JSON.stringify`.
6+
*
7+
* Durability strategy
8+
* ───────────────────
9+
* - Append with `O_APPEND` semantics (Node's `fs.appendFile` is
10+
* atomic for sub-PIPE_BUF-sized writes; events are well under 4 KiB).
11+
* - Read by streaming the file line-by-line and JSON.parse-ing each.
12+
* - On a corrupt line we skip and continue — the body files are the
13+
* source of truth; the log is a denormalised history index.
14+
*/
15+
16+
import fs from 'node:fs/promises';
17+
import path from 'node:path';
18+
import readline from 'node:readline';
19+
import { createReadStream, existsSync } from 'node:fs';
20+
import type { MetadataEvent } from '@objectstack/metadata-core';
21+
22+
export class JsonlLog {
23+
constructor(private readonly file: string) {}
24+
25+
async append(evt: MetadataEvent): Promise<void> {
26+
await fs.mkdir(path.dirname(this.file), { recursive: true });
27+
await fs.appendFile(this.file, JSON.stringify(evt) + '\n', 'utf8');
28+
}
29+
30+
/** Read all events in seq order (i.e. file order). */
31+
async *readAll(): AsyncIterable<MetadataEvent> {
32+
if (!existsSync(this.file)) return;
33+
const rl = readline.createInterface({
34+
input: createReadStream(this.file, { encoding: 'utf8' }),
35+
crlfDelay: Infinity,
36+
});
37+
try {
38+
for await (const line of rl) {
39+
if (!line.trim()) continue;
40+
try {
41+
yield JSON.parse(line) as MetadataEvent;
42+
} catch {
43+
// Skip corrupt line.
44+
}
45+
}
46+
} finally {
47+
rl.close();
48+
}
49+
}
50+
51+
/** Return the highest seq number in the log, or 0 if empty. */
52+
async highestSeq(): Promise<number> {
53+
let max = 0;
54+
for await (const evt of this.readAll()) {
55+
if (typeof evt.seq === 'number' && evt.seq > max) max = evt.seq;
56+
}
57+
return max;
58+
}
59+
}

packages/metadata-fs/src/layout.ts

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
2+
3+
/**
4+
* Disk layout helpers — see ADR-0008 §10 PR-4 / packages/metadata-fs README.
5+
*
6+
* <root>/<type>/<name>.json — canonical body
7+
* <root>/.objectstack/.log/<branch>.jsonl — append-only change log
8+
*/
9+
10+
import path from 'node:path';
11+
import type { MetadataType } from '@objectstack/metadata-core';
12+
13+
export interface FsLayout {
14+
/** Absolute path to the metadata root. */
15+
root: string;
16+
/** Branch name (e.g. "main"). */
17+
branch: string;
18+
}
19+
20+
export function itemPath(layout: FsLayout, type: MetadataType, name: string): string {
21+
return path.join(layout.root, type, `${name}.json`);
22+
}
23+
24+
export function typeDir(layout: FsLayout, type: MetadataType): string {
25+
return path.join(layout.root, type);
26+
}
27+
28+
export function logDir(layout: FsLayout): string {
29+
return path.join(layout.root, '.objectstack', '.log');
30+
}
31+
32+
export function logFile(layout: FsLayout): string {
33+
return path.join(logDir(layout), `${layout.branch}.jsonl`);
34+
}
35+
36+
/** Parse a path like ".../view/case_grid.json" into {type, name}. */
37+
export function parseItemPath(
38+
layout: FsLayout,
39+
absPath: string,
40+
): { type: string; name: string } | null {
41+
const rel = path.relative(layout.root, absPath);
42+
if (rel.startsWith('..') || rel.startsWith('.objectstack')) return null;
43+
const segments = rel.split(path.sep);
44+
if (segments.length !== 2) return null;
45+
const type = segments[0]!;
46+
const file = segments[1]!;
47+
if (!file.endsWith('.json')) return null;
48+
const name = file.slice(0, -'.json'.length);
49+
return { type, name };
50+
}

0 commit comments

Comments
 (0)