From 6757a8749bdfd7197db86b8bf00cece866667afe Mon Sep 17 00:00:00 2001 From: Rinse Date: Fri, 7 Aug 2026 12:25:07 +0000 Subject: [PATCH 1/2] feat(daemon): periodic IPFS repo GC over kubo RPC, upgrade kubo 0.43.0 + pkc-js 0.0.77 Upgrades kubo 0.42.0 -> 0.43.0 and @pkcprotocol/pkc-js 0.0.73 -> 0.0.77, and adds a periodic repo GC so a long-running daemon reclaims unpinned blocks. Without it one production node reached ~190GB against a 10GB StorageMax and exhausted disk and inodes. GC is driven over the kubo RPC API on our own schedule rather than via kubo's --enable-gc daemon flag. A daemon started with --enable-gc never exits in response to POST /api/v0/shutdown, lingering as a half-shutdown zombie (SIGTERM still works). pkc-js POSTs that endpoint when it rewrites the kubo Routing config on first connect and relies on the daemon restarting kubo, so the flag wedges the supervision loop -- it fails 4 kubo restart tests deterministically. Reported upstream as ipfs/kubo#11424; reproduces on 0.24.0, 0.42.0 and 0.43.0, which is also why the earlier attempt at the flag (f228d7d) was reverted two days later. The scheduler mirrors kubo's own policy: check hourly, GC only once the repo passes 90% of Datastore.StorageMax, and use size-only repo stats so it does not walk the entire flatfs blockstore. It is single-flighted, its timer is unref'd, and errors are contained inside the tick so a rejected interval callback cannot take the daemon down. This also covers a gap in pkc-js, which only GCs from a started local community's IPNS sync -- a daemon that is up with no community started would otherwise never reclaim anything. New flags: --enableIpfsGc / --no-enableIpfsGc (default on) and --ipfsGcIntervalMinutes (default 60). Datastore.StorageMax is left at kubo's 10GB default. --- README.md | 30 +++--- package-lock.json | 160 +++++++++++++++++++++++++++-- package.json | 4 +- src/cli/commands/daemon.ts | 48 +++++++-- src/ipfs/repoGc.ts | 188 ++++++++++++++++++++++++++++++++++ test/kubo/repoGc.test.ts | 204 +++++++++++++++++++++++++++++++++++++ 6 files changed, 604 insertions(+), 30 deletions(-) create mode 100644 src/ipfs/repoGc.ts create mode 100644 test/kubo/repoGc.test.ts diff --git a/README.md b/README.md index 6dde810..bcc9878 100644 --- a/README.md +++ b/README.md @@ -872,20 +872,26 @@ Run a network-connected Bitsocial node. Once the daemon is running you can creat ``` USAGE - $ bitsocial daemon --pkcRpcUrl --logPath [--chainProviderUrls ...] - [--allowPrivateKeyExport] + $ bitsocial daemon --pkcRpcUrl --logPath [--chainProviderUrls ...] [--enableIpfsGc] + [--ipfsGcIntervalMinutes ] [--allowPrivateKeyExport] FLAGS - --[no-]allowPrivateKeyExport Allow RPC clients to request community exports that include the community signer's - private key (`bitsocial community export --includePrivateKey`). Disable with - --no-allowPrivateKeyExport when exposing the RPC to untrusted clients - --chainProviderUrls=... [default: - https://eth.drpc.org,https://ethereum.publicnode.com,https://ethereum-rpc.publicnode.c - om,https://rpc.mevblocker.io,https://1rpc.io/eth,https://eth-pokt.nodies.app] RPC - URL(s) for .bso name resolution. Can be specified multiple times. - --logPath= (required) [default: /home/runner/.local/state/bitsocial] Specify a directory which - will be used to store logs - --pkcRpcUrl= (required) [default: ws://localhost:9138/] Specify PKC RPC URL to listen on + --[no-]allowPrivateKeyExport Allow RPC clients to request community exports that include the community signer's + private key (`bitsocial community export --includePrivateKey`). Disable with + --no-allowPrivateKeyExport when exposing the RPC to untrusted clients + --chainProviderUrls=... [default: + https://eth.drpc.org,https://ethereum.publicnode.com,https://ethereum-rpc.publicnode. + com,https://rpc.mevblocker.io,https://1rpc.io/eth,https://eth-pokt.nodies.app] RPC + URL(s) for .bso name resolution. Can be specified multiple times. + --[no-]enableIpfsGc Periodically garbage-collect the IPFS repo over the kubo RPC API while the daemon is + up. GC only runs once the repo passes 90% of Datastore.StorageMax (default 10GB), and + only reclaims unpinned blocks — pinned data and MFS are never collected. Disable with + --no-enableIpfsGc + --ipfsGcIntervalMinutes= [default: 60] How often to check whether the IPFS repo needs garbage collection, in + minutes + --logPath= (required) [default: /home/runner/.local/state/bitsocial] Specify a directory which + will be used to store logs + --pkcRpcUrl= (required) [default: ws://localhost:9138/] Specify PKC RPC URL to listen on DESCRIPTION Run a network-connected Bitsocial node. Once the daemon is running you can create and start your communities and diff --git a/package-lock.json b/package-lock.json index 0f0ee98..a1e8190 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,13 +16,13 @@ "@oclif/plugin-help": "6.2.36", "@oclif/plugin-not-found": "3.2.73", "@oclif/table": "0.5.1", - "@pkcprotocol/pkc-js": "0.0.73", + "@pkcprotocol/pkc-js": "0.0.77", "dataobject-parser": "1.2.22", "decompress": "4.2.1", "env-paths": "2.2.1", "exit-hook": "4.0.0", "express": "4.19.2", - "kubo": "0.42.0", + "kubo": "0.43.0", "p-limit": "7.3.0", "strip-json-comments": "5.0.3", "tcp-port-used": "1.0.2", @@ -5850,9 +5850,9 @@ } }, "node_modules/@pkcprotocol/pkc-js": { - "version": "0.0.73", - "resolved": "https://registry.npmjs.org/@pkcprotocol/pkc-js/-/pkc-js-0.0.73.tgz", - "integrity": "sha512-c1XfXLug6P7mG1G5JH/k+4lZCSBt8lAfAzCmO2RTZGll59NLPDSGQtjwOx4KLldWNeyWS9kFaG00qKOS1UkCFg==", + "version": "0.0.77", + "resolved": "https://registry.npmjs.org/@pkcprotocol/pkc-js/-/pkc-js-0.0.77.tgz", + "integrity": "sha512-Tl9LGf14BGfQCZJowtTNMI722MtRwEqmFmeVuhnI6Er6mj48Ap2gVPQvF3nDK5fWvFxes5dDsLfUVFz0njH3UQ==", "license": "GPL-3.0-or-later", "dependencies": { "@enhances/with-resolvers": "0.0.5", @@ -5873,6 +5873,8 @@ "assert": "2.1.0", "better-sqlite3": "12.9.0", "blockstore-core": "7.0.1", + "blockstore-fs": "^4.0.1", + "blockstore-idb": "^4.0.1", "buffer": "6.0.3", "cbor": "10.0.11", "cborg": "4.5.8", @@ -9784,6 +9786,130 @@ "integrity": "sha512-Gvqj6MO4GMLnFdtE68HZRpGBskNC+9+GQ+JevTGNYLyhjUuPhjDLU3jN1LpBemXJDW1bRSkczqA/qGyKlPKrcQ==", "license": "Apache-2.0 OR MIT" }, + "node_modules/blockstore-fs": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/blockstore-fs/-/blockstore-fs-4.0.1.tgz", + "integrity": "sha512-UXTG85jT+b4xumiHwqyZBnAqfxCqGgGl5ZszSGpnje7yEIrv2pO3NzttAz9DHwzaZxE2nBPMaPlg+SJ8VGlPrg==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "abort-error": "^1.0.2", + "interface-blockstore": "^7.0.0", + "interface-store": "^8.0.0", + "it-glob": "^3.0.4", + "it-map": "^3.1.4", + "it-parallel-batch": "^3.0.9", + "multiformats": "^14.0.0", + "race-signal": "^2.0.0", + "steno": "^4.0.2" + } + }, + "node_modules/blockstore-fs/node_modules/interface-blockstore": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/interface-blockstore/-/interface-blockstore-7.0.1.tgz", + "integrity": "sha512-a3NsgRXpRppVWtQnJjmZ9jxvjcttY1+WNWqHFWwgKhRcNMMGwWGtFNYpQmcZ+mEnIt5NTJVYzNwLCcALMWPNsA==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "interface-store": "^8.0.0", + "multiformats": "^14.0.0" + } + }, + "node_modules/blockstore-fs/node_modules/interface-store": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/interface-store/-/interface-store-8.0.0.tgz", + "integrity": "sha512-e2+s3EEROzM+Wlas4hU3zveTUscvVMf1BOvdsJfpzFm19SoEXLVadpACjWOnM491HqGpvtfFnevyiaN8W+I6Eg==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "abort-error": "^1.0.2" + } + }, + "node_modules/blockstore-fs/node_modules/multiformats": { + "version": "14.0.5", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-14.0.5.tgz", + "integrity": "sha512-vbIm83F2yZ1pWJGS0yl0ysracIvv56LtbrIyiIQHoLdYDJOMoLfVFsXhh9DUH4SFdkdkFhucyWniihsNzVEjkQ==", + "license": "Apache-2.0 OR MIT" + }, + "node_modules/blockstore-idb": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/blockstore-idb/-/blockstore-idb-4.0.1.tgz", + "integrity": "sha512-ZfqKiJZ7wCokOrWsKtGULypIEYH+58mS/jvLk43wDMMjvNxAUPobzblzGtf+uDEE3qpYELTqjeBbS+5YCGUwFA==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "abort-error": "^1.0.2", + "blockstore-core": "^7.0.0", + "idb": "^8.0.3", + "interface-blockstore": "^7.0.0", + "interface-store": "^8.0.0", + "it-all": "^3.0.9", + "it-to-buffer": "^5.0.0", + "multiformats": "^14.0.0", + "race-signal": "^2.0.0" + } + }, + "node_modules/blockstore-idb/node_modules/blockstore-core": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/blockstore-core/-/blockstore-core-7.0.1.tgz", + "integrity": "sha512-HgLzdH8oL6DrMC6J7snxmV0TXzJ/BMPSbskH4v7BBuUO9foDBOMEKUfoMX9yoBpxbH+r3Z9afXHDgXlTifMxHQ==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "@libp2p/logger": "^6.2.4", + "abort-error": "^1.0.2", + "interface-blockstore": "^7.0.0", + "interface-store": "^8.0.0", + "it-all": "^3.0.9", + "it-filter": "^3.1.4", + "it-merge": "^3.0.12", + "multiformats": "^14.0.0" + } + }, + "node_modules/blockstore-idb/node_modules/interface-blockstore": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/interface-blockstore/-/interface-blockstore-7.0.1.tgz", + "integrity": "sha512-a3NsgRXpRppVWtQnJjmZ9jxvjcttY1+WNWqHFWwgKhRcNMMGwWGtFNYpQmcZ+mEnIt5NTJVYzNwLCcALMWPNsA==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "interface-store": "^8.0.0", + "multiformats": "^14.0.0" + } + }, + "node_modules/blockstore-idb/node_modules/interface-store": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/interface-store/-/interface-store-8.0.0.tgz", + "integrity": "sha512-e2+s3EEROzM+Wlas4hU3zveTUscvVMf1BOvdsJfpzFm19SoEXLVadpACjWOnM491HqGpvtfFnevyiaN8W+I6Eg==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "abort-error": "^1.0.2" + } + }, + "node_modules/blockstore-idb/node_modules/it-all": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/it-all/-/it-all-3.0.11.tgz", + "integrity": "sha512-Gvqj6MO4GMLnFdtE68HZRpGBskNC+9+GQ+JevTGNYLyhjUuPhjDLU3jN1LpBemXJDW1bRSkczqA/qGyKlPKrcQ==", + "license": "Apache-2.0 OR MIT" + }, + "node_modules/blockstore-idb/node_modules/it-to-buffer": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/it-to-buffer/-/it-to-buffer-5.0.0.tgz", + "integrity": "sha512-DyinWj+79wxFDQNiPZcmV8Fz2PJFOcM3KRRN4GiZiIdEfxO11cCkrZJMfUMrxryy+rLoNhAfh1bpoVUFHPR6pQ==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "uint8arrays": "^6.1.0" + } + }, + "node_modules/blockstore-idb/node_modules/multiformats": { + "version": "14.0.5", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-14.0.5.tgz", + "integrity": "sha512-vbIm83F2yZ1pWJGS0yl0ysracIvv56LtbrIyiIQHoLdYDJOMoLfVFsXhh9DUH4SFdkdkFhucyWniihsNzVEjkQ==", + "license": "Apache-2.0 OR MIT" + }, + "node_modules/blockstore-idb/node_modules/uint8arrays": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-6.1.1.tgz", + "integrity": "sha512-iz7JN0XCSZYA111lhFG2Ui9EhFvTNekqSRHw3lvMHq+dzwWy1OQftxFQREEh4rffU0oSoXdQHsk2TiHKVm4fsA==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "multiformats": "^14.0.0" + } + }, "node_modules/body-parser": { "version": "1.20.2", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", @@ -16881,6 +17007,12 @@ "url": "https://opencollective.com/express" } }, + "node_modules/idb": { + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/idb/-/idb-8.0.3.tgz", + "integrity": "sha512-LtwtVyVYO5BqRvcsKuB2iUMnHwPVByPCXFXOpuU96IZPPoPN6xjOGxZQ74pgSVVLQWtUOYgyeL4GE98BY5D3wg==", + "license": "ISC" + }, "node_modules/ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -19301,9 +19433,9 @@ } }, "node_modules/kubo": { - "version": "0.42.0", - "resolved": "https://registry.npmjs.org/kubo/-/kubo-0.42.0.tgz", - "integrity": "sha512-980G+BC9sLKJZIjaJ2hI6sWMkYP55Fo00BRcBaoIZ0uPUUuiHu7u69ZTSsWTyBA6MbkLPAWCgbCD9jh3GWN2yA==", + "version": "0.43.0", + "resolved": "https://registry.npmjs.org/kubo/-/kubo-0.43.0.tgz", + "integrity": "sha512-hbTw6RhFUQiS71UIBX6+StQtGrISoAArEZykaxQsJlPLlE+50lz3sPAeSAZvWU2yb6VkjZIALMCJsHC3GDwucw==", "hasInstallScript": true, "license": "MIT", "dependencies": { @@ -25350,6 +25482,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/steno": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/steno/-/steno-4.0.2.tgz", + "integrity": "sha512-yhPIQXjrlt1xv7dyPQg2P17URmXbuM5pdGkpiMB3RenprfiBlvK415Lctfe0eshk90oA7/tNq7WEiMK8RSP39A==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/typicode" + } + }, "node_modules/stop-iteration-iterator": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/stop-iteration-iterator/-/stop-iteration-iterator-1.1.0.tgz", diff --git a/package.json b/package.json index dd23c03..2b877f2 100644 --- a/package.json +++ b/package.json @@ -119,13 +119,13 @@ "@oclif/plugin-help": "6.2.36", "@oclif/plugin-not-found": "3.2.73", "@oclif/table": "0.5.1", - "@pkcprotocol/pkc-js": "0.0.73", + "@pkcprotocol/pkc-js": "0.0.77", "dataobject-parser": "1.2.22", "decompress": "4.2.1", "env-paths": "2.2.1", "exit-hook": "4.0.0", "express": "4.19.2", - "kubo": "0.42.0", + "kubo": "0.43.0", "p-limit": "7.3.0", "strip-json-comments": "5.0.3", "tcp-port-used": "1.0.2", diff --git a/src/cli/commands/daemon.ts b/src/cli/commands/daemon.ts index 7b7e38e..c19081b 100644 --- a/src/cli/commands/daemon.ts +++ b/src/cli/commands/daemon.ts @@ -3,6 +3,7 @@ import { ChildProcessWithoutNullStreams } from "child_process"; import defaults from "../../common-utils/defaults.js"; import { startKuboNode } from "../../ipfs/startIpfs.js"; +import { startRepoGcScheduler, DEFAULT_REPO_GC_INTERVAL_MS } from "../../ipfs/repoGc.js"; import path from "path"; import tcpPortUsed from "tcp-port-used"; import { @@ -109,6 +110,19 @@ export default class Daemon extends Command { default: DEFAULT_PROVIDERS }), + enableIpfsGc: Flags.boolean({ + description: + "Periodically garbage-collect the IPFS repo over the kubo RPC API while the daemon is up. GC only runs once the repo passes 90% of Datastore.StorageMax (default 10GB), and only reclaims unpinned blocks — pinned data and MFS are never collected. Disable with --no-enableIpfsGc", + allowNo: true, + default: true + }), + + ipfsGcIntervalMinutes: Flags.integer({ + description: "How often to check whether the IPFS repo needs garbage collection, in minutes", + default: DEFAULT_REPO_GC_INTERVAL_MS / 60_000, + min: 1 + }), + allowPrivateKeyExport: Flags.boolean({ description: "Allow RPC clients to request community exports that include the community signer's private key (`bitsocial community export --includePrivateKey`). Disable with --no-allowPrivateKeyExport when exposing the RPC to untrusted clients", @@ -403,15 +417,20 @@ export default class Daemon extends Command { ); } let spawnedProcess: ChildProcessWithoutNullStreams | undefined; - const startPromise = startKuboNode(kuboRpcEndpoint, ipfsGatewayEndpoint, mergedPkcOptions.dataPath!, (process) => { - spawnedProcess = process; - kuboProcess = process; - if (process.pid) { - const pid = process.pid; - liveKuboPids.add(pid); - process.once("exit", () => liveKuboPids.delete(pid)); + const startPromise = startKuboNode( + kuboRpcEndpoint, + ipfsGatewayEndpoint, + mergedPkcOptions.dataPath!, + (process) => { + spawnedProcess = process; + kuboProcess = process; + if (process.pid) { + const pid = process.pid; + liveKuboPids.add(pid); + process.once("exit", () => liveKuboPids.delete(pid)); + } } - }); + ); pendingKuboStart = startPromise; let startedProcess: ChildProcessWithoutNullStreams | undefined; try { @@ -537,6 +556,7 @@ export default class Daemon extends Command { }; let keepKuboUpInterval: NodeJS.Timeout | undefined; + let stopRepoGcScheduler: (() => void) | undefined; const { asyncExitHook } = await import("exit-hook"); const killKuboProcessGroup = (pid: number, signal: NodeJS.Signals) => { // Kill the entire process group (negative PID) on non-Windows. @@ -613,6 +633,7 @@ export default class Daemon extends Command { const shutdownDaemon = async () => { if (keepKuboUpInterval) clearInterval(keepKuboUpInterval); + stopRepoGcScheduler?.(); if (mainProcessExited) return; // we already exited console.log( "\nShutting down Bitsocial daemon, it may take a few seconds to shut down all communities and the IPFS node..." @@ -715,6 +736,17 @@ export default class Daemon extends Command { if (!pkcOptionsFromFlag?.kuboRpcClientsOptions) await keepKuboUp(); await createOrConnectRpc(); + // Runs against whichever kubo the daemon ends up talking to, including one started by + // another program (--pkcOptions.kuboRpcClientsOptions). pkc-js also GCs on the same + // watermark, but only from a started local community's IPNS sync — a daemon that is up + // with no community started would otherwise never reclaim anything (issue #119). + if (flags.enableIpfsGc) + stopRepoGcScheduler = startRepoGcScheduler({ + kuboApiUrl: kuboRpcEndpoint, + intervalMs: flags.ipfsGcIntervalMinutes * 60 * 1000, + log: PKCLogger("bitsocial-cli:ipfs:repoGc") + }); + keepKuboUpInterval = setInterval(async () => { if (mainProcessExited) return; await runKeepKuboUpTick({ diff --git a/src/ipfs/repoGc.ts b/src/ipfs/repoGc.ts new file mode 100644 index 0000000..5e80f84 --- /dev/null +++ b/src/ipfs/repoGc.ts @@ -0,0 +1,188 @@ +import { PKCLogger } from "../util.js"; + +// GC once the repo is within this fraction of Datastore.StorageMax. kubo's own `--enable-gc` +// makes the same decision from Datastore.StorageGCWatermark (default 90) and so does pkc-js's +// cleanUpIpfsRepoIfDue, so we mirror it rather than invent a third policy. +export const GC_HIGH_WATERMARK = 0.9; + +// Matches kubo's default Datastore.GCPeriod. A repo that is over the watermark tends to STAY over +// it (GC only reclaims unpinned blocks), so this interval is the floor between runs, not a promise +// that anything is reclaimed each time. +export const DEFAULT_REPO_GC_INTERVAL_MS = 60 * 60 * 1000; + +export type RepoGcOutcome = { + ran: boolean; + skippedReason?: "below-watermark" | "repo-stat-failed" | "gc-failed"; + reclaimedCids?: number; + repoSizeBefore?: number; + repoSizeAfter?: number; + storageMax?: number; +}; + +type FetchLike = (input: string, init?: { method?: string; signal?: AbortSignal }) => Promise; + +// The daemon tracks kubo at its configured bind address, which is routinely a wildcard +// (`http://0.0.0.0:5001/api/v0`). Connecting to 0.0.0.0 fails with EINVAL on macOS, so resolve +// wildcards to loopback the same way daemon.ts does for its port checks. +function toConnectableApiBase(kuboApiUrl: URL | string): string { + const url = new URL(String(kuboApiUrl)); + if (url.hostname === "0.0.0.0") url.hostname = "127.0.0.1"; + else if (url.hostname === "::" || url.hostname === "[::]") url.hostname = "[::1]"; + return url.toString().replace(/\/+$/, ""); +} + +async function postRpc(fetchImpl: FetchLike, apiBase: string, pathAndQuery: string, signal?: AbortSignal): Promise { + const response = await fetchImpl(`${apiBase}/${pathAndQuery}`, { method: "POST", signal }); + if (!response.ok) throw new Error(`kubo RPC ${pathAndQuery} responded ${response.status} ${response.statusText}`); + return response; +} + +// `size-only` matters: the default repo/stat also counts every object, which walks the entire +// flatfs blockstore. On the repos this feature exists for that is millions of files. +async function readRepoStat( + fetchImpl: FetchLike, + apiBase: string, + signal?: AbortSignal +): Promise<{ repoSize: number; storageMax: number }> { + const response = await postRpc(fetchImpl, apiBase, "repo/stat?size-only=true", signal); + const body = (await response.json()) as { RepoSize?: number; StorageMax?: number }; + return { repoSize: Number(body.RepoSize ?? 0), storageMax: Number(body.StorageMax ?? 0) }; +} + +/** + * Runs `repo gc` over the kubo RPC API if the repo has crossed the watermark. + * + * We drive GC over RPC rather than passing kubo's own `--enable-gc` daemon flag. A kubo daemon + * started with `--enable-gc` never exits in response to `POST /api/v0/shutdown` — it logs + * "cannot access config, repo not open" and lingers indefinitely as a half-shutdown zombie + * (SIGTERM still works; verified permanent past 300s). pkc-js POSTs that exact endpoint when it + * rewrites the kubo Routing config on first connect and relies on the daemon restarting kubo + * afterwards, so `--enable-gc` wedges the supervision loop. + * + * Reported upstream as ipfs/kubo#11424: `maybeRunGC` gives `PeriodicGC` the command request + * context, but the shutdown command only calls `nd.Close()`, which cancels the node context — so + * `gcErrc` never closes and `daemonFunc` blocks forever draining it. Reproduces on 0.24.0, 0.42.0 + * and 0.43.0, which is also why the earlier attempt at the flag (f228d7d, Dec 2023, kubo ~0.24) + * was reverted two days later. Do not add `--enable-gc` back without re-testing that path. + * + * `force` skips the watermark check but is still subject to the caller's interval. + */ +export async function runRepoGcIfDue(options: { + kuboApiUrl: URL | string; + log?: any; + force?: boolean; + signal?: AbortSignal; + fetchImpl?: FetchLike; +}): Promise { + const log = options.log ?? PKCLogger("bitsocial-cli:ipfs:repoGc"); + const fetchImpl = options.fetchImpl ?? (globalThis.fetch as unknown as FetchLike); + const apiBase = toConnectableApiBase(options.kuboApiUrl); + + let repoSizeBefore: number | undefined; + let storageMax: number | undefined; + + if (!options.force) { + let stat: { repoSize: number; storageMax: number }; + try { + stat = await readRepoStat(fetchImpl, apiBase, options.signal); + } catch (error) { + // A daemon we can't stat is one to back off from, not to blindly GC. + log.error?.("Skipping repo gc: failed to read repo/stat from the kubo node", apiBase, error); + return { ran: false, skippedReason: "repo-stat-failed" }; + } + repoSizeBefore = stat.repoSize; + storageMax = stat.storageMax; + + // storageMax comes from Datastore.StorageMax. If the daemon reports no ceiling there is + // nothing to compare against, so fall back to GCing on the interval alone rather than + // never GCing at all. + if (stat.storageMax > 0) { + const threshold = stat.storageMax * GC_HIGH_WATERMARK; + if (stat.repoSize < threshold) { + log.trace?.( + `Skipping repo gc on ${apiBase} - repo size ${stat.repoSize} is below the ${GC_HIGH_WATERMARK * 100}% watermark ${threshold} of StorageMax ${stat.storageMax}` + ); + return { ran: false, skippedReason: "below-watermark", repoSizeBefore, storageMax }; + } + } + } + + let reclaimedCids = 0; + try { + const response = await postRpc(fetchImpl, apiBase, "repo/gc?quiet=true", options.signal); + // repo/gc streams newline-delimited JSON, one object per reclaimed CID. Draining it fully + // is what makes this await mean "GC finished" rather than "GC started". + const text = await response.text(); + for (const line of text.split("\n")) { + const trimmed = line.trim(); + if (!trimmed) continue; + try { + const parsed = JSON.parse(trimmed) as { Key?: unknown; Error?: string }; + if (parsed.Error) log.error?.("Failed to GC a block out of the ipfs repo", parsed.Error); + else if (parsed.Key) reclaimedCids++; + } catch { + // A malformed line is not worth aborting a completed GC over. + } + } + } catch (error) { + log.error?.("Failed to GC ipfs repo", apiBase, error); + return { ran: false, skippedReason: "gc-failed", repoSizeBefore, storageMax }; + } + + let repoSizeAfter: number | undefined; + try { + repoSizeAfter = (await readRepoStat(fetchImpl, apiBase, options.signal)).repoSize; + } catch (error) { + log.trace?.("repo gc finished but the follow-up repo/stat failed", error); + } + + // How much a GC actually reclaims is worth logging rather than assuming: GC never touches + // pinned data, and a node with thousands of recursive pins can stay over the watermark. + log( + `GC reclaimed ${reclaimedCids} cids from the IPFS node ${apiBase} - repo size ${repoSizeBefore ?? "unknown"} -> ${repoSizeAfter ?? "unknown"}` + ); + return { ran: true, reclaimedCids, repoSizeBefore, repoSizeAfter, storageMax }; +} + +/** + * Starts the periodic repo GC. Returns a stop function. + * + * The timer is unref'd so it never by itself keeps the daemon process alive, and runs are + * single-flighted: a GC that outlives its own interval must not have a second one stacked on top + * of it. Errors are swallowed inside the tick — an interval callback that rejects becomes an + * unhandledRejection and takes the daemon down (same failure mode as issue #37 bug 3). + */ +export function startRepoGcScheduler(options: { + kuboApiUrl: URL | string; + intervalMs?: number; + log?: any; + fetchImpl?: FetchLike; +}): () => void { + const log = options.log ?? PKCLogger("bitsocial-cli:ipfs:repoGc"); + const intervalMs = options.intervalMs ?? DEFAULT_REPO_GC_INTERVAL_MS; + const abortController = new AbortController(); + let inFlight: Promise | undefined; + + const tick = () => { + if (inFlight) return; + inFlight = runRepoGcIfDue({ + kuboApiUrl: options.kuboApiUrl, + log, + signal: abortController.signal, + fetchImpl: options.fetchImpl + }) + .catch((error) => log.error?.("repo gc tick error (will retry next interval)", error)) + .finally(() => { + inFlight = undefined; + }); + }; + + const timer = setInterval(tick, intervalMs); + timer.unref?.(); + log(`Scheduled IPFS repo GC every ${intervalMs}ms against ${String(options.kuboApiUrl)}`); + + return () => { + clearInterval(timer); + abortController.abort(); + }; +} diff --git a/test/kubo/repoGc.test.ts b/test/kubo/repoGc.test.ts new file mode 100644 index 0000000..3fab21b --- /dev/null +++ b/test/kubo/repoGc.test.ts @@ -0,0 +1,204 @@ +import { describe, it, expect, vi } from "vitest"; +import { runRepoGcIfDue, startRepoGcScheduler, GC_HIGH_WATERMARK, DEFAULT_REPO_GC_INTERVAL_MS } from "../../src/ipfs/repoGc.js"; + +const STORAGE_MAX = 10_000_000_000; + +const silentLog: any = Object.assign(() => {}, { error: () => {}, trace: () => {} }); + +// Minimal kubo RPC stub: repo/stat answers with the sizes the case needs, repo/gc streams one +// newline-delimited JSON object per reclaimed CID exactly like the real endpoint. +const makeKuboStub = (options: { + repoSizes: number[]; + storageMax?: number; + gcCids?: string[]; + statStatus?: number; + gcStatus?: number; +}) => { + const calls: string[] = []; + let statCall = 0; + const fetchImpl = vi.fn(async (url: string) => { + calls.push(url); + if (url.includes("repo/stat")) { + if (options.statStatus && options.statStatus >= 400) + return new Response("nope", { status: options.statStatus, statusText: "Server Error" }); + const size = options.repoSizes[Math.min(statCall++, options.repoSizes.length - 1)]; + return new Response(JSON.stringify({ RepoSize: size, StorageMax: options.storageMax ?? STORAGE_MAX }), { status: 200 }); + } + if (url.includes("repo/gc")) { + if (options.gcStatus && options.gcStatus >= 400) + return new Response("nope", { status: options.gcStatus, statusText: "Server Error" }); + const body = (options.gcCids ?? []).map((cid) => JSON.stringify({ Key: { "/": cid }, Error: "" })).join("\n"); + return new Response(body, { status: 200 }); + } + throw new Error(`unexpected kubo RPC call ${url}`); + }); + return { fetchImpl, calls }; +}; + +describe("runRepoGcIfDue (issue #119)", () => { + it("skips GC when the repo is below the 90% StorageMax watermark", async () => { + const { fetchImpl, calls } = makeKuboStub({ repoSizes: [STORAGE_MAX * 0.5] }); + const outcome = await runRepoGcIfDue({ kuboApiUrl: "http://127.0.0.1:5001/api/v0", log: silentLog, fetchImpl }); + + expect(outcome.ran).toBe(false); + expect(outcome.skippedReason).toBe("below-watermark"); + expect(calls.some((url) => url.includes("repo/gc"))).toBe(false); + }); + + it("runs GC once the repo crosses the watermark, and reports what it reclaimed", async () => { + const before = STORAGE_MAX * GC_HIGH_WATERMARK + 1; + const { fetchImpl, calls } = makeKuboStub({ repoSizes: [before, 1_000], gcCids: ["QmA", "QmB", "QmC"] }); + const outcome = await runRepoGcIfDue({ kuboApiUrl: "http://127.0.0.1:5001/api/v0", log: silentLog, fetchImpl }); + + expect(outcome.ran).toBe(true); + expect(outcome.reclaimedCids).toBe(3); + expect(outcome.repoSizeBefore).toBe(before); + expect(outcome.repoSizeAfter).toBe(1_000); + expect(calls.some((url) => url.includes("repo/gc"))).toBe(true); + }); + + it("asks for size-only repo stats so it does not walk the whole blockstore", async () => { + const { fetchImpl, calls } = makeKuboStub({ repoSizes: [1] }); + await runRepoGcIfDue({ kuboApiUrl: "http://127.0.0.1:5001/api/v0", log: silentLog, fetchImpl }); + + expect(calls[0]).toContain("size-only=true"); + }); + + it("GCs on the interval alone when the daemon reports no StorageMax ceiling", async () => { + const { fetchImpl } = makeKuboStub({ repoSizes: [1_000, 500], storageMax: 0, gcCids: ["QmA"] }); + const outcome = await runRepoGcIfDue({ kuboApiUrl: "http://127.0.0.1:5001/api/v0", log: silentLog, fetchImpl }); + + expect(outcome.ran).toBe(true); + }); + + it("skips the watermark check when forced", async () => { + const { fetchImpl, calls } = makeKuboStub({ repoSizes: [1_000], gcCids: ["QmA"] }); + const outcome = await runRepoGcIfDue({ kuboApiUrl: "http://127.0.0.1:5001/api/v0", log: silentLog, force: true, fetchImpl }); + + expect(outcome.ran).toBe(true); + // Goes straight to repo/gc — the only repo/stat is the follow-up that measures what was + // reclaimed, never a pre-GC watermark check. + expect(calls[0]).toContain("repo/gc"); + expect(calls.filter((url) => url.includes("repo/stat")).length).toBe(1); + }); + + it("backs off instead of blindly GCing when repo/stat fails", async () => { + const { fetchImpl, calls } = makeKuboStub({ repoSizes: [1], statStatus: 500 }); + const outcome = await runRepoGcIfDue({ kuboApiUrl: "http://127.0.0.1:5001/api/v0", log: silentLog, fetchImpl }); + + expect(outcome.ran).toBe(false); + expect(outcome.skippedReason).toBe("repo-stat-failed"); + expect(calls.some((url) => url.includes("repo/gc"))).toBe(false); + }); + + it("resolves rather than throws when repo/gc itself fails", async () => { + const { fetchImpl } = makeKuboStub({ repoSizes: [STORAGE_MAX], gcStatus: 500 }); + const outcome = await runRepoGcIfDue({ kuboApiUrl: "http://127.0.0.1:5001/api/v0", log: silentLog, fetchImpl }); + + expect(outcome.ran).toBe(false); + expect(outcome.skippedReason).toBe("gc-failed"); + }); + + // The daemon tracks kubo at its configured bind address, which is routinely a wildcard. + // Connecting to 0.0.0.0 fails with EINVAL on macOS. + it("rewrites a wildcard bind address to loopback before connecting", async () => { + const { fetchImpl, calls } = makeKuboStub({ repoSizes: [1] }); + await runRepoGcIfDue({ kuboApiUrl: "http://0.0.0.0:5001/api/v0", log: silentLog, fetchImpl }); + + expect(calls[0]).toContain("127.0.0.1"); + expect(calls[0]).not.toContain("0.0.0.0"); + }); +}); + +describe("startRepoGcScheduler (issue #119)", () => { + it("defaults to kubo's own 1h GC period", () => { + expect(DEFAULT_REPO_GC_INTERVAL_MS).toBe(60 * 60 * 1000); + }); + + it("GCs on each interval tick and stops when told to", async () => { + vi.useFakeTimers(); + try { + const { fetchImpl, calls } = makeKuboStub({ repoSizes: [STORAGE_MAX], gcCids: ["QmA"] }); + const stop = startRepoGcScheduler({ + kuboApiUrl: "http://127.0.0.1:5001/api/v0", + intervalMs: 1000, + log: silentLog, + fetchImpl + }); + + expect(calls.length).toBe(0); // nothing on construction + + await vi.advanceTimersByTimeAsync(1000); + expect(calls.some((url) => url.includes("repo/gc"))).toBe(true); + const afterFirstTick = calls.length; + + stop(); + await vi.advanceTimersByTimeAsync(5000); + expect(calls.length).toBe(afterFirstTick); // no further ticks after stop + } finally { + vi.useRealTimers(); + } + }); + + // A GC that outlives its own interval must not have a second one stacked on top of it — both + // would contend for kubo's GC lock. + it("does not stack a second GC on top of one still running", async () => { + vi.useFakeTimers(); + try { + let releaseGc: (() => void) | undefined; + const gcStarted: number[] = []; + const fetchImpl = vi.fn(async (url: string) => { + if (url.includes("repo/stat")) return new Response(JSON.stringify({ RepoSize: STORAGE_MAX, StorageMax: STORAGE_MAX })); + gcStarted.push(1); + await new Promise((resolve) => (releaseGc = resolve)); + return new Response(""); + }); + + const stop = startRepoGcScheduler({ + kuboApiUrl: "http://127.0.0.1:5001/api/v0", + intervalMs: 1000, + log: silentLog, + fetchImpl: fetchImpl as any + }); + + await vi.advanceTimersByTimeAsync(1000); + expect(gcStarted.length).toBe(1); + + // Three more intervals elapse while the first GC is still streaming. + await vi.advanceTimersByTimeAsync(3000); + expect(gcStarted.length).toBe(1); + + releaseGc?.(); + stop(); + } finally { + vi.useRealTimers(); + } + }); + + // An interval callback that rejects becomes an unhandledRejection and takes the daemon down. + it("does not emit an unhandledRejection when the kubo node is unreachable", async () => { + vi.useFakeTimers(); + const unhandled: unknown[] = []; + const handler = (err: unknown) => unhandled.push(err); + process.on("unhandledRejection", handler); + try { + const fetchImpl = vi.fn(async () => { + throw Object.assign(new Error("connect ECONNREFUSED"), { code: "ECONNREFUSED" }); + }); + const stop = startRepoGcScheduler({ + kuboApiUrl: "http://127.0.0.1:5001/api/v0", + intervalMs: 1000, + log: silentLog, + fetchImpl: fetchImpl as any + }); + + await vi.advanceTimersByTimeAsync(3000); + stop(); + await vi.advanceTimersByTimeAsync(0); + } finally { + process.off("unhandledRejection", handler); + vi.useRealTimers(); + } + expect(unhandled).toEqual([]); + }); +}); From d67d4863b9ae44514aa72f156eda0403c160d922 Mon Sep 17 00:00:00 2001 From: Rinse Date: Fri, 7 Aug 2026 12:43:02 +0000 Subject: [PATCH 2/2] refactor(daemon): run repo gc unconditionally every hour Drops the repo/stat watermark check. GC only ever reclaims unpinned blocks, so kubo already decides what is collectable -- reading Datastore.StorageMax to gate the call just duplicated that decision on our side. Removes both repo/stat calls (the pre-GC watermark check and the post-GC size probe), GC_HIGH_WATERMARK and the `force` option. `runRepoGcIfDue` is now `runRepoGc`, which POSTs repo/gc and counts reclaimed cids from the response stream. Behaviour change: GC now runs on every tick regardless of repo size, where before it was skipped below 90% of StorageMax. Scheduling is unchanged -- hourly by default, single-flighted, unref'd timer, errors contained inside the tick. --- README.md | 8 +-- src/cli/commands/daemon.ts | 10 ++-- src/ipfs/repoGc.ts | 98 ++++-------------------------- test/kubo/repoGc.test.ts | 120 ++++++++++++------------------------- 4 files changed, 59 insertions(+), 177 deletions(-) diff --git a/README.md b/README.md index bcc9878..dd642ee 100644 --- a/README.md +++ b/README.md @@ -884,11 +884,9 @@ FLAGS com,https://rpc.mevblocker.io,https://1rpc.io/eth,https://eth-pokt.nodies.app] RPC URL(s) for .bso name resolution. Can be specified multiple times. --[no-]enableIpfsGc Periodically garbage-collect the IPFS repo over the kubo RPC API while the daemon is - up. GC only runs once the repo passes 90% of Datastore.StorageMax (default 10GB), and - only reclaims unpinned blocks — pinned data and MFS are never collected. Disable with - --no-enableIpfsGc - --ipfsGcIntervalMinutes= [default: 60] How often to check whether the IPFS repo needs garbage collection, in - minutes + up. Only reclaims unpinned blocks — pinned data and MFS are never collected. Disable + with --no-enableIpfsGc + --ipfsGcIntervalMinutes= [default: 60] How often to garbage-collect the IPFS repo, in minutes --logPath= (required) [default: /home/runner/.local/state/bitsocial] Specify a directory which will be used to store logs --pkcRpcUrl= (required) [default: ws://localhost:9138/] Specify PKC RPC URL to listen on diff --git a/src/cli/commands/daemon.ts b/src/cli/commands/daemon.ts index c19081b..acdd880 100644 --- a/src/cli/commands/daemon.ts +++ b/src/cli/commands/daemon.ts @@ -112,13 +112,13 @@ export default class Daemon extends Command { enableIpfsGc: Flags.boolean({ description: - "Periodically garbage-collect the IPFS repo over the kubo RPC API while the daemon is up. GC only runs once the repo passes 90% of Datastore.StorageMax (default 10GB), and only reclaims unpinned blocks — pinned data and MFS are never collected. Disable with --no-enableIpfsGc", + "Periodically garbage-collect the IPFS repo over the kubo RPC API while the daemon is up. Only reclaims unpinned blocks — pinned data and MFS are never collected. Disable with --no-enableIpfsGc", allowNo: true, default: true }), ipfsGcIntervalMinutes: Flags.integer({ - description: "How often to check whether the IPFS repo needs garbage collection, in minutes", + description: "How often to garbage-collect the IPFS repo, in minutes", default: DEFAULT_REPO_GC_INTERVAL_MS / 60_000, min: 1 }), @@ -737,9 +737,9 @@ export default class Daemon extends Command { await createOrConnectRpc(); // Runs against whichever kubo the daemon ends up talking to, including one started by - // another program (--pkcOptions.kuboRpcClientsOptions). pkc-js also GCs on the same - // watermark, but only from a started local community's IPNS sync — a daemon that is up - // with no community started would otherwise never reclaim anything (issue #119). + // another program (--pkcOptions.kuboRpcClientsOptions). pkc-js also GCs, but only from + // a started local community's IPNS sync — a daemon that is up with no community + // started would otherwise never reclaim anything (issue #119). if (flags.enableIpfsGc) stopRepoGcScheduler = startRepoGcScheduler({ kuboApiUrl: kuboRpcEndpoint, diff --git a/src/ipfs/repoGc.ts b/src/ipfs/repoGc.ts index 5e80f84..264f649 100644 --- a/src/ipfs/repoGc.ts +++ b/src/ipfs/repoGc.ts @@ -1,24 +1,8 @@ import { PKCLogger } from "../util.js"; -// GC once the repo is within this fraction of Datastore.StorageMax. kubo's own `--enable-gc` -// makes the same decision from Datastore.StorageGCWatermark (default 90) and so does pkc-js's -// cleanUpIpfsRepoIfDue, so we mirror it rather than invent a third policy. -export const GC_HIGH_WATERMARK = 0.9; - -// Matches kubo's default Datastore.GCPeriod. A repo that is over the watermark tends to STAY over -// it (GC only reclaims unpinned blocks), so this interval is the floor between runs, not a promise -// that anything is reclaimed each time. +// Matches kubo's default Datastore.GCPeriod. export const DEFAULT_REPO_GC_INTERVAL_MS = 60 * 60 * 1000; -export type RepoGcOutcome = { - ran: boolean; - skippedReason?: "below-watermark" | "repo-stat-failed" | "gc-failed"; - reclaimedCids?: number; - repoSizeBefore?: number; - repoSizeAfter?: number; - storageMax?: number; -}; - type FetchLike = (input: string, init?: { method?: string; signal?: AbortSignal }) => Promise; // The daemon tracks kubo at its configured bind address, which is routinely a wildcard @@ -31,26 +15,9 @@ function toConnectableApiBase(kuboApiUrl: URL | string): string { return url.toString().replace(/\/+$/, ""); } -async function postRpc(fetchImpl: FetchLike, apiBase: string, pathAndQuery: string, signal?: AbortSignal): Promise { - const response = await fetchImpl(`${apiBase}/${pathAndQuery}`, { method: "POST", signal }); - if (!response.ok) throw new Error(`kubo RPC ${pathAndQuery} responded ${response.status} ${response.statusText}`); - return response; -} - -// `size-only` matters: the default repo/stat also counts every object, which walks the entire -// flatfs blockstore. On the repos this feature exists for that is millions of files. -async function readRepoStat( - fetchImpl: FetchLike, - apiBase: string, - signal?: AbortSignal -): Promise<{ repoSize: number; storageMax: number }> { - const response = await postRpc(fetchImpl, apiBase, "repo/stat?size-only=true", signal); - const body = (await response.json()) as { RepoSize?: number; StorageMax?: number }; - return { repoSize: Number(body.RepoSize ?? 0), storageMax: Number(body.StorageMax ?? 0) }; -} - /** - * Runs `repo gc` over the kubo RPC API if the repo has crossed the watermark. + * Runs `repo gc` over the kubo RPC API. GC only ever reclaims unpinned blocks, so this is safe to + * run unconditionally — kubo decides what is actually collectable. * * We drive GC over RPC rather than passing kubo's own `--enable-gc` daemon flag. A kubo daemon * started with `--enable-gc` never exits in response to `POST /api/v0/shutdown` — it logs @@ -64,52 +31,22 @@ async function readRepoStat( * `gcErrc` never closes and `daemonFunc` blocks forever draining it. Reproduces on 0.24.0, 0.42.0 * and 0.43.0, which is also why the earlier attempt at the flag (f228d7d, Dec 2023, kubo ~0.24) * was reverted two days later. Do not add `--enable-gc` back without re-testing that path. - * - * `force` skips the watermark check but is still subject to the caller's interval. */ -export async function runRepoGcIfDue(options: { +export async function runRepoGc(options: { kuboApiUrl: URL | string; log?: any; - force?: boolean; signal?: AbortSignal; fetchImpl?: FetchLike; -}): Promise { +}): Promise<{ ran: boolean; reclaimedCids?: number }> { const log = options.log ?? PKCLogger("bitsocial-cli:ipfs:repoGc"); const fetchImpl = options.fetchImpl ?? (globalThis.fetch as unknown as FetchLike); const apiBase = toConnectableApiBase(options.kuboApiUrl); - let repoSizeBefore: number | undefined; - let storageMax: number | undefined; - - if (!options.force) { - let stat: { repoSize: number; storageMax: number }; - try { - stat = await readRepoStat(fetchImpl, apiBase, options.signal); - } catch (error) { - // A daemon we can't stat is one to back off from, not to blindly GC. - log.error?.("Skipping repo gc: failed to read repo/stat from the kubo node", apiBase, error); - return { ran: false, skippedReason: "repo-stat-failed" }; - } - repoSizeBefore = stat.repoSize; - storageMax = stat.storageMax; - - // storageMax comes from Datastore.StorageMax. If the daemon reports no ceiling there is - // nothing to compare against, so fall back to GCing on the interval alone rather than - // never GCing at all. - if (stat.storageMax > 0) { - const threshold = stat.storageMax * GC_HIGH_WATERMARK; - if (stat.repoSize < threshold) { - log.trace?.( - `Skipping repo gc on ${apiBase} - repo size ${stat.repoSize} is below the ${GC_HIGH_WATERMARK * 100}% watermark ${threshold} of StorageMax ${stat.storageMax}` - ); - return { ran: false, skippedReason: "below-watermark", repoSizeBefore, storageMax }; - } - } - } - let reclaimedCids = 0; try { - const response = await postRpc(fetchImpl, apiBase, "repo/gc?quiet=true", options.signal); + const response = await fetchImpl(`${apiBase}/repo/gc?quiet=true`, { method: "POST", signal: options.signal }); + if (!response.ok) throw new Error(`kubo RPC repo/gc responded ${response.status} ${response.statusText}`); + // repo/gc streams newline-delimited JSON, one object per reclaimed CID. Draining it fully // is what makes this await mean "GC finished" rather than "GC started". const text = await response.text(); @@ -126,22 +63,11 @@ export async function runRepoGcIfDue(options: { } } catch (error) { log.error?.("Failed to GC ipfs repo", apiBase, error); - return { ran: false, skippedReason: "gc-failed", repoSizeBefore, storageMax }; - } - - let repoSizeAfter: number | undefined; - try { - repoSizeAfter = (await readRepoStat(fetchImpl, apiBase, options.signal)).repoSize; - } catch (error) { - log.trace?.("repo gc finished but the follow-up repo/stat failed", error); + return { ran: false }; } - // How much a GC actually reclaims is worth logging rather than assuming: GC never touches - // pinned data, and a node with thousands of recursive pins can stay over the watermark. - log( - `GC reclaimed ${reclaimedCids} cids from the IPFS node ${apiBase} - repo size ${repoSizeBefore ?? "unknown"} -> ${repoSizeAfter ?? "unknown"}` - ); - return { ran: true, reclaimedCids, repoSizeBefore, repoSizeAfter, storageMax }; + log(`GC reclaimed ${reclaimedCids} cids from the IPFS node ${apiBase}`); + return { ran: true, reclaimedCids }; } /** @@ -165,7 +91,7 @@ export function startRepoGcScheduler(options: { const tick = () => { if (inFlight) return; - inFlight = runRepoGcIfDue({ + inFlight = runRepoGc({ kuboApiUrl: options.kuboApiUrl, log, signal: abortController.signal, diff --git a/test/kubo/repoGc.test.ts b/test/kubo/repoGc.test.ts index 3fab21b..dfc78a7 100644 --- a/test/kubo/repoGc.test.ts +++ b/test/kubo/repoGc.test.ts @@ -1,109 +1,66 @@ import { describe, it, expect, vi } from "vitest"; -import { runRepoGcIfDue, startRepoGcScheduler, GC_HIGH_WATERMARK, DEFAULT_REPO_GC_INTERVAL_MS } from "../../src/ipfs/repoGc.js"; - -const STORAGE_MAX = 10_000_000_000; +import { runRepoGc, startRepoGcScheduler, DEFAULT_REPO_GC_INTERVAL_MS } from "../../src/ipfs/repoGc.js"; const silentLog: any = Object.assign(() => {}, { error: () => {}, trace: () => {} }); -// Minimal kubo RPC stub: repo/stat answers with the sizes the case needs, repo/gc streams one -// newline-delimited JSON object per reclaimed CID exactly like the real endpoint. -const makeKuboStub = (options: { - repoSizes: number[]; - storageMax?: number; - gcCids?: string[]; - statStatus?: number; - gcStatus?: number; -}) => { +// Minimal kubo RPC stub: repo/gc streams one newline-delimited JSON object per reclaimed CID, +// exactly like the real endpoint. +const makeKuboStub = (options: { gcCids?: string[]; gcStatus?: number; gcErrors?: string[] } = {}) => { const calls: string[] = []; - let statCall = 0; const fetchImpl = vi.fn(async (url: string) => { calls.push(url); - if (url.includes("repo/stat")) { - if (options.statStatus && options.statStatus >= 400) - return new Response("nope", { status: options.statStatus, statusText: "Server Error" }); - const size = options.repoSizes[Math.min(statCall++, options.repoSizes.length - 1)]; - return new Response(JSON.stringify({ RepoSize: size, StorageMax: options.storageMax ?? STORAGE_MAX }), { status: 200 }); - } - if (url.includes("repo/gc")) { - if (options.gcStatus && options.gcStatus >= 400) - return new Response("nope", { status: options.gcStatus, statusText: "Server Error" }); - const body = (options.gcCids ?? []).map((cid) => JSON.stringify({ Key: { "/": cid }, Error: "" })).join("\n"); - return new Response(body, { status: 200 }); - } - throw new Error(`unexpected kubo RPC call ${url}`); + if (!url.includes("repo/gc")) throw new Error(`unexpected kubo RPC call ${url}`); + if (options.gcStatus && options.gcStatus >= 400) + return new Response("nope", { status: options.gcStatus, statusText: "Server Error" }); + const lines = [ + ...(options.gcCids ?? []).map((cid) => JSON.stringify({ Key: { "/": cid }, Error: "" })), + ...(options.gcErrors ?? []).map((err) => JSON.stringify({ Error: err })) + ]; + return new Response(lines.join("\n"), { status: 200 }); }); return { fetchImpl, calls }; }; -describe("runRepoGcIfDue (issue #119)", () => { - it("skips GC when the repo is below the 90% StorageMax watermark", async () => { - const { fetchImpl, calls } = makeKuboStub({ repoSizes: [STORAGE_MAX * 0.5] }); - const outcome = await runRepoGcIfDue({ kuboApiUrl: "http://127.0.0.1:5001/api/v0", log: silentLog, fetchImpl }); - - expect(outcome.ran).toBe(false); - expect(outcome.skippedReason).toBe("below-watermark"); - expect(calls.some((url) => url.includes("repo/gc"))).toBe(false); - }); - - it("runs GC once the repo crosses the watermark, and reports what it reclaimed", async () => { - const before = STORAGE_MAX * GC_HIGH_WATERMARK + 1; - const { fetchImpl, calls } = makeKuboStub({ repoSizes: [before, 1_000], gcCids: ["QmA", "QmB", "QmC"] }); - const outcome = await runRepoGcIfDue({ kuboApiUrl: "http://127.0.0.1:5001/api/v0", log: silentLog, fetchImpl }); +describe("runRepoGc (issue #119)", () => { + it("GCs unconditionally — no repo/stat, no watermark check", async () => { + const { fetchImpl, calls } = makeKuboStub({ gcCids: ["QmA", "QmB", "QmC"] }); + const outcome = await runRepoGc({ kuboApiUrl: "http://127.0.0.1:5001/api/v0", log: silentLog, fetchImpl }); expect(outcome.ran).toBe(true); expect(outcome.reclaimedCids).toBe(3); - expect(outcome.repoSizeBefore).toBe(before); - expect(outcome.repoSizeAfter).toBe(1_000); - expect(calls.some((url) => url.includes("repo/gc"))).toBe(true); - }); - - it("asks for size-only repo stats so it does not walk the whole blockstore", async () => { - const { fetchImpl, calls } = makeKuboStub({ repoSizes: [1] }); - await runRepoGcIfDue({ kuboApiUrl: "http://127.0.0.1:5001/api/v0", log: silentLog, fetchImpl }); - - expect(calls[0]).toContain("size-only=true"); + expect(calls.length).toBe(1); + expect(calls[0]).toContain("repo/gc"); + expect(calls.some((url) => url.includes("repo/stat"))).toBe(false); }); - it("GCs on the interval alone when the daemon reports no StorageMax ceiling", async () => { - const { fetchImpl } = makeKuboStub({ repoSizes: [1_000, 500], storageMax: 0, gcCids: ["QmA"] }); - const outcome = await runRepoGcIfDue({ kuboApiUrl: "http://127.0.0.1:5001/api/v0", log: silentLog, fetchImpl }); + it("still reports success when kubo had nothing to collect", async () => { + const { fetchImpl } = makeKuboStub({ gcCids: [] }); + const outcome = await runRepoGc({ kuboApiUrl: "http://127.0.0.1:5001/api/v0", log: silentLog, fetchImpl }); expect(outcome.ran).toBe(true); + expect(outcome.reclaimedCids).toBe(0); }); - it("skips the watermark check when forced", async () => { - const { fetchImpl, calls } = makeKuboStub({ repoSizes: [1_000], gcCids: ["QmA"] }); - const outcome = await runRepoGcIfDue({ kuboApiUrl: "http://127.0.0.1:5001/api/v0", log: silentLog, force: true, fetchImpl }); + it("counts only reclaimed keys, not per-block errors in the stream", async () => { + const { fetchImpl } = makeKuboStub({ gcCids: ["QmA"], gcErrors: ["could not remove QmZ"] }); + const outcome = await runRepoGc({ kuboApiUrl: "http://127.0.0.1:5001/api/v0", log: silentLog, fetchImpl }); expect(outcome.ran).toBe(true); - // Goes straight to repo/gc — the only repo/stat is the follow-up that measures what was - // reclaimed, never a pre-GC watermark check. - expect(calls[0]).toContain("repo/gc"); - expect(calls.filter((url) => url.includes("repo/stat")).length).toBe(1); + expect(outcome.reclaimedCids).toBe(1); }); - it("backs off instead of blindly GCing when repo/stat fails", async () => { - const { fetchImpl, calls } = makeKuboStub({ repoSizes: [1], statStatus: 500 }); - const outcome = await runRepoGcIfDue({ kuboApiUrl: "http://127.0.0.1:5001/api/v0", log: silentLog, fetchImpl }); + it("resolves rather than throws when repo/gc fails", async () => { + const { fetchImpl } = makeKuboStub({ gcStatus: 500 }); + const outcome = await runRepoGc({ kuboApiUrl: "http://127.0.0.1:5001/api/v0", log: silentLog, fetchImpl }); expect(outcome.ran).toBe(false); - expect(outcome.skippedReason).toBe("repo-stat-failed"); - expect(calls.some((url) => url.includes("repo/gc"))).toBe(false); - }); - - it("resolves rather than throws when repo/gc itself fails", async () => { - const { fetchImpl } = makeKuboStub({ repoSizes: [STORAGE_MAX], gcStatus: 500 }); - const outcome = await runRepoGcIfDue({ kuboApiUrl: "http://127.0.0.1:5001/api/v0", log: silentLog, fetchImpl }); - - expect(outcome.ran).toBe(false); - expect(outcome.skippedReason).toBe("gc-failed"); }); // The daemon tracks kubo at its configured bind address, which is routinely a wildcard. // Connecting to 0.0.0.0 fails with EINVAL on macOS. it("rewrites a wildcard bind address to loopback before connecting", async () => { - const { fetchImpl, calls } = makeKuboStub({ repoSizes: [1] }); - await runRepoGcIfDue({ kuboApiUrl: "http://0.0.0.0:5001/api/v0", log: silentLog, fetchImpl }); + const { fetchImpl, calls } = makeKuboStub({ gcCids: [] }); + await runRepoGc({ kuboApiUrl: "http://0.0.0.0:5001/api/v0", log: silentLog, fetchImpl }); expect(calls[0]).toContain("127.0.0.1"); expect(calls[0]).not.toContain("0.0.0.0"); @@ -118,7 +75,7 @@ describe("startRepoGcScheduler (issue #119)", () => { it("GCs on each interval tick and stops when told to", async () => { vi.useFakeTimers(); try { - const { fetchImpl, calls } = makeKuboStub({ repoSizes: [STORAGE_MAX], gcCids: ["QmA"] }); + const { fetchImpl, calls } = makeKuboStub({ gcCids: ["QmA"] }); const stop = startRepoGcScheduler({ kuboApiUrl: "http://127.0.0.1:5001/api/v0", intervalMs: 1000, @@ -129,12 +86,14 @@ describe("startRepoGcScheduler (issue #119)", () => { expect(calls.length).toBe(0); // nothing on construction await vi.advanceTimersByTimeAsync(1000); - expect(calls.some((url) => url.includes("repo/gc"))).toBe(true); - const afterFirstTick = calls.length; + expect(calls.length).toBe(1); + + await vi.advanceTimersByTimeAsync(2000); + expect(calls.length).toBe(3); // one GC per interval stop(); await vi.advanceTimersByTimeAsync(5000); - expect(calls.length).toBe(afterFirstTick); // no further ticks after stop + expect(calls.length).toBe(3); // no further ticks after stop } finally { vi.useRealTimers(); } @@ -147,8 +106,7 @@ describe("startRepoGcScheduler (issue #119)", () => { try { let releaseGc: (() => void) | undefined; const gcStarted: number[] = []; - const fetchImpl = vi.fn(async (url: string) => { - if (url.includes("repo/stat")) return new Response(JSON.stringify({ RepoSize: STORAGE_MAX, StorageMax: STORAGE_MAX })); + const fetchImpl = vi.fn(async () => { gcStarted.push(1); await new Promise((resolve) => (releaseGc = resolve)); return new Response("");