diff --git a/.changeset/agent-metadata-readmes.md b/.changeset/agent-metadata-readmes.md new file mode 100644 index 000000000..3fb3eb917 --- /dev/null +++ b/.changeset/agent-metadata-readmes.md @@ -0,0 +1,11 @@ +--- +'@walkeros/mcp': patch +'@walkeros/mcp-source-browser': patch +'@walkeros/cli': patch +'@walkeros/transformer-validate': patch +'@walkeros/server-transformer-fingerprint': patch +--- + +Add package READMEs and npm keywords. The MCP packages now ship install +instructions for Claude Code, Cursor, and VS Code plus MCP registry metadata +(mcpName). diff --git a/.changeset/cli-path-authoritative-pin.md b/.changeset/cli-path-authoritative-pin.md new file mode 100644 index 000000000..2fed58c40 --- /dev/null +++ b/.changeset/cli-path-authoritative-pin.md @@ -0,0 +1,7 @@ +--- +'@walkeros/cli': patch +--- + +A `path` entry in `config.bundle.packages` now counts as the package pin, so +bundling and setup use the local package instead of failing when steps still +declare conflicting inline versions of it. diff --git a/.changeset/cli-unified-setup-resolution.md b/.changeset/cli-unified-setup-resolution.md new file mode 100644 index 000000000..2c39b6bbb --- /dev/null +++ b/.changeset/cli-unified-setup-resolution.md @@ -0,0 +1,8 @@ +--- +'@walkeros/cli': minor +--- + +`walkeros setup` now resolves a component's package the same way +`walkeros bundle` does: the flow's pinned version is downloaded from the +registry (sharing the bundle cache) and imported from there. Setup works via npx +without a local install; `path:` packages are supported for local development. diff --git a/.changeset/express-async-per-method.md b/.changeset/express-async-per-method.md new file mode 100644 index 000000000..c650152c0 --- /dev/null +++ b/.changeset/express-async-per-method.md @@ -0,0 +1,13 @@ +--- +'@walkeros/server-source-express': minor +'@walkeros/core': minor +--- + +The Express source's `async` option now resolves per HTTP method: a boolean +still applies to the whole source, and a record like `{ "GET": true }` or +`{ "POST": false }` overrides one method while the other keeps its default. The +default changed: GET is now synchronous, so a step such as the file transformer +or a cache can serve real content instead of the tracking GIF, while POST keeps +the fast respond-first acknowledgement. To restore respond-first GET set +`async: true` or `async: { "GET": true }`; configs that set `async: false` only +to fix asset serving can drop it. diff --git a/.changeset/gtag-gtm-bootstrap-push.md b/.changeset/gtag-gtm-bootstrap-push.md new file mode 100644 index 000000000..98ed183c9 --- /dev/null +++ b/.changeset/gtag-gtm-bootstrap-push.md @@ -0,0 +1,8 @@ +--- +'@walkeros/web-destination-gtag': minor +--- + +The GTM destination no longer pushes the `gtm.js` start event into the dataLayer +unless it loads the container itself. Set `loadScript: true` to have walkerOS +install GTM, start event included. Without it, walkerOS only pushes your mapped +events, so an existing container's triggers are left alone. diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json new file mode 100644 index 000000000..272d2d032 --- /dev/null +++ b/.claude-plugin/marketplace.json @@ -0,0 +1,15 @@ +{ + "name": "elbwalker", + "owner": { + "name": "elbwalker", + "url": "https://github.com/elbwalker" + }, + "description": "Claude Code plugins for walkerOS, the open-source event data collection platform.", + "plugins": [ + { + "name": "walkeros", + "source": "./", + "description": "Skills and MCP tools for walkerOS - build sources, destinations, transformers, and flows." + } + ] +} diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 03b11c034..509eb348f 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -2,7 +2,10 @@ "name": "walkeros", "version": "0.2.0", "description": "Skills and MCP tools for walkerOS - the open-source event data collection platform. Learn architecture, create sources/destinations/transformers, validate events, bundle flows, and manage projects via API.", - "author": "elbwalker", + "author": { + "name": "elbwalker", + "url": "https://github.com/elbwalker" + }, "repository": "https://github.com/elbwalker/walkerOS", "license": "MIT", "keywords": [ @@ -21,28 +24,28 @@ "mcp" ], "skills": [ - "skills/walkeros-understanding-events", - "skills/walkeros-understanding-mapping", - "skills/walkeros-understanding-flow", - "skills/walkeros-understanding-destinations", - "skills/walkeros-understanding-sources", - "skills/walkeros-understanding-transformers", - "skills/walkeros-understanding-stores", - "skills/walkeros-understanding-development", - "skills/walkeros-using-logger", - "skills/walkeros-using-cli", - "skills/walkeros-using-store-cache", - "skills/walkeros-using-transformer-ga4", - "skills/walkeros-mcp-actions", - "skills/walkeros-create-cmp-source", - "skills/walkeros-create-destination", - "skills/walkeros-create-source", - "skills/walkeros-create-transformer", - "skills/walkeros-mapping-configuration", - "skills/walkeros-testing-strategy", - "skills/walkeros-debugging", - "skills/walkeros-writing-documentation", - "skills/walkeros-using-step-examples" + "./skills/walkeros-understanding-events", + "./skills/walkeros-understanding-mapping", + "./skills/walkeros-understanding-flow", + "./skills/walkeros-understanding-destinations", + "./skills/walkeros-understanding-sources", + "./skills/walkeros-understanding-transformers", + "./skills/walkeros-understanding-stores", + "./skills/walkeros-understanding-development", + "./skills/walkeros-using-logger", + "./skills/walkeros-using-cli", + "./skills/walkeros-using-store-cache", + "./skills/walkeros-using-transformer-ga4", + "./skills/walkeros-mcp-actions", + "./skills/walkeros-create-cmp-source", + "./skills/walkeros-create-destination", + "./skills/walkeros-create-source", + "./skills/walkeros-create-transformer", + "./skills/walkeros-mapping-configuration", + "./skills/walkeros-testing-strategy", + "./skills/walkeros-debugging", + "./skills/walkeros-writing-documentation", + "./skills/walkeros-using-step-examples" ], "mcpServers": { "walkeros-flow": { diff --git a/README.md b/README.md index 1f7af323c..847f7783f 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,36 @@ walkerOS exposes a Model Context Protocol (MCP) interface. AI agents can read your event schema, suggest tracking definitions, and generate integration code - making your event layer programmable, not just configurable. +In Claude Code, one plugin installs both MCP servers and the walkerOS skills: + +``` +/plugin marketplace add elbwalker/walkerOS +/plugin install walkeros@elbwalker +``` + +For any other MCP client, add the servers to its configuration: + +```json +{ + "mcpServers": { + "walkeros-flow": { + "command": "npx", + "args": ["@walkeros/mcp"] + }, + "walkeros-source-browser": { + "command": "npx", + "args": ["@walkeros/mcp-source-browser"] + } + } +} +``` + +Loading a flow, validating it, and simulating an event all run locally, no +account needed. See the [MCP docs](https://www.walkeros.io/docs/apps/mcp). + +Coming from Google Tag Manager? See +[walkerOS vs. GTM](https://www.walkeros.io/docs/comparisons/gtm). + ## Contributing ⭐️ Help us grow and star us. See our diff --git a/apps/scripts/generate-skills-index.ts b/apps/scripts/generate-skills-index.ts index 2097028ce..878858efc 100644 --- a/apps/scripts/generate-skills-index.ts +++ b/apps/scripts/generate-skills-index.ts @@ -36,7 +36,9 @@ function parseFrontmatter(input: string): Record { if (top && !indented) { flush(); key = top[1]; - parts.push(top[2]); + // A block-scalar indicator (">-", "|", ...) is syntax, not value; the + // value is the indented continuation lines that follow. + if (!/^[>|][+-]?$/.test(top[2].trim())) parts.push(top[2]); } else if (key !== null) { parts.push(raw.trim()); } diff --git a/apps/scripts/validate-llms.ts b/apps/scripts/validate-llms.ts index 43a220189..65b92423d 100644 --- a/apps/scripts/validate-llms.ts +++ b/apps/scripts/validate-llms.ts @@ -27,15 +27,42 @@ const BUILD_DIR = join(ROOT, 'website', 'build'); // production builds resolve identically. const BASE_URL = (process.env.DOCUSAURUS_BASEURL || '/').replace(/\/*$/, '/'); +// The export stamps the site url on every link (`content.relativePaths: false`) +// so llms.txt survives being pasted into a model's context or chunked away from +// the site. Read that url from the Docusaurus config rather than repeating it, +// so the guard and the build cannot disagree on what "absolute" means. +function readSiteUrl(): string { + const config = readFileSync( + join(ROOT, 'website', 'docusaurus.config.ts'), + 'utf-8', + ); + // Top-level config key, two-space indented, so a nested plugin option named + // `url` can never be picked up instead. + const match = /^ {2}url: '([^']+)'/m.exec(config); + if (!match) { + console.log( + '❌ LLM export validation failed: no site `url` in website/docusaurus.config.ts.\n', + ); + process.exit(1); + } + return match[1].replace(/\/+$/, ''); +} + +const SITE_URL = readSiteUrl(); + function emittedPath(target: string): string { + const pathname = target.startsWith(SITE_URL) + ? target.slice(SITE_URL.length) + : target; const relativeTarget = - BASE_URL !== '/' && target.startsWith(BASE_URL) - ? target.slice(BASE_URL.length) - : target.replace(/^\//, ''); + BASE_URL !== '/' && pathname.startsWith(BASE_URL) + ? pathname.slice(BASE_URL.length) + : pathname.replace(/^\//, ''); return join(BUILD_DIR, relativeTarget); } const LLMS_INDEX = join(BUILD_DIR, 'llms.txt'); const LLMS_FULL = join(BUILD_DIR, 'llms-full.txt'); +const SITEMAP = join(BUILD_DIR, 'sitemap.xml'); // How many issues of one kind to print before summarising the rest. const MAX_REPORTED_PER_CHECK = 15; @@ -47,27 +74,37 @@ interface LinkRef { line: number; } -// Extract the link targets from `[text](target)` pairs, keeping only the -// root-relative Markdown pages the plugin emits (e.g. `/docs/...md`, -// `/docs.md`). External URLs, anchors, and non-.md targets are not part of the -// export contract this guard protects. -function extractMarkdownLinks(content: string): LinkRef[] { +// Every `[text](target)` pair in the document, targets kept verbatim. +function extractLinks(content: string): LinkRef[] { const links: LinkRef[] = []; const regex = /\[[^\]]*\]\(([^)]+)\)/g; - const lines = content.split('\n'); - lines.forEach((line, index) => { + content.split('\n').forEach((line, index) => { let match: RegExpExecArray | null; regex.lastIndex = 0; while ((match = regex.exec(line)) !== null) { - const target = match[1].split('#')[0].split('?')[0]; - if (target.startsWith('/') && target.endsWith('.md')) { - links.push({ target, line: index + 1 }); - } + links.push({ target: match[1], line: index + 1 }); } }); return links; } +// The links pointing at the Markdown pages the plugin emits. Targets carry the +// site url (`https://site/docs/...md`); a root-relative one still counts here so +// a stale link gets reported rather than skipped. Off-site URLs, anchors, and +// non-.md targets are not part of the export contract this guard protects. +function extractMarkdownLinks(content: string): LinkRef[] { + return extractLinks(content) + .map(({ target, line }) => ({ + target: target.split('#')[0].split('?')[0], + line, + })) + .filter( + ({ target }) => + (target.startsWith('/') || target.startsWith(`${SITE_URL}/`)) && + target.endsWith('.md'), + ); +} + // Every emitted Markdown page, relative to website/build. function collectMarkdownPages(dir: string, acc: string[] = []): string[] { for (const entry of readdirSync(dir, { withFileTypes: true })) { @@ -114,6 +151,91 @@ function checkIndexLinks(): void { } } +// llms.txt is read detached from the site: pasted into a model's context, split +// into chunks by a retriever, fetched on its own. A root-relative target has no +// document to resolve against there, so every link the index hands out has to +// carry the origin or it is dead on arrival. +function checkIndexLinksAbsolute(): void { + // Absolute links are a production-only guarantee. Preview builds emit + // relative links on purpose: the llms-txt plugin appends the baseUrl to the + // site url while route paths already carry it, doubling any non-root prefix + // (see the plugin block in website/docusaurus.config.ts). + if (BASE_URL !== '/') return; + const content = readFileSync(LLMS_INDEX, 'utf-8'); + const found = extractLinks(content) + .filter(({ target }) => !target.startsWith('https://')) + .map(({ target, line }) => ({ + file: `website/build/llms.txt:${line}`, + message: `relative link target ${target} (llms.txt links must be fully qualified, e.g. ${SITE_URL}${target.startsWith('/') ? target : `/${target}`})`, + })); + pushCapped('relative links in llms.txt', found); +} + +// Internal planning documents live under website/docs/plans/. One left there is +// published: it gets a public page, a sitemap entry, and its own section in +// llms.txt, which is the copy every agent reads. Deleting the source file fixes +// one leak; asserting on the published artifacts prevents the next one. +const INTERNAL_ROUTE = /[^\s<>"'()[\]]*\/docs\/plans(?![\w-])[^\s<>"'()[\]]*/g; + +function findInternalRoutes(content: string): LinkRef[] { + const found: LinkRef[] = []; + let match: RegExpExecArray | null; + INTERNAL_ROUTE.lastIndex = 0; + while ((match = INTERNAL_ROUTE.exec(content)) !== null) { + found.push({ + target: match[0], + line: content.slice(0, match.index).split('\n').length, + }); + } + return found; +} + +function checkNoInternalPlans(): void { + const found: Issue[] = []; + + // The index renders one heading per route category, so a "plans" section + // means the whole directory shipped, not a single stray page. + readFileSync(LLMS_INDEX, 'utf-8') + .split('\n') + .forEach((line, index) => { + if (!/^#{1,6}\s+plans\s*$/i.test(line.trim())) return; + found.push({ + file: `website/build/llms.txt:${index + 1}`, + message: `internal "${line.trim()}" section published in the LLM export (delete the page from website/docs/plans/)`, + }); + }); + + const artifacts = [ + { rel: 'website/build/llms.txt', abs: LLMS_INDEX, optional: false }, + { rel: 'website/build/llms-full.txt', abs: LLMS_FULL, optional: true }, + { rel: 'website/build/sitemap.xml', abs: SITEMAP, optional: false }, + ]; + + for (const { rel, abs, optional } of artifacts) { + if (!existsSync(abs)) { + // A guard whose input is missing is a guard that is switched off. + if (!optional) { + issues.push({ + file: rel, + message: + 'not emitted, so the internal-plans check could not run against it', + }); + } + continue; + } + for (const { target, line } of findInternalRoutes( + readFileSync(abs, 'utf-8'), + )) { + found.push({ + file: `${rel}:${line}`, + message: `internal plans route published: ${target} (delete the page from website/docs/plans/)`, + }); + } + } + + pushCapped('published internal plans', found); +} + // Every internal Markdown link in every emitted page body must resolve to a // file that was actually emitted. A link the export mangles (most commonly a // trailing-slash route turned into `/docs/mapping/.md`) 404s for every reader @@ -370,6 +492,8 @@ function main(): void { console.log(` ${pages.length} exported pages\n`); checkIndexLinks(); + checkIndexLinksAbsolute(); + checkNoInternalPlans(); checkBodyLinks(pages); checkPlaceholders(pages); checkCodeIndentation(); diff --git a/apps/scripts/validate-skills-index.ts b/apps/scripts/validate-skills-index.ts index c1ce66e9a..bc204330b 100644 --- a/apps/scripts/validate-skills-index.ts +++ b/apps/scripts/validate-skills-index.ts @@ -35,8 +35,9 @@ function readPluginSkillNames(): string[] { skills?: string[]; }; const skills = plugin.skills ?? []; - // Entries look like "skills/walkeros-using-cli"; reduce to the basename. - return skills.map((entry) => entry.replace(/^skills\//, '')); + // Entries look like "./skills/walkeros-using-cli" (the plugin schema + // requires the "./" prefix); reduce to the basename. + return skills.map((entry) => entry.replace(/^(\.\/)?skills\//, '')); } // 1. Regenerate-and-compare: a hand edit to INDEX.json that was not produced by diff --git a/packages/cli/package.json b/packages/cli/package.json index b8cd342a2..32b921b23 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -95,5 +95,21 @@ "directory": "packages/cli" }, "author": "elbwalker ", - "homepage": "https://github.com/elbwalker/walkerOS#readme" + "homepage": "https://github.com/elbwalker/walkerOS#readme", + "keywords": [ + "walkerOS", + "cli", + "flow", + "analytics", + "event-tracking", + "event-data", + "data-collection", + "tag-management", + "first-party-data", + "privacy", + "privacy-first", + "consent", + "bundler", + "mcp" + ] } diff --git a/packages/cli/src/__tests__/integration/setup/setup-loader.integration.test.ts b/packages/cli/src/__tests__/integration/setup/setup-loader.integration.test.ts new file mode 100644 index 000000000..4a1530316 --- /dev/null +++ b/packages/cli/src/__tests__/integration/setup/setup-loader.integration.test.ts @@ -0,0 +1,99 @@ +import path from 'path'; +import os from 'os'; +import fs from 'fs-extra'; +import type { Flow } from '@walkeros/core'; +import { loadStepPackage } from '../../../core/step-loader.js'; +import { createMockLogger } from '../../helpers/mock-logger.js'; + +jest.setTimeout(120_000); + +const logger = createMockLogger(); +const REGISTRY_PIN = '4.4.0'; + +describe('setup package acquisition (integration)', () => { + let workDir: string; + + beforeEach(async () => { + workDir = await fs.mkdtemp(path.join(os.tmpdir(), 'setup-int-')); + }); + + afterEach(async () => { + await fs.remove(workDir); + }); + + it('acquires a path: package offline and runs its setup', async () => { + const pkgDir = path.join(workDir, 'fixture-dest'); + await fs.ensureDir(pkgDir); + await fs.writeJson(path.join(pkgDir, 'package.json'), { + name: '@walkeros/fixture-dest', + version: '1.0.0', + main: './index.mjs', + }); + await fs.writeFile( + path.join(pkgDir, 'index.mjs'), + 'export default { type: "fixture", push: () => {}, setup: async () => ({ provisioned: true }) };\n', + ); + + const flow: Flow = { + config: { + platform: 'server', + bundle: { packages: { '@walkeros/fixture-dest': { path: pkgDir } } }, + }, + destinations: { + d: { package: '@walkeros/fixture-dest', config: { setup: true } }, + }, + }; + + const loaded = await loadStepPackage(flow, 'destination', 'd', { + configDir: workDir, + logger, + }); + try { + const def = loaded.module.default; + // Narrow without casts, mirroring isComponentDefault in setup/index.ts + expect(def).toBeDefined(); + if (def === null || typeof def !== 'object' || !('setup' in def)) { + throw new Error('fixture default export lost its setup function'); + } + const setupFn = def.setup; + if (typeof setupFn !== 'function') { + throw new Error('setup is not callable'); + } + await expect( + setupFn({ id: 'd', config: {}, env: {}, logger }), + ).resolves.toEqual({ provisioned: true }); + } finally { + await fs.remove(loaded.installDir); + } + }); + + it('downloads the pinned version from the registry and imports it', async () => { + const flow: Flow = { + config: { + platform: 'web', + bundle: { + packages: { + '@walkeros/web-destination-api': { version: REGISTRY_PIN }, + }, + }, + }, + destinations: { + api: { package: '@walkeros/web-destination-api', config: {} }, + }, + }; + + const loaded = await loadStepPackage(flow, 'destination', 'api', { + configDir: workDir, + logger, + }); + try { + const manifest = await fs.readJson( + path.join(loaded.packageDir, 'package.json'), + ); + expect(manifest.version).toBe(REGISTRY_PIN); + expect(loaded.module.default).toBeDefined(); + } finally { + await fs.remove(loaded.installDir); + } + }); +}); diff --git a/packages/cli/src/commands/bundle/__tests__/bundler.test.ts b/packages/cli/src/commands/bundle/__tests__/bundler.test.ts index 218237c45..63134997d 100644 --- a/packages/cli/src/commands/bundle/__tests__/bundler.test.ts +++ b/packages/cli/src/commands/bundle/__tests__/bundler.test.ts @@ -30,8 +30,8 @@ jest.mock('../nft-trace', () => { traceAndCopy: jest.fn(actual.traceAndCopy), }; }); -jest.mock('../package-manager.js', () => { - const actual = jest.requireActual('../package-manager.js'); +jest.mock('../../../core/package-manager.js', () => { + const actual = jest.requireActual('../../../core/package-manager.js'); return { ...actual, downloadPackagesWithResolution: jest.fn( @@ -49,7 +49,7 @@ jest.mock('../../../core/build-cache.js', () => { }); import { traceAndCopy } from '../nft-trace'; -import { downloadPackagesWithResolution } from '../package-manager.js'; +import { downloadPackagesWithResolution } from '../../../core/package-manager.js'; import { getCachedBuild } from '../../../core/build-cache.js'; const mockTraceAndCopy = traceAndCopy as jest.MockedFunction< diff --git a/packages/cli/src/commands/bundle/__tests__/nft-regression.test.ts b/packages/cli/src/commands/bundle/__tests__/nft-regression.test.ts index c1900e47d..4c891d78a 100644 --- a/packages/cli/src/commands/bundle/__tests__/nft-regression.test.ts +++ b/packages/cli/src/commands/bundle/__tests__/nft-regression.test.ts @@ -47,8 +47,8 @@ jest.mock('../nft-trace', () => { traceAndCopy: jest.fn(actual.traceAndCopy), }; }); -jest.mock('../package-manager.js', () => { - const actual = jest.requireActual('../package-manager.js'); +jest.mock('../../../core/package-manager.js', () => { + const actual = jest.requireActual('../../../core/package-manager.js'); return { ...actual, downloadPackagesWithResolution: jest.fn( @@ -58,7 +58,7 @@ jest.mock('../package-manager.js', () => { }); import { traceAndCopy } from '../nft-trace'; -import { downloadPackagesWithResolution } from '../package-manager.js'; +import { downloadPackagesWithResolution } from '../../../core/package-manager.js'; const mockTraceAndCopy = traceAndCopy as jest.MockedFunction< typeof traceAndCopy diff --git a/packages/cli/src/commands/bundle/__tests__/package-manager-resilience.test.ts b/packages/cli/src/commands/bundle/__tests__/package-manager-resilience.test.ts index 3d21ddb42..845b0222f 100644 --- a/packages/cli/src/commands/bundle/__tests__/package-manager-resilience.test.ts +++ b/packages/cli/src/commands/bundle/__tests__/package-manager-resilience.test.ts @@ -4,7 +4,7 @@ import { extractWithResilience, manifestWithResilience, PACOTE_RETRY_ATTEMPTS, -} from '../package-manager'; +} from '../../../core/package-manager'; /** * Build a fully-typed pacote manifest result so tests never cast. The shape is diff --git a/packages/cli/src/commands/bundle/__tests__/range-resolution.test.ts b/packages/cli/src/commands/bundle/__tests__/range-resolution.test.ts index 675568d1b..6018b530d 100644 --- a/packages/cli/src/commands/bundle/__tests__/range-resolution.test.ts +++ b/packages/cli/src/commands/bundle/__tests__/range-resolution.test.ts @@ -3,7 +3,7 @@ import { collectAllSpecs, type VersionSpec, type Package, -} from '../package-manager'; +} from '../../../core/package-manager'; import { applyStepPackages } from '../bundler'; import type { BuildOptions } from '../../../types/bundle.js'; import { createMockLogger } from '@walkeros/core'; diff --git a/packages/cli/src/commands/bundle/bundler.ts b/packages/cli/src/commands/bundle/bundler.ts index 3bd6f83b0..539dc107a 100644 --- a/packages/cli/src/commands/bundle/bundler.ts +++ b/packages/cli/src/commands/bundle/bundler.ts @@ -30,6 +30,23 @@ export { validateStoreReferences, } from './structural-validators.js'; +import { + parsePackageSpec, + detectStepPackages, + collectAllStepPackages, + applyStepPackages, +} from '../../core/step-packages.js'; + +// Re-export so existing import sites (tests, public entry) keep resolving +// them from './bundler'. Implementations live in core/step-packages so they +// can run without loading esbuild. +export { + parsePackageSpec, + detectStepPackages, + collectAllStepPackages, + applyStepPackages, +}; + /** * Type guard to check if a code value is an InlineCode object. * InlineCode has { push: string, type?: string, init?: string } @@ -43,36 +60,6 @@ function isInlineCode(code: unknown): code is Flow.Code { ); } -/** - * Type-narrowed accessor for a Flow section. Returns the typed step record - * (or undefined) — exhaustive switch over the literal-union parameter avoids - * a generic indexed-access cast. - * - * Returns the union of all section types when the caller passes a runtime - * variable. Call sites that need a specific section type access the field - * directly (e.g. `flow.sources`). - */ -type FlowStepRecord = - | Record - | Record - | Record - | Record; -function getFlowSection( - flow: Flow, - section: 'sources' | 'destinations' | 'transformers' | 'stores', -): FlowStepRecord | undefined { - switch (section) { - case 'sources': - return flow.sources; - case 'destinations': - return flow.destinations; - case 'transformers': - return flow.transformers; - case 'stores': - return flow.stores; - } -} - /** * A reference carries inline code when it is an InlineCode object or (legacy) * a string. Used by the codegen step-filtering below. @@ -145,7 +132,7 @@ import type { BuildOptions } from '../../types/bundle.js'; import { downloadPackagesWithResolution, loadNpmConfigForPacote, -} from './package-manager.js'; +} from '../../core/package-manager.js'; import { traceAndCopy, assertDepsTraced } from './nft-trace.js'; import { assertConsumerDepsSatisfied } from './assert-consumer-deps.js'; import type { Logger } from '@walkeros/core'; @@ -888,34 +875,6 @@ function createEsbuildOptions( return baseOptions; } -/** - * Detects destination packages from flow configuration. - * Extracts package names from destinations that have explicit 'package' field. - */ -/** - * Detects packages from a flow config section (sources, destinations, transformers, stores). - * Extracts package names from steps that have an explicit 'package' field. - */ -export function detectStepPackages( - flowSettings: Flow, - section: 'sources' | 'destinations' | 'transformers' | 'stores', -): Set { - const packages = new Set(); - const steps = getFlowSection(flowSettings, section); - - if (steps) { - for (const [, stepConfig] of Object.entries(steps)) { - if (typeof stepConfig !== 'object' || stepConfig === null) continue; - // Require explicit package field - if (typeof stepConfig.package === 'string') { - packages.add(stepConfig.package); - } - } - } - - return packages; -} - /** * Get the complete list of Node.js built-in modules for esbuild external config. * Includes bare names, node: prefixed, and subpath patterns. @@ -1029,141 +988,6 @@ async function runNftServerPath( ); } -/** - * Collects all package names declared in flow steps. - * Returns both npm packages and local paths — caller handles routing. - */ -export function collectAllStepPackages(flowSettings: Flow): Set { - const allPackages = new Set(); - const sections = [ - 'sources', - 'destinations', - 'transformers', - 'stores', - ] as const; - - for (const section of sections) { - for (const pkg of detectStepPackages(flowSettings, section)) { - allPackages.add(pkg); - } - } - - return allPackages; -} - -/** - * Auto-adds every step-declared package (sources, destinations, transformers, - * stores) to `packages`, mutating `flowSettings` in place so each step's - * `package` field points at the key that ends up in `packages`. - * - * Local paths (`.` or `/` prefixed) are normalized to a synthetic - * `packageNameToVariable` key so the regular default-import codegen wires - * them up automatically. - * - * npm specs go through `parsePackageSpec` to split an inline version - * (`@walkeros/x@1.2.3`) from the bare name. Precedence policy: - * - An explicit `config.bundle.packages` version always wins; a disagreeing - * inline version only warns (bundle pin is authoritative). - * - An unversioned bundle entry is filled from the first inline version seen - * for that bare name, preserving any other fields already on the entry. - * - Two different inline versions for the same bare name, with no bundle - * pin to arbitrate, are ambiguous — throw naming both steps' versions - * rather than silently picking one. - * - Identical inline versions across steps are fine (no-op). - * Alias/git/file suffixes are not special-cased: `parsePackageSpec` splits on - * the last `@` and the resolver handles or rejects the rest downstream. - */ -export function applyStepPackages( - flowSettings: Flow, - packages: BuildOptions['packages'], - logger: Logger.Instance, -): void { - const stepPackages = collectAllStepPackages(flowSettings); - // Bundle-pinned version per bare name, captured the first time each name is - // encountered — i.e. before this function's own fill-ins can be mistaken - // for a real `config.bundle.packages` pin on a later iteration. - const originalVersions = new Map(); - // Inline version already seen per bare name (with no real bundle pin), - // used to detect a second, different inline version for the same name. - const inlineSeen = new Map(); - - const rewriteSteps = (from: string, to: string): void => { - for (const section of [ - 'sources', - 'destinations', - 'transformers', - 'stores', - ] as const) { - const steps = getFlowSection(flowSettings, section); - if (!steps) continue; - for (const step of Object.values(steps)) { - if (step.package === from) { - step.package = to; - } - } - } - }; - - for (const pkg of stepPackages) { - const isLocalPath = pkg.startsWith('.') || pkg.startsWith('/'); - - if (isLocalPath) { - // Normalize: convert path-based package: to packages section entry. - // The synthetic key acts as the package name for downstream codegen, - // so the regular default-import flow wires it up automatically. - const varName = packageNameToVariable(pkg); - if (!packages[varName]) { - packages[varName] = { - path: pkg, - }; - } - - // Rewrite all components that reference the raw path to point at the - // synthetic packages-section key instead. - rewriteSteps(pkg, varName); - continue; - } - - const { name, version } = parsePackageSpec(pkg); - - if (!originalVersions.has(name)) { - originalVersions.set(name, packages[name]?.version); - } - const bundlePinnedVersion = originalVersions.get(name); - - if (name !== pkg) { - // Rewrite every step that declared the versioned spec to the bare - // name, mirroring the local-path rewrite above. - rewriteSteps(pkg, name); - } - - if (version && !bundlePinnedVersion) { - const seen = inlineSeen.get(name); - if (seen !== undefined && seen !== version) { - throw new Error( - `Conflicting inline versions for ${name}: "${seen}" and "${version}" are ` + - `declared by different steps. Pin one version in config.bundle.packages.`, - ); - } - inlineSeen.set(name, version); - } - - const existing = packages[name]; - if (!existing) { - packages[name] = version ? { version } : {}; - } else if (version) { - if (!existing.version) { - existing.version = version; // fill an unversioned bundle entry - } else if (existing.version !== version) { - logger.warn( - `Package ${name}: config.bundle.packages pins ${existing.version}; ` + - `a step declares ${version} inline. Using the bundle pin.`, - ); - } - } - } -} - /** * Detects named-import requests across sources, destinations, transformers, stores. * Returns a map of package names to sets of export names that must be named-imported. @@ -1337,25 +1161,6 @@ export async function computeDevPackages( /** Package name of the browser source, the single writer of window[settings.elb]. */ const BROWSER_SOURCE_PACKAGE = '@walkeros/web-source-browser'; -/** - * Split a step package spec into bare name and optional version suffix. - * `@walkeros/x@1.2.3` → { name: '@walkeros/x', version: '1.2.3' }. - * The scope `@` at index 0 is never a separator. A trailing `@` is ignored. - * Alias/git/file suffixes are not interpreted — the resolver handles or - * rejects them downstream. - */ -export function parsePackageSpec(spec: string): { - name: string; - version?: string; -} { - const at = spec.lastIndexOf('@'); - if (at <= 0) return { name: spec }; - const version = spec.slice(at + 1); - return version - ? { name: spec.slice(0, at), version } - : { name: spec.slice(0, at) }; -} - /** * Strip an optional version/range suffix from a package spec so * `@walkeros/web-source-browser@2.0.0` matches the bare package name. diff --git a/packages/cli/src/commands/setup/__tests__/lifecycle.test.ts b/packages/cli/src/commands/setup/__tests__/lifecycle.test.ts index 4a22a01f1..cce5cd6ee 100644 --- a/packages/cli/src/commands/setup/__tests__/lifecycle.test.ts +++ b/packages/cli/src/commands/setup/__tests__/lifecycle.test.ts @@ -1,6 +1,10 @@ +import path from 'path'; +import os from 'os'; +import fs from 'fs-extra'; import type { Flow, Logger } from '@walkeros/core'; import { setupCommand } from '../index'; import { loadFlowConfig } from '../../../config/loader.js'; +import { loadStepPackage } from '../../../core/step-loader.js'; import { createMockLogger } from '../../../__tests__/helpers/mock-logger.js'; jest.mock('../../../config/loader.js'); @@ -8,6 +12,11 @@ jest.mock('../../../core/output.js', () => { const actual = jest.requireActual('../../../core/output.js'); return { ...actual, writeResult: jest.fn().mockResolvedValue(undefined) }; }); +jest.mock('../../../core/step-loader.js', () => ({ + loadStepPackage: jest.fn(), +})); + +const mockedLoadStep = jest.mocked(loadStepPackage); /** * Bug G fixture: a destination that exposes the full lifecycle. Each @@ -18,120 +27,103 @@ jest.mock('../../../core/output.js', () => { type Call = { fn: 'init' | 'setup' | 'destroy'; config: unknown }; const calls: Call[] = []; -jest.mock( - '@walkeros/__test-lifecycle-destination', - () => ({ - __esModule: true, - default: { - type: 'lifecycle', - push: () => {}, - init: async (ctx: { config: unknown }) => { - calls.push({ fn: 'init', config: ctx.config }); - // Return a brand-new config object that setup/destroy must receive. - return { resolved: true, originalSettings: ctx.config }; - }, - setup: async (ctx: { config: unknown }) => { - calls.push({ fn: 'setup', config: ctx.config }); - return { ok: true }; - }, - destroy: async (ctx: { config: unknown }) => { - calls.push({ fn: 'destroy', config: ctx.config }); - }, +const lifecycleModule: Record = { + default: { + type: 'lifecycle', + push: () => {}, + init: async (ctx: { config: unknown }) => { + calls.push({ fn: 'init', config: ctx.config }); + // Return a brand-new config object that setup/destroy must receive. + return { resolved: true, originalSettings: ctx.config }; }, - }), - { virtual: true }, -); - -jest.mock( - '@walkeros/__test-init-void-destination', - () => ({ - __esModule: true, - default: { - type: 'init-void', - push: () => {}, - init: async (ctx: { config: { settings?: { mutated?: boolean } } }) => { - // Mutate input config in place; legal but rare path. Returning - // void must keep the original config object as the resolved one. - if (ctx.config && typeof ctx.config === 'object') { - ctx.config.settings = { - ...(ctx.config.settings ?? {}), - mutated: true, - }; - } - // No return statement → undefined. - }, - setup: async (ctx: { config: unknown }) => { - calls.push({ fn: 'setup', config: ctx.config }); - return { ok: true }; - }, - destroy: async (ctx: { config: unknown }) => { - calls.push({ fn: 'destroy', config: ctx.config }); - }, + setup: async (ctx: { config: unknown }) => { + calls.push({ fn: 'setup', config: ctx.config }); + return { ok: true }; }, - }), - { virtual: true }, -); - -jest.mock( - '@walkeros/__test-init-aborts-destination', - () => ({ - __esModule: true, - default: { - type: 'init-aborts', - push: () => {}, - init: async () => { - calls.push({ fn: 'init', config: undefined }); - return false; - }, - setup: async (ctx: { config: unknown }) => { - calls.push({ fn: 'setup', config: ctx.config }); - return { ok: true }; - }, - destroy: async (ctx: { config: unknown }) => { - calls.push({ fn: 'destroy', config: ctx.config }); - }, + destroy: async (ctx: { config: unknown }) => { + calls.push({ fn: 'destroy', config: ctx.config }); }, - }), - { virtual: true }, -); - -jest.mock( - '@walkeros/__test-no-init-destination', - () => ({ - __esModule: true, - default: { - type: 'no-init', - push: () => {}, - // No init, no destroy: setup-only path stays unchanged. - setup: async (ctx: { config: unknown }) => { - calls.push({ fn: 'setup', config: ctx.config }); - return { ok: true }; - }, + }, +}; + +const initVoidModule: Record = { + default: { + type: 'init-void', + push: () => {}, + init: async (ctx: { config: { settings?: { mutated?: boolean } } }) => { + // Mutate input config in place; legal but rare path. Returning + // void must keep the original config object as the resolved one. + if (ctx.config && typeof ctx.config === 'object') { + ctx.config.settings = { + ...(ctx.config.settings ?? {}), + mutated: true, + }; + } + // No return statement → undefined. }, - }), - { virtual: true }, -); + setup: async (ctx: { config: unknown }) => { + calls.push({ fn: 'setup', config: ctx.config }); + return { ok: true }; + }, + destroy: async (ctx: { config: unknown }) => { + calls.push({ fn: 'destroy', config: ctx.config }); + }, + }, +}; + +const initAbortsModule: Record = { + default: { + type: 'init-aborts', + push: () => {}, + init: async () => { + calls.push({ fn: 'init', config: undefined }); + return false; + }, + setup: async (ctx: { config: unknown }) => { + calls.push({ fn: 'setup', config: ctx.config }); + return { ok: true }; + }, + destroy: async (ctx: { config: unknown }) => { + calls.push({ fn: 'destroy', config: ctx.config }); + }, + }, +}; + +const noInitModule: Record = { + default: { + type: 'no-init', + push: () => {}, + // No init, no destroy: setup-only path stays unchanged. + setup: async (ctx: { config: unknown }) => { + calls.push({ fn: 'setup', config: ctx.config }); + return { ok: true }; + }, + }, +}; // Captures the literal config init received, so the env-marker resolver // test can verify markers were replaced before init was invoked. const envInitCalls: { config: unknown; env: unknown }[] = []; -jest.mock( - '@walkeros/__test-env-marker-destination', - () => ({ - __esModule: true, - default: { - type: 'env-marker', - push: () => {}, - init: async (ctx: { config: unknown; env: unknown }) => { - envInitCalls.push({ config: ctx.config, env: ctx.env }); - return { resolved: true }; - }, - setup: async (_ctx: unknown) => ({ ok: true }), - destroy: async () => {}, +const envMarkerModule: Record = { + default: { + type: 'env-marker', + push: () => {}, + init: async (ctx: { config: unknown; env: unknown }) => { + envInitCalls.push({ config: ctx.config, env: ctx.env }); + return { resolved: true }; }, - }), - { virtual: true }, -); + setup: async (_ctx: unknown) => ({ ok: true }), + destroy: async () => {}, + }, +}; + +const moduleFixtures: Record> = { + '@walkeros/__test-lifecycle-destination': lifecycleModule, + '@walkeros/__test-init-void-destination': initVoidModule, + '@walkeros/__test-init-aborts-destination': initAbortsModule, + '@walkeros/__test-no-init-destination': noInitModule, + '@walkeros/__test-env-marker-destination': envMarkerModule, +}; const mockedLoadFlowConfig = jest.mocked(loadFlowConfig); @@ -167,14 +159,32 @@ function mockLoad(flow: Flow): void { describe('setupCommand: init → setup → destroy lifecycle (Bug G)', () => { let logger: Logger.Instance; + let installDir: string; - beforeEach(() => { + beforeEach(async () => { calls.length = 0; logger = createMockLogger(); + installDir = await fs.mkdtemp(path.join(os.tmpdir(), 'setup-lc-install-')); + mockedLoadStep.mockImplementation(async (flow, _kind, id) => { + const pkg = flow.destinations?.[id]?.package; + const module = pkg !== undefined ? moduleFixtures[pkg] : undefined; + if (pkg === undefined || module === undefined) { + throw new Error(`no module fixture for package "${String(pkg)}"`); + } + return { + module, + packageName: pkg, + normalizedFlow: flow, + packageDir: path.join(installDir, 'node_modules', pkg), + installDir, + }; + }); }); - afterEach(() => { + afterEach(async () => { mockedLoadFlowConfig.mockReset(); + mockedLoadStep.mockReset(); + await fs.remove(installDir); }); test('runs init before setup, then destroy, with the resolved config flowing through', async () => { diff --git a/packages/cli/src/commands/setup/__tests__/setup.test.ts b/packages/cli/src/commands/setup/__tests__/setup.test.ts index 843a9a3d8..27464c967 100644 --- a/packages/cli/src/commands/setup/__tests__/setup.test.ts +++ b/packages/cli/src/commands/setup/__tests__/setup.test.ts @@ -1,6 +1,10 @@ +import path from 'path'; +import os from 'os'; +import fs from 'fs-extra'; import type { Flow, Logger } from '@walkeros/core'; import { setupCommand } from '../index'; import { loadFlowConfig } from '../../../config/loader.js'; +import { loadStepPackage } from '../../../core/step-loader.js'; import { createMockLogger } from '../../../__tests__/helpers/mock-logger.js'; import { writeResult } from '../../../core/output.js'; @@ -9,46 +13,34 @@ jest.mock('../../../core/output.js', () => { const actual = jest.requireActual('../../../core/output.js'); return { ...actual, writeResult: jest.fn().mockResolvedValue(undefined) }; }); +jest.mock('../../../core/step-loader.js', () => ({ + loadStepPackage: jest.fn(), +})); + +const mockedLoadStep = jest.mocked(loadStepPackage); const setupCalls: unknown[] = []; -// `__esModule: true` is required: dynamic `import()` on a CJS module wraps the -// whole module in `.default`. Marking ESM lets jest expose our `default` field -// as the resolved module's `default` directly, mirroring real packages compiled -// from `export default ...`. -jest.mock( - '@walkeros/__test-fake-destination', - () => ({ - __esModule: true, - default: { - type: 'fake', - push: () => {}, - setup: async (ctx: unknown) => { - setupCalls.push(ctx); - return { datasetCreated: true, tableCreated: false }; - }, +// Package fixtures: plain module namespaces the mocked step loader resolves +// to, mirroring what a real dynamic import of the extracted entry returns. +const fakeDestinationModule: Record = { + default: { + type: 'fake', + push: () => {}, + setup: async (ctx: unknown) => { + setupCalls.push(ctx); + return { datasetCreated: true, tableCreated: false }; }, - }), - { virtual: true }, -); - -jest.mock( - '@walkeros/__test-no-setup-destination', - () => ({ - __esModule: true, - default: { type: 'no-setup', push: () => {} }, - }), - { virtual: true }, -); - -jest.mock( - '@walkeros/__test-no-default-export', - () => ({ - __esModule: true, - Named: { type: 'noop', push: () => {} }, - }), - { virtual: true }, -); + }, +}; + +const noSetupDestinationModule: Record = { + default: { type: 'no-setup', push: () => {} }, +}; + +const noDefaultExportModule: Record = { + Named: { type: 'noop', push: () => {} }, +}; // Multi-export package fixture used to exercise the export-name resolver. // Default export simulates the package's "primary" component (componentA); @@ -56,37 +48,39 @@ jest.mock( // bundle.packages.imports[0]. const multiExportSetupCalls: { name: string; ctx: unknown }[] = []; -jest.mock( - '@walkeros/__test-multi-export', - () => ({ - __esModule: true, - componentA: { - type: 'a', - push: () => {}, - setup: async (ctx: unknown) => { - multiExportSetupCalls.push({ name: 'componentA', ctx }); - return { ran: 'A' }; - }, +const multiExportModule: Record = { + componentA: { + type: 'a', + push: () => {}, + setup: async (ctx: unknown) => { + multiExportSetupCalls.push({ name: 'componentA', ctx }); + return { ran: 'A' }; }, - componentB: { - type: 'b', - push: () => {}, - setup: async (ctx: unknown) => { - multiExportSetupCalls.push({ name: 'componentB', ctx }); - return { ran: 'B' }; - }, + }, + componentB: { + type: 'b', + push: () => {}, + setup: async (ctx: unknown) => { + multiExportSetupCalls.push({ name: 'componentB', ctx }); + return { ran: 'B' }; }, - default: { - type: 'a', - push: () => {}, - setup: async (ctx: unknown) => { - multiExportSetupCalls.push({ name: 'default', ctx }); - return { ran: 'default' }; - }, + }, + default: { + type: 'a', + push: () => {}, + setup: async (ctx: unknown) => { + multiExportSetupCalls.push({ name: 'default', ctx }); + return { ran: 'default' }; }, - }), - { virtual: true }, -); + }, +}; + +const moduleFixtures: Record> = { + '@walkeros/__test-fake-destination': fakeDestinationModule, + '@walkeros/__test-no-setup-destination': noSetupDestinationModule, + '@walkeros/__test-no-default-export': noDefaultExportModule, + '@walkeros/__test-multi-export': multiExportModule, +}; const baseFlowSettings: Flow = { config: { platform: 'server' }, @@ -119,6 +113,24 @@ const mockedWriteResult = writeResult as jest.MockedFunction< typeof writeResult >; +function mockLoad(flow: Flow): void { + mockedLoadFlowConfig.mockResolvedValue({ + flowSettings: flow, + buildOptions: { + format: 'esm', + target: 'node18', + platform: 'node', + output: './dist', + packages: {}, + minify: false, + sourcemap: false, + }, + flowName: 'default', + isMultiFlow: false, + availableFlows: ['default'], + }); +} + /** Pull the rendered string off a logger.info mock call. */ function infoMessages(logger: Logger.Instance): string[] { const fn = logger.info as jest.Mock; @@ -127,30 +139,35 @@ function infoMessages(logger: Logger.Instance): string[] { describe('setupCommand', () => { let logger: Logger.Instance; + let installDir: string; - beforeEach(() => { + beforeEach(async () => { setupCalls.length = 0; - mockedLoadFlowConfig.mockResolvedValue({ - flowSettings: baseFlowSettings, - buildOptions: { - format: 'esm', - target: 'node18', - platform: 'node', - output: './dist', - packages: {}, - minify: false, - sourcemap: false, - }, - flowName: 'default', - isMultiFlow: false, - availableFlows: ['default'], - }); + mockLoad(baseFlowSettings); mockedWriteResult.mockClear(); logger = createMockLogger(); + // Real temp dir so the cleanup-in-finally behavior is observable. + installDir = await fs.mkdtemp(path.join(os.tmpdir(), 'setup-install-')); + mockedLoadStep.mockImplementation(async (flow, _kind, id) => { + const pkg = flow.destinations?.[id]?.package; + const module = pkg !== undefined ? moduleFixtures[pkg] : undefined; + if (pkg === undefined || module === undefined) { + throw new Error(`no module fixture for package "${String(pkg)}"`); + } + return { + module, + packageName: pkg, + normalizedFlow: flow, + packageDir: path.join(installDir, 'node_modules', pkg), + installDir, + }; + }); }); - afterEach(() => { + afterEach(async () => { mockedLoadFlowConfig.mockReset(); + mockedLoadStep.mockReset(); + await fs.remove(installDir); }); test('invokes setup with component context and narrates start/ok', async () => { @@ -175,6 +192,28 @@ describe('setupCommand', () => { expect(mockedWriteResult).not.toHaveBeenCalled(); }); + test('loads the package through the shared step loader', async () => { + await setupCommand({ target: 'destination.fake', logger }); + expect(mockedLoadStep).toHaveBeenCalledTimes(1); + expect(mockedLoadStep).toHaveBeenCalledWith( + baseFlowSettings, + 'destination', + 'fake', + expect.objectContaining({ logger: expect.anything() }), + ); + }); + + test('removes the temp install tree after the lifecycle completes', async () => { + await setupCommand({ target: 'destination.fake', logger }); + expect(await fs.pathExists(installDir)).toBe(false); + }); + + test('removes the temp install tree on skip paths too', async () => { + await setupCommand({ target: 'destination.unset', logger }); + expect(setupCalls).toHaveLength(0); + expect(await fs.pathExists(installDir)).toBe(false); + }); + test.each([ { name: 'no setup function', @@ -239,29 +278,15 @@ describe('setupCommand', () => { }); test('routes to componentA when destinations.X.import = "componentA"', async () => { - mockedLoadFlowConfig.mockResolvedValue({ - flowSettings: { - config: { platform: 'server' }, - destinations: { - x: { - package: '@walkeros/__test-multi-export', - import: 'componentA', - config: { setup: true }, - }, + mockLoad({ + config: { platform: 'server' }, + destinations: { + x: { + package: '@walkeros/__test-multi-export', + import: 'componentA', + config: { setup: true }, }, }, - buildOptions: { - format: 'esm', - target: 'node18', - platform: 'node', - output: './dist', - packages: {}, - minify: false, - sourcemap: false, - }, - flowName: 'default', - isMultiFlow: false, - availableFlows: ['default'], }); await setupCommand({ target: 'destination.x', logger }); @@ -271,29 +296,15 @@ describe('setupCommand', () => { }); test('routes to componentB via explicit import field', async () => { - mockedLoadFlowConfig.mockResolvedValue({ - flowSettings: { - config: { platform: 'server' }, - destinations: { - x: { - package: '@walkeros/__test-multi-export', - import: 'componentB', - config: { setup: true }, - }, + mockLoad({ + config: { platform: 'server' }, + destinations: { + x: { + package: '@walkeros/__test-multi-export', + import: 'componentB', + config: { setup: true }, }, }, - buildOptions: { - format: 'esm', - target: 'node18', - platform: 'node', - output: './dist', - packages: {}, - minify: false, - sourcemap: false, - }, - flowName: 'default', - isMultiFlow: false, - availableFlows: ['default'], }); await setupCommand({ target: 'destination.x', logger }); @@ -303,37 +314,68 @@ describe('setupCommand', () => { }); test('routes to componentB via bundle.packages.imports[0] when import is unset', async () => { - mockedLoadFlowConfig.mockResolvedValue({ - flowSettings: { - config: { - platform: 'server', - bundle: { - packages: { - '@walkeros/__test-multi-export': { - imports: ['componentB'], - }, + mockLoad({ + config: { + platform: 'server', + bundle: { + packages: { + '@walkeros/__test-multi-export': { + imports: ['componentB'], }, }, }, - destinations: { - x: { - package: '@walkeros/__test-multi-export', - config: { setup: true }, + }, + destinations: { + x: { + package: '@walkeros/__test-multi-export', + config: { setup: true }, + }, + }, + }); + + await setupCommand({ target: 'destination.x', logger }); + + expect(multiExportSetupCalls).toHaveLength(1); + expect(multiExportSetupCalls[0].name).toBe('componentB'); + }); + + test('resolves the export name against the loader-normalized flow', async () => { + // The RAW flow carries no imports hint; only the normalizedFlow the + // loader returns does. componentB running proves the command resolves + // the export name against the loader's normalized flow, not the raw + // flowSettings (inline-versioned steps only match after normalization). + const rawFlow: Flow = { + config: { platform: 'server' }, + destinations: { + x: { + package: '@walkeros/__test-multi-export', + config: { setup: true }, + }, + }, + }; + const normalizedFlow: Flow = { + config: { + platform: 'server', + bundle: { + packages: { + '@walkeros/__test-multi-export': { imports: ['componentB'] }, }, }, }, - buildOptions: { - format: 'esm', - target: 'node18', - platform: 'node', - output: './dist', - packages: {}, - minify: false, - sourcemap: false, + destinations: { + x: { + package: '@walkeros/__test-multi-export', + config: { setup: true }, + }, }, - flowName: 'default', - isMultiFlow: false, - availableFlows: ['default'], + }; + mockLoad(rawFlow); + mockedLoadStep.mockResolvedValue({ + module: multiExportModule, + packageName: '@walkeros/__test-multi-export', + normalizedFlow, + packageDir: path.join(installDir, 'node_modules', 'multi'), + installDir, }); await setupCommand({ target: 'destination.x', logger }); @@ -343,28 +385,14 @@ describe('setupCommand', () => { }); test('falls back to default export when neither code nor imports set (back-compat)', async () => { - mockedLoadFlowConfig.mockResolvedValue({ - flowSettings: { - config: { platform: 'server' }, - destinations: { - x: { - package: '@walkeros/__test-multi-export', - config: { setup: true }, - }, + mockLoad({ + config: { platform: 'server' }, + destinations: { + x: { + package: '@walkeros/__test-multi-export', + config: { setup: true }, }, }, - buildOptions: { - format: 'esm', - target: 'node18', - platform: 'node', - output: './dist', - packages: {}, - minify: false, - sourcemap: false, - }, - flowName: 'default', - isMultiFlow: false, - availableFlows: ['default'], }); await setupCommand({ target: 'destination.x', logger }); @@ -374,29 +402,15 @@ describe('setupCommand', () => { }); test('throws actionable error when explicit import names a missing export', async () => { - mockedLoadFlowConfig.mockResolvedValue({ - flowSettings: { - config: { platform: 'server' }, - destinations: { - x: { - package: '@walkeros/__test-multi-export', - import: 'componentZ', - config: { setup: true }, - }, + mockLoad({ + config: { platform: 'server' }, + destinations: { + x: { + package: '@walkeros/__test-multi-export', + import: 'componentZ', + config: { setup: true }, }, }, - buildOptions: { - format: 'esm', - target: 'node18', - platform: 'node', - output: './dist', - packages: {}, - minify: false, - sourcemap: false, - }, - flowName: 'default', - isMultiFlow: false, - availableFlows: ['default'], }); await expect( @@ -405,37 +419,23 @@ describe('setupCommand', () => { }); test('throws actionable error when imports[0] names a missing export', async () => { - mockedLoadFlowConfig.mockResolvedValue({ - flowSettings: { - config: { - platform: 'server', - bundle: { - packages: { - '@walkeros/__test-multi-export': { - imports: ['componentZ'], - }, + mockLoad({ + config: { + platform: 'server', + bundle: { + packages: { + '@walkeros/__test-multi-export': { + imports: ['componentZ'], }, }, }, - destinations: { - x: { - package: '@walkeros/__test-multi-export', - config: { setup: true }, - }, - }, }, - buildOptions: { - format: 'esm', - target: 'node18', - platform: 'node', - output: './dist', - packages: {}, - minify: false, - sourcemap: false, + destinations: { + x: { + package: '@walkeros/__test-multi-export', + config: { setup: true }, + }, }, - flowName: 'default', - isMultiFlow: false, - availableFlows: ['default'], }); await expect( diff --git a/packages/cli/src/commands/setup/index.ts b/packages/cli/src/commands/setup/index.ts index 513585be6..d2f18fbb8 100644 --- a/packages/cli/src/commands/setup/index.ts +++ b/packages/cli/src/commands/setup/index.ts @@ -1,9 +1,11 @@ +import fs from 'fs-extra'; import type { Logger } from '@walkeros/core'; import { ENV_MARKER_PREFIX } from '@walkeros/core'; import { loadFlowConfig } from '../../config/loader.js'; import { createCLILogger } from '../../core/cli-logger.js'; import { createSuccessOutput, writeResult } from '../../core/output.js'; import { resolveExportName } from '../../core/resolve-export-name.js'; +import { loadStepPackage } from '../../core/step-loader.js'; import { resolveComponent } from './resolve.js'; export interface SetupCommandOptions { @@ -171,169 +173,189 @@ export async function setupCommand(opts: SetupCommandOptions): Promise { json: opts.json, }); - const { flowSettings } = await loadFlowConfig(opts.config ?? './flow.json', { - flowName: opts.flow, - }); + const { flowSettings, buildOptions } = await loadFlowConfig( + opts.config ?? './flow.json', + { flowName: opts.flow }, + ); const component = resolveComponent(flowSettings, opts.target); const scoped = baseLogger.scope(component.kind).scope(component.id); scoped.info(`setup: starting ${component.kind}.${component.id}`); - // Mirror bundle's resolution so multi-export packages (e.g. gcp exporting - // both destinationBigQuery and destinationPubSub) route to the right - // export instead of always grabbing the package default. - const { exportName, source: resolveSource } = resolveExportName( + // Acquire the package exactly like `walkeros bundle` does: the flow's + // version pin (or path) through the shared pacote pipeline, imported from + // the extracted tree. Never resolved against the CLI's own install. + const loaded = await loadStepPackage( flowSettings, component.kind, component.id, + { configDir: buildOptions.configDir, logger: scoped }, ); - const mod: Record = await import(component.packageName); - const pickedExport: unknown = - exportName !== undefined ? mod[exportName] : mod.default; + try { + // Mirror bundle's resolution so multi-export packages (e.g. gcp exporting + // both destinationBigQuery and destinationPubSub) route to the right + // export instead of always grabbing the package default. Resolve against + // the normalized flow so bundle.packages[pkg].imports matches even when + // the step declares an inline version. + const { exportName, source: resolveSource } = resolveExportName( + loaded.normalizedFlow, + component.kind, + component.id, + ); + + const mod = loaded.module; + const pickedExport: unknown = + exportName !== undefined ? mod[exportName] : mod.default; - if (!isComponentDefault(pickedExport)) { - if (exportName !== undefined) { - const origin = - resolveSource === 'import' - ? `${component.kind}.${component.id}.import` - : `bundle.packages["${component.packageName}"].imports[0]`; + if (!isComponentDefault(pickedExport)) { + if (exportName !== undefined) { + const origin = + resolveSource === 'import' + ? `${component.kind}.${component.id}.import` + : `bundle.packages["${loaded.packageName}"].imports[0]`; + throw new Error( + `Package ${loaded.packageName} has no export "${exportName}" ` + + `(referenced by ${origin}).`, + ); + } throw new Error( - `Package ${component.packageName} has no export "${exportName}" ` + - `(referenced by ${origin}).`, + `Package ${loaded.packageName} has no default export. ` + + `walkerOS components are expected to use 'export default'.`, ); } - throw new Error( - `Package ${component.packageName} has no default export. ` + - `walkerOS components are expected to use 'export default'.`, - ); - } - const initFn = pickedExport.init; - const setupFn = pickedExport.setup; - const destroyFn = pickedExport.destroy; + const initFn = pickedExport.init; + const setupFn = pickedExport.setup; + const destroyFn = pickedExport.destroy; - const emitSkipEnvelope = async (reason: string): Promise => { - if (!opts.json) return; - const envelope = createSuccessOutput( - { - kind: component.kind, - id: component.id, - status: 'skipped', - reason, - }, - Date.now() - startTime, - ); - await writeResult(JSON.stringify(envelope, null, 2) + '\n', {}); - }; - - if (!isLifecycleFn(setupFn)) { - // No setup defined on the package, narrate explicitly, exit ok. - scoped.info( - `setup: skipped ${component.kind}.${component.id} (no setup function)`, - ); - await emitSkipEnvelope('no setup function'); - return; - } - - // Honor config.setup explicitly. If user wrote `setup: false`, narrate and skip. - // (Omitted setup is also falsy and gets the same skip message.) - const setupConfig = readSetupField(component.config); - if (setupConfig === false || setupConfig === undefined) { - const reason = setupConfig === false ? 'false' : 'unset'; - scoped.info( - `setup: skipped ${component.kind}.${component.id} (config.setup is ${reason})`, - ); - await emitSkipEnvelope(`config.setup is ${reason}`); - return; - } - - // Server flows are loaded in deferred mode so the bundler can rewrite - // `$env.NAME` markers into `process.env[NAME]` expressions. The setup - // command imports the package and runs its lifecycle directly in this - // Node process, so markers must be replaced with their actual values - // here. Web flows are already eagerly resolved by `loadBundleConfig`, - // so this is a no-op for them. - const resolvedInputConfig = resolveEnvMarkers(component.config); - const resolvedEnv = resolveEnvMarkers(component.env); + const emitSkipEnvelope = async (reason: string): Promise => { + if (!opts.json) return; + const envelope = createSuccessOutput( + { + kind: component.kind, + id: component.id, + status: 'skipped', + reason, + }, + Date.now() - startTime, + ); + await writeResult(JSON.stringify(envelope, null, 2) + '\n', {}); + }; - // Run the package's lifecycle in proper order: init → setup → destroy. - // - // Many destinations rely on `init` to: parse `$env`-injected JSON - // strings (e.g. service-account credentials), construct an SDK client, - // and validate required settings. Calling `setup` directly skips that - // preparation and forces setup to re-implement client construction - // from raw, unparsed config — which is what previously caused - // `walkeros setup destination.pubsub` to fail with "Could not load the - // default credentials". Mirroring the collector's behavior (see - // collector/src/destination.ts:526-540) keeps both invocation paths - // honest about what the package promises. - // - // The init result is classified the same way the collector does: - // - object → use as resolved config for setup + destroy - // - void → init mutated the input config in place; reuse it - // - false → init explicitly aborted; do NOT run setup - let resolvedConfig: unknown = resolvedInputConfig; - if (isLifecycleFn(initFn)) { - const initResult = await initFn({ - id: component.id, - config: resolvedInputConfig, - env: resolvedEnv, - logger: scoped, - }); - const outcome = classifyInitResult(initResult); - if (outcome.kind === 'aborted') { + if (!isLifecycleFn(setupFn)) { + // No setup defined on the package, narrate explicitly, exit ok. scoped.info( - `setup: skipped ${component.kind}.${component.id} (init returned false)`, + `setup: skipped ${component.kind}.${component.id} (no setup function)`, ); - await emitSkipEnvelope('init returned false'); + await emitSkipEnvelope('no setup function'); return; } - if (outcome.kind === 'config') { - resolvedConfig = outcome.config; + + // Honor config.setup explicitly. If user wrote `setup: false`, narrate and skip. + // (Omitted setup is also falsy and gets the same skip message.) + const setupConfig = readSetupField(component.config); + if (setupConfig === false || setupConfig === undefined) { + const reason = setupConfig === false ? 'false' : 'unset'; + scoped.info( + `setup: skipped ${component.kind}.${component.id} (config.setup is ${reason})`, + ); + await emitSkipEnvelope(`config.setup is ${reason}`); + return; } - } - const result = await setupFn({ - id: component.id, - config: resolvedConfig, - env: resolvedEnv, - logger: scoped, - }); + // Server flows are loaded in deferred mode so the bundler can rewrite + // `$env.NAME` markers into `process.env[NAME]` expressions. The setup + // command imports the package and runs its lifecycle directly in this + // Node process, so markers must be replaced with their actual values + // here. Web flows are already eagerly resolved by `loadBundleConfig`, + // so this is a no-op for them. + const resolvedInputConfig = resolveEnvMarkers(component.config); + const resolvedEnv = resolveEnvMarkers(component.env); - // Always run `destroy` after setup if the package provides one, so that - // SDK clients constructed in `init` (e.g. PubSub `client.close()`, - // BigQuery `writeClient.close()`) release sockets and timers cleanly - // before the CLI exits. We log destroy failures but never let them - // mask a successful setup result. - if (isLifecycleFn(destroyFn)) { - try { - await destroyFn({ + // Run the package's lifecycle in proper order: init → setup → destroy. + // + // Many destinations rely on `init` to: parse `$env`-injected JSON + // strings (e.g. service-account credentials), construct an SDK client, + // and validate required settings. Calling `setup` directly skips that + // preparation and forces setup to re-implement client construction + // from raw, unparsed config — which is what previously caused + // `walkeros setup destination.pubsub` to fail with "Could not load the + // default credentials". Mirroring the collector's behavior (see + // collector/src/destination.ts:526-540) keeps both invocation paths + // honest about what the package promises. + // + // The init result is classified the same way the collector does: + // - object → use as resolved config for setup + destroy + // - void → init mutated the input config in place; reuse it + // - false → init explicitly aborted; do NOT run setup + let resolvedConfig: unknown = resolvedInputConfig; + if (isLifecycleFn(initFn)) { + const initResult = await initFn({ id: component.id, - config: resolvedConfig, + config: resolvedInputConfig, env: resolvedEnv, logger: scoped, }); - } catch (err) { - scoped.warn(`setup: destroy failed`, { - error: err instanceof Error ? err.message : String(err), - }); + const outcome = classifyInitResult(initResult); + if (outcome.kind === 'aborted') { + scoped.info( + `setup: skipped ${component.kind}.${component.id} (init returned false)`, + ); + await emitSkipEnvelope('init returned false'); + return; + } + if (outcome.kind === 'config') { + resolvedConfig = outcome.config; + } } - } - // In --json mode, emit the standard envelope so this command matches the - // rest of the CLI (createSuccessOutput → writeResult). In human mode we - // skip the raw JSON dump entirely; sibling commands narrate, they don't - // splice JSON between lines. - if (opts.json) { - const envelope = createSuccessOutput( - { result: result ?? null }, - Date.now() - startTime, - ); - await writeResult(JSON.stringify(envelope, null, 2) + '\n', {}); - return; - } + const result = await setupFn({ + id: component.id, + config: resolvedConfig, + env: resolvedEnv, + logger: scoped, + }); + + // Always run `destroy` after setup if the package provides one, so that + // SDK clients constructed in `init` (e.g. PubSub `client.close()`, + // BigQuery `writeClient.close()`) release sockets and timers cleanly + // before the CLI exits. We log destroy failures but never let them + // mask a successful setup result. + if (isLifecycleFn(destroyFn)) { + try { + await destroyFn({ + id: component.id, + config: resolvedConfig, + env: resolvedEnv, + logger: scoped, + }); + } catch (err) { + scoped.warn(`setup: destroy failed`, { + error: err instanceof Error ? err.message : String(err), + }); + } + } - scoped.info(`setup: ok ${component.kind}.${component.id}`); + // In --json mode, emit the standard envelope so this command matches the + // rest of the CLI (createSuccessOutput → writeResult). In human mode we + // skip the raw JSON dump entirely; sibling commands narrate, they don't + // splice JSON between lines. + if (opts.json) { + const envelope = createSuccessOutput( + { result: result ?? null }, + Date.now() - startTime, + ); + await writeResult(JSON.stringify(envelope, null, 2) + '\n', {}); + return; + } + + scoped.info(`setup: ok ${component.kind}.${component.id}`); + } finally { + // The lifecycle may lazy-load files from the install tree (grpc protos, + // schema assets), so the tree is removed only after init/setup/destroy + // are done. Best effort: a leftover tmp dir must never mask a result. + await fs.remove(loaded.installDir).catch(() => undefined); + } } diff --git a/packages/cli/src/core/__tests__/step-loader.test.ts b/packages/cli/src/core/__tests__/step-loader.test.ts new file mode 100644 index 000000000..f2e3b9cfd --- /dev/null +++ b/packages/cli/src/core/__tests__/step-loader.test.ts @@ -0,0 +1,410 @@ +import path from 'path'; +import os from 'os'; +import fs from 'fs-extra'; +import type { Flow } from '@walkeros/core'; +import { + resolveStepPackage, + resolvePackageEntry, + loadStepPackage, +} from '../step-loader'; +import * as packageManager from '../package-manager'; +import { createMockLogger } from '../../__tests__/helpers/mock-logger.js'; + +jest.mock('../package-manager', () => { + const actual = jest.requireActual('../package-manager'); + return { + ...actual, + loadNpmConfigForPacote: jest.fn(), + downloadPackagesWithResolution: jest.fn(), + }; +}); + +const mockedDownload = jest.mocked( + packageManager.downloadPackagesWithResolution, +); +const mockedNpmConfig = jest.mocked(packageManager.loadNpmConfigForPacote); + +const logger = createMockLogger(); + +function flowWith(overrides: Partial): Flow { + return { config: { platform: 'server' }, ...overrides }; +} + +describe('resolveStepPackage', () => { + it('returns the version pinned in config.bundle.packages', () => { + const flow = flowWith({ + config: { + platform: 'server', + bundle: { + packages: { + '@walkeros/server-destination-gcp': { version: '4.4.0' }, + }, + }, + }, + destinations: { + bigquery: { package: '@walkeros/server-destination-gcp', config: {} }, + }, + }); + const result = resolveStepPackage(flow, 'destination', 'bigquery', logger); + expect(result.packageName).toBe('@walkeros/server-destination-gcp'); + expect(result.spec.version).toBe('4.4.0'); + }); + + it('uses an inline step version when no bundle pin exists', () => { + const flow = flowWith({ + destinations: { + bigquery: { + package: '@walkeros/server-destination-gcp@4.3.0', + config: {}, + }, + }, + }); + const result = resolveStepPackage(flow, 'destination', 'bigquery', logger); + expect(result.packageName).toBe('@walkeros/server-destination-gcp'); + expect(result.spec.version).toBe('4.3.0'); + }); + + it('prefers the bundle pin over a disagreeing inline version', () => { + const flow = flowWith({ + config: { + platform: 'server', + bundle: { + packages: { + '@walkeros/server-destination-gcp': { version: '4.4.0' }, + }, + }, + }, + destinations: { + bigquery: { + package: '@walkeros/server-destination-gcp@4.3.0', + config: {}, + }, + }, + }); + const result = resolveStepPackage(flow, 'destination', 'bigquery', logger); + expect(result.spec.version).toBe('4.4.0'); + }); + + it('carries a path entry through', () => { + const flow = flowWith({ + config: { + platform: 'server', + bundle: { packages: { 'my-dest': { path: './pkgs/my-dest' } } }, + }, + destinations: { d: { package: 'my-dest', config: {} } }, + }); + const result = resolveStepPackage(flow, 'destination', 'd', logger); + expect(result.spec.path).toBe('./pkgs/my-dest'); + }); + + it('does not mutate the input flow and rewrites the clone to bare names', () => { + const flow = flowWith({ + destinations: { + bigquery: { + package: '@walkeros/server-destination-gcp@4.3.0', + config: {}, + }, + }, + }); + const result = resolveStepPackage(flow, 'destination', 'bigquery', logger); + expect(flow.destinations?.bigquery?.package).toBe( + '@walkeros/server-destination-gcp@4.3.0', + ); + expect(result.normalizedFlow.destinations?.bigquery?.package).toBe( + '@walkeros/server-destination-gcp', + ); + }); + + it('surfaces overrides from config.bundle.overrides', () => { + const flow = flowWith({ + config: { + platform: 'server', + bundle: { overrides: { '@amplitude/analytics-types': '2.11.1' } }, + }, + destinations: { + d: { package: '@walkeros/web-destination-api', config: {} }, + }, + }); + const result = resolveStepPackage(flow, 'destination', 'd', logger); + expect(result.overrides).toEqual({ + '@amplitude/analytics-types': '2.11.1', + }); + }); + + it('throws when the step does not exist', () => { + expect(() => + resolveStepPackage(flowWith({}), 'destination', 'missing', logger), + ).toThrow(/missing/); + }); + + it('throws when the step has inline code instead of a package', () => { + const flow = flowWith({ + destinations: { d: { code: { push: '$code:() => {}' }, config: {} } }, + }); + expect(() => resolveStepPackage(flow, 'destination', 'd', logger)).toThrow( + /package/, + ); + }); +}); + +describe('resolvePackageEntry', () => { + let treeDir: string; + let pkgDir: string; + + beforeEach(async () => { + treeDir = await fs.mkdtemp(path.join(os.tmpdir(), 'step-loader-entry-')); + pkgDir = path.join(treeDir, 'node_modules', 'x'); + await fs.ensureDir(pkgDir); + }); + + afterEach(async () => { + await fs.remove(treeDir); + }); + + async function writePkg(pkgJson: Record, files: string[]) { + await fs.writeJson(path.join(pkgDir, 'package.json'), pkgJson); + for (const f of files) { + await fs.ensureFile(path.join(pkgDir, f)); + } + } + + it.each([ + ['exports as string', { exports: './dist/index.mjs' }, 'dist/index.mjs'], + [ + 'exports dot string', + { exports: { '.': './dist/index.mjs' } }, + 'dist/index.mjs', + ], + [ + 'exports dot conditions', + { + exports: { + '.': { import: './dist/index.mjs', require: './dist/index.js' }, + }, + }, + 'dist/index.mjs', + ], + [ + 'nested import condition', + { exports: { '.': { import: { default: './dist/index.mjs' } } } }, + 'dist/index.mjs', + ], + [ + 'node condition declared before default', + { + exports: { '.': { node: './dist/node.js', default: './dist/def.js' } }, + }, + 'dist/node.js', + ], + [ + 'default condition declared before node', + { + exports: { '.': { default: './dist/def.js', node: './dist/node.js' } }, + }, + 'dist/def.js', + ], + [ + 'export array, skipping invalid targets', + { exports: { '.': ['invalid-target', './dist/ok.js'] } }, + 'dist/ok.js', + ], + ['main field', { main: './dist/index.js' }, 'dist/index.js'], + [ + 'main over module (Node never reads module)', + { module: './dist/index.mjs', main: './dist/index.js' }, + 'dist/index.js', + ], + ])('resolves %s', async (_label, pkgJson, expected) => { + await writePkg({ name: 'x', ...pkgJson }, [expected]); + await expect(resolvePackageEntry(pkgDir, 'x')).resolves.toBe( + path.join(pkgDir, expected), + ); + }); + + it('rejects a blocked root export instead of falling back to main', async () => { + await writePkg( + { + name: 'x', + exports: { '.': null }, + module: './m.js', + main: './main.js', + }, + ['m.js', 'main.js'], + ); + await expect(resolvePackageEntry(pkgDir, 'x')).rejects.toThrow(/exports/); + }); + + it('rejects a require-only exports root, matching Node import()', async () => { + await writePkg({ name: 'x', exports: { '.': { require: './index.js' } } }, [ + 'index.js', + ]); + await expect(resolvePackageEntry(pkgDir, 'x')).rejects.toThrow(/exports/); + }); + + it('rejects a missing package.json with a clear error', async () => { + await expect(resolvePackageEntry(pkgDir, 'x')).rejects.toThrow( + /package\.json/, + ); + }); + + it('rejects a TypeScript entry (setup imports with Node, not esbuild)', async () => { + await writePkg({ name: 'x', main: './index.ts' }, ['index.ts']); + await expect(resolvePackageEntry(pkgDir, 'x')).rejects.toThrow( + /built package/, + ); + }); + + it('rejects when an exports-declared entry file does not exist', async () => { + await writePkg({ name: 'x', exports: './dist/index.mjs' }, []); + await expect(resolvePackageEntry(pkgDir, 'x')).rejects.toThrow( + /dist\/index\.mjs/, + ); + }); + + it('rejects when the main entry file does not exist', async () => { + await writePkg({ name: 'x', main: './dist/index.js' }, []); + await expect(resolvePackageEntry(pkgDir, 'x')).rejects.toThrow( + /dist\/index\.js/, + ); + }); + + it('removes its probe file after resolving', async () => { + await writePkg({ name: 'x', main: './index.js' }, ['index.js']); + await resolvePackageEntry(pkgDir, 'x'); + const entries = await fs.readdir(pkgDir); + expect(entries.sort()).toEqual(['index.js', 'package.json']); + }); +}); + +describe('loadStepPackage', () => { + let fixtureDir: string; + + beforeEach(async () => { + jest.clearAllMocks(); + mockedNpmConfig.mockResolvedValue({ + registry: 'https://registry.npmjs.org/', + }); + // Real fixture package: acquisition is mocked but installs a real tree + // in the production layout (installDir/node_modules/), and the + // entry resolution and import are real. + fixtureDir = await fs.mkdtemp(path.join(os.tmpdir(), 'step-loader-pkg-')); + await fs.writeJson(path.join(fixtureDir, 'package.json'), { + name: '@walkeros/fixture-dest', + version: '4.4.0', + main: './index.mjs', + }); + await fs.writeFile( + path.join(fixtureDir, 'index.mjs'), + 'export default { type: "fixture", push: () => {}, setup: async () => ({ ok: true }) };\n', + ); + mockedDownload.mockImplementation(async (_packages, targetDir) => { + const packageDir = path.join( + targetDir, + 'node_modules', + '@walkeros/fixture-dest', + ); + await fs.copy(fixtureDir, packageDir); + return { + packagePaths: new Map([['@walkeros/fixture-dest', packageDir]]), + resolution: { topLevel: new Map(), nested: [] }, + }; + }); + }); + + afterEach(async () => { + await fs.remove(fixtureDir); + }); + + const pinnedFlow: Flow = { + config: { + platform: 'server', + bundle: { + packages: { '@walkeros/fixture-dest': { version: '4.4.0' } }, + overrides: { arrify: '2.0.1' }, + }, + }, + destinations: { d: { package: '@walkeros/fixture-dest', config: {} } }, + }; + + it('passes the flow-pinned version to the shared acquisition pipeline', async () => { + const loaded = await loadStepPackage(pinnedFlow, 'destination', 'd', { + logger, + }); + try { + expect(mockedDownload).toHaveBeenCalledTimes(1); + const [packages, , , useCache, , , overrides] = + mockedDownload.mock.calls[0]; + expect(packages).toEqual([ + { name: '@walkeros/fixture-dest', version: '4.4.0' }, + ]); + expect(useCache).toBe(true); + expect(overrides).toEqual({ arrify: '2.0.1' }); + } finally { + await fs.remove(loaded.installDir); + } + }); + + it('defaults an unpinned package to latest, exactly like bundle', async () => { + const flow: Flow = { + config: { platform: 'server' }, + destinations: { d: { package: '@walkeros/fixture-dest', config: {} } }, + }; + const loaded = await loadStepPackage(flow, 'destination', 'd', { logger }); + try { + const [packages] = mockedDownload.mock.calls[0]; + expect(packages).toEqual([ + { name: '@walkeros/fixture-dest', version: 'latest' }, + ]); + } finally { + await fs.remove(loaded.installDir); + } + }); + + it('imports the extracted entry and returns the module namespace', async () => { + const loaded = await loadStepPackage(pinnedFlow, 'destination', 'd', { + logger, + }); + try { + expect(loaded.packageName).toBe('@walkeros/fixture-dest'); + expect(loaded.packageDir).toBe( + path.join(loaded.installDir, 'node_modules', '@walkeros/fixture-dest'), + ); + const def = loaded.module.default; + expect(def).toMatchObject({ type: 'fixture' }); + } finally { + await fs.remove(loaded.installDir); + } + }); + + it('wraps acquisition failures with an offline/path hint', async () => { + mockedDownload.mockRejectedValue( + new Error('Failed to download @walkeros/fixture-dest@4.4.0: ENOTFOUND'), + ); + await expect( + loadStepPackage(pinnedFlow, 'destination', 'd', { logger }), + ).rejects.toThrow(/registry.*offline|offline.*registry/i); + }); + + it('allocates a distinct install dir per concurrent load', async () => { + const [a, b] = await Promise.all([ + loadStepPackage(pinnedFlow, 'destination', 'd', { logger }), + loadStepPackage(pinnedFlow, 'destination', 'd', { logger }), + ]); + try { + expect(a.installDir).not.toBe(b.installDir); + } finally { + await fs.remove(a.installDir); + await fs.remove(b.installDir); + } + }); + + it('errors when the resolved tree misses the requested package', async () => { + mockedDownload.mockResolvedValue({ + packagePaths: new Map(), + resolution: { topLevel: new Map(), nested: [] }, + }); + await expect( + loadStepPackage(pinnedFlow, 'destination', 'd', { logger }), + ).rejects.toThrow(/@walkeros\/fixture-dest/); + }); +}); diff --git a/packages/cli/src/core/__tests__/step-packages.test.ts b/packages/cli/src/core/__tests__/step-packages.test.ts new file mode 100644 index 000000000..676077707 --- /dev/null +++ b/packages/cli/src/core/__tests__/step-packages.test.ts @@ -0,0 +1,75 @@ +import { applyStepPackages } from '../step-packages'; +import { createMockLogger } from '@walkeros/core'; +import type { Flow } from '@walkeros/core'; + +describe('applyStepPackages path precedence', () => { + const logger = createMockLogger(); + + it('keeps a local path when steps declare conflicting inline versions', () => { + const flow: Flow = { + destinations: { + a: { package: '@walkeros/dest@1.0.0' }, + b: { package: '@walkeros/dest@2.0.0' }, + }, + }; + const packages: Record = { + '@walkeros/dest': { path: './pkgs/dest' }, + }; + + applyStepPackages(flow, packages, logger); + + expect(packages['@walkeros/dest']).toEqual({ path: './pkgs/dest' }); + }); + + it('does not stamp an inline version onto a path entry', () => { + const flow: Flow = { + destinations: { a: { package: '@walkeros/dest@1.0.0' } }, + }; + const packages: Record = { + '@walkeros/dest': { path: './pkgs/dest' }, + }; + + applyStepPackages(flow, packages, logger); + + expect(packages['@walkeros/dest']).toEqual({ path: './pkgs/dest' }); + }); + + it('still throws on conflicting inline versions without a bundle entry', () => { + const flow: Flow = { + destinations: { + a: { package: '@walkeros/dest@1.0.0' }, + b: { package: '@walkeros/dest@2.0.0' }, + }, + }; + + expect(() => applyStepPackages(flow, {}, logger)).toThrow( + /Conflicting inline versions/, + ); + }); + + it('still lets a version pin win over a disagreeing inline version', () => { + const flow: Flow = { + destinations: { a: { package: '@walkeros/dest@1.0.0' } }, + }; + const packages: Record = { + '@walkeros/dest': { version: '2.0.0' }, + }; + + applyStepPackages(flow, packages, logger); + + expect(packages['@walkeros/dest'].version).toBe('2.0.0'); + }); + + it('still fills an unversioned bundle entry from the inline version', () => { + const flow: Flow = { + destinations: { a: { package: '@walkeros/dest@1.0.0' } }, + }; + const packages: Record = { + '@walkeros/dest': {}, + }; + + applyStepPackages(flow, packages, logger); + + expect(packages['@walkeros/dest'].version).toBe('1.0.0'); + }); +}); diff --git a/packages/cli/src/commands/bundle/package-manager.ts b/packages/cli/src/core/package-manager.ts similarity index 99% rename from packages/cli/src/commands/bundle/package-manager.ts rename to packages/cli/src/core/package-manager.ts index 92dbddde8..0862d8f3f 100644 --- a/packages/cli/src/commands/bundle/package-manager.ts +++ b/packages/cli/src/core/package-manager.ts @@ -4,10 +4,10 @@ import fs from 'fs-extra'; import { readFile } from 'fs/promises'; import os from 'os'; import semver from 'semver'; -import { resolveLocalPackage, copyLocalPackage } from '../../core/index.js'; +import { resolveLocalPackage, copyLocalPackage } from './local-packages.js'; import type { Logger } from '@walkeros/core'; -import { getPackageCacheKey } from '../../core/cache-utils.js'; -import { getTmpPath } from '../../core/tmp.js'; +import { getPackageCacheKey } from './cache-utils.js'; +import { getTmpPath } from './tmp.js'; export interface NpmConfig { registry: string; diff --git a/packages/cli/src/core/step-loader.ts b/packages/cli/src/core/step-loader.ts new file mode 100644 index 000000000..5893153d2 --- /dev/null +++ b/packages/cli/src/core/step-loader.ts @@ -0,0 +1,261 @@ +/** + * Shared step-package loader. + * + * One code path from "a step's package + pinned version" to "a loaded + * module": normalize the pin with the exact bundle precedence + * (applyStepPackages), download the package and its transitive deps with + * the same pacote pipeline and caches the bundler uses, resolve the + * package's entry file, and import it. `walkeros setup` consumes the + * loaded module; keeping acquisition identical to bundle guarantees setup + * can never act on a different package version than the bundle ships. + */ +import path from 'path'; +import { pathToFileURL, fileURLToPath } from 'url'; +import { execFile } from 'child_process'; +import { promisify } from 'util'; +import fs from 'fs-extra'; +import type { Flow, Logger } from '@walkeros/core'; +import { getTmpPath } from './tmp.js'; +import { applyStepPackages, getFlowSection } from './step-packages.js'; +import { + downloadPackagesWithResolution, + loadNpmConfigForPacote, +} from './package-manager.js'; + +export type StepKind = 'source' | 'destination' | 'store'; + +export interface ResolvedStepPackage { + /** Bare package name after normalization (inline version stripped, local path replaced by synthetic key). */ + packageName: string; + /** Effective bundle entry: version pin and/or local path per bundle precedence. */ + spec: Flow.BundlePackage; + /** flow.config.bundle.overrides, for transitive pins. */ + overrides: Record; + /** Deep clone of the flow with step.package rewritten to bare names; the input flow is never mutated. */ + normalizedFlow: Flow; +} + +const KIND_TO_SECTION = { + source: 'sources', + destination: 'destinations', + store: 'stores', +} as const; + +/** + * Resolve the effective package spec for one step by running the SAME + * normalization the bundler runs over the whole flow (applyStepPackages), + * on a clone so the caller's flow stays untouched. The clone comes back as + * `normalizedFlow` with `step.package` rewritten to the bare name, which is + * also what resolveExportName needs to match `bundle.packages[pkg].imports`. + */ +export function resolveStepPackage( + flow: Flow, + kind: StepKind, + id: string, + logger: Logger.Instance, +): ResolvedStepPackage { + const normalizedFlow = structuredClone(flow); + const packages: Record = structuredClone( + normalizedFlow.config?.bundle?.packages ?? {}, + ); + applyStepPackages(normalizedFlow, packages, logger); + + const bucket = getFlowSection(normalizedFlow, KIND_TO_SECTION[kind]); + const step = bucket?.[id]; + if (!step) { + const available = Object.keys(bucket ?? {}).join(', ') || '(none)'; + throw new Error( + `${kind} "${id}" not found in flow. Available: ${available}`, + ); + } + if (typeof step.package !== 'string') { + throw new Error( + `${kind}.${id} has no "package" (inline code cannot be set up).`, + ); + } + + return { + packageName: step.package, + spec: packages[step.package] ?? {}, + overrides: normalizedFlow.config?.bundle?.overrides ?? {}, + normalizedFlow, + }; +} + +const execFileAsync = promisify(execFile); + +/** + * Probe module for entry resolution. Written into the extracted package + * and run in a child Node process, so `import.meta.resolve` applies the + * runtime's own resolution rules (conditional exports in declaration + * order, export arrays, blocked roots, legacy main) exactly as the later + * `import()` of the entry will. A child process rather than an in-process + * import keeps the probe out of the test runner's module pipeline. + */ +const ENTRY_PROBE_FILENAME = '.walkeros-entry-probe.mjs'; +const ENTRY_PROBE_SOURCE = [ + 'try {', + ' process.stdout.write(import.meta.resolve(process.argv[2]));', + '} catch (error) {', + ' process.stderr.write(error instanceof Error ? error.message : String(error));', + ' process.exit(1);', + '}', + '', +].join('\n'); + +/** + * Resolve the entry file of an extracted package for a real Node import, + * by asking Node itself (see ENTRY_PROBE_SOURCE). The package must sit in + * a node_modules tree, as extracted install trees always do. TypeScript + * entries are rejected with a targeted message: the bundler can compile + * them, Node cannot. + */ +export async function resolvePackageEntry( + packageDir: string, + packageName: string, +): Promise { + const pkgJsonPath = path.join(packageDir, 'package.json'); + if (!(await fs.pathExists(pkgJsonPath))) { + throw new Error( + `No readable package.json in ${packageDir}. ` + + `A "path" package used with setup must be a real package directory.`, + ); + } + + const probePath = path.join(packageDir, ENTRY_PROBE_FILENAME); + await fs.writeFile(probePath, ENTRY_PROBE_SOURCE); + let entryUrl: string; + try { + const { stdout } = await execFileAsync(process.execPath, [ + probePath, + packageName, + ]); + entryUrl = stdout.trim(); + } catch (error) { + // The probe forwards Node's own resolution error (e.g. "No "exports" + // main defined in .../package.json") on stderr. + let message = error instanceof Error ? error.message : String(error); + if ( + error instanceof Error && + 'stderr' in error && + typeof error.stderr === 'string' && + error.stderr.trim() !== '' + ) { + message = error.stderr.trim(); + } + throw new Error(message); + } finally { + await fs.remove(probePath).catch(() => undefined); + } + + const entryAbs = fileURLToPath(entryUrl); + const entryRel = path.relative(packageDir, entryAbs); + if (entryAbs.endsWith('.ts') || entryAbs.endsWith('.tsx')) { + throw new Error( + `Entry ${entryRel} of ${packageDir} is TypeScript. Setup imports ` + + `packages with Node at runtime; point config.bundle.packages ` + + `"path" at a built package (with dist output), not sources.`, + ); + } + + // import.meta.resolve does not stat exports targets; catch dead + // declarations here with a message that names the package.json. + if (!(await fs.pathExists(entryAbs))) { + throw new Error( + `Entry file ${entryRel} declared by ${pkgJsonPath} does not exist.`, + ); + } + return entryAbs; +} + +export interface LoadStepOptions { + configDir?: string; + tmpDir?: string; + cache?: boolean; + logger: Logger.Instance; +} + +export interface LoadedStepPackage { + module: Record; + packageName: string; + normalizedFlow: Flow; + /** Extracted package dir inside the install tree (for diagnostics/tests). */ + packageDir: string; + /** Temp install root. Caller MUST remove it, and only AFTER the component lifecycle completes (packages lazy-load files, e.g. grpc .proto, at init/setup time). */ + installDir: string; +} + +/** + * Acquire and import one step's package. Downloads run through the same + * pipeline and caches as `walkeros bundle` (exact pins are served from the + * package cache offline; latest/range specs re-resolve against the + * registry at most daily; `path` entries never touch the network). + * + * The returned installDir must be removed by the caller, and only after + * the component's lifecycle (init/setup/destroy) has completed: packages + * may lazy-load files from their install tree at call time. + */ +export async function loadStepPackage( + flow: Flow, + kind: StepKind, + id: string, + opts: LoadStepOptions, +): Promise { + const { packageName, spec, overrides, normalizedFlow } = resolveStepPackage( + flow, + kind, + id, + opts.logger, + ); + + const npmConfig = await loadNpmConfigForPacote(opts.configDir); + // mkdtemp guarantees a unique tree per load: concurrent loads must never + // share an installDir, since each caller removes its tree when done. + await fs.ensureDir(getTmpPath(opts.tmpDir)); + const installDir = await fs.mkdtemp( + getTmpPath(opts.tmpDir, 'walkeros-setup-'), + ); + + try { + const { packagePaths } = await downloadPackagesWithResolution( + [ + { + name: packageName, + version: spec.version || 'latest', + ...(spec.path ? { path: spec.path } : {}), + }, + ], + installDir, + opts.logger, + opts.cache ?? true, + opts.configDir, + opts.tmpDir, + overrides, + npmConfig, + ); + + const packageDir = packagePaths.get(packageName); + if (!packageDir) { + throw new Error( + `Acquisition resolved no install path for ${packageName}.`, + ); + } + + const entry = await resolvePackageEntry(packageDir, packageName); + const module: Record = await import( + pathToFileURL(entry).href + ); + return { module, packageName, normalizedFlow, packageDir, installDir }; + } catch (error) { + await fs.remove(installDir).catch(() => undefined); + const message = error instanceof Error ? error.message : String(error); + throw new Error( + `Failed to load ${packageName} for ${kind}.${id}: ${message}\n` + + `Setup fetches the flow's pinned package from the npm registry, ` + + `sharing the bundle cache. If you are offline, run once online to ` + + `warm the cache (exact version pins stay cached), or point ` + + `config.bundle.packages["${packageName}"].path at a local built package.`, + { cause: error }, + ); + } +} diff --git a/packages/cli/src/core/step-packages.ts b/packages/cli/src/core/step-packages.ts new file mode 100644 index 000000000..4f07008e3 --- /dev/null +++ b/packages/cli/src/core/step-packages.ts @@ -0,0 +1,233 @@ +/** + * Step-package normalization shared by bundle and setup. + * + * Pure flow-config logic: which packages do the flow's steps declare, and + * what is the effective {name, version | path} for each after merging inline + * step specs with config.bundle.packages pins. Lives in core/ so both + * commands/bundle/* and commands/setup/* run the exact same precedence + * (a bundle pin always wins; an unversioned bundle entry is filled from the + * first inline version; conflicting inline versions with no pin throw; + * local paths become synthetic packageNameToVariable keys). + */ +import type { Flow, Logger } from '@walkeros/core'; +import { packageNameToVariable } from '@walkeros/core'; + +/** + * Type-narrowed accessor for a Flow section. Returns the typed step record + * (or undefined) — exhaustive switch over the literal-union parameter avoids + * a generic indexed-access cast. + * + * Returns the union of all section types when the caller passes a runtime + * variable. Call sites that need a specific section type access the field + * directly (e.g. `flow.sources`). + */ +export type FlowStepRecord = + | Record + | Record + | Record + | Record; +export function getFlowSection( + flow: Flow, + section: 'sources' | 'destinations' | 'transformers' | 'stores', +): FlowStepRecord | undefined { + switch (section) { + case 'sources': + return flow.sources; + case 'destinations': + return flow.destinations; + case 'transformers': + return flow.transformers; + case 'stores': + return flow.stores; + } +} + +/** + * Split a step package spec into bare name and optional version suffix. + * `@walkeros/x@1.2.3` → { name: '@walkeros/x', version: '1.2.3' }. + * The scope `@` at index 0 is never a separator. A trailing `@` is ignored. + * Alias/git/file suffixes are not interpreted — the resolver handles or + * rejects them downstream. + */ +export function parsePackageSpec(spec: string): { + name: string; + version?: string; +} { + const at = spec.lastIndexOf('@'); + if (at <= 0) return { name: spec }; + const version = spec.slice(at + 1); + return version + ? { name: spec.slice(0, at), version } + : { name: spec.slice(0, at) }; +} + +/** + * Detects destination packages from flow configuration. + * Extracts package names from destinations that have explicit 'package' field. + */ +/** + * Detects packages from a flow config section (sources, destinations, transformers, stores). + * Extracts package names from steps that have an explicit 'package' field. + */ +export function detectStepPackages( + flowSettings: Flow, + section: 'sources' | 'destinations' | 'transformers' | 'stores', +): Set { + const packages = new Set(); + const steps = getFlowSection(flowSettings, section); + + if (steps) { + for (const [, stepConfig] of Object.entries(steps)) { + if (typeof stepConfig !== 'object' || stepConfig === null) continue; + // Require explicit package field + if (typeof stepConfig.package === 'string') { + packages.add(stepConfig.package); + } + } + } + + return packages; +} + +/** + * Collects all package names declared in flow steps. + * Returns both npm packages and local paths — caller handles routing. + */ +export function collectAllStepPackages(flowSettings: Flow): Set { + const allPackages = new Set(); + const sections = [ + 'sources', + 'destinations', + 'transformers', + 'stores', + ] as const; + + for (const section of sections) { + for (const pkg of detectStepPackages(flowSettings, section)) { + allPackages.add(pkg); + } + } + + return allPackages; +} + +/** + * Auto-adds every step-declared package (sources, destinations, transformers, + * stores) to `packages`, mutating `flowSettings` in place so each step's + * `package` field points at the key that ends up in `packages`. + * + * Local paths (`.` or `/` prefixed) are normalized to a synthetic + * `packageNameToVariable` key so the regular default-import codegen wires + * them up automatically. + * + * npm specs go through `parsePackageSpec` to split an inline version + * (`@walkeros/x@1.2.3`) from the bare name. Precedence policy: + * - An explicit `config.bundle.packages` version always wins; a disagreeing + * inline version only warns (bundle pin is authoritative). + * - An unversioned bundle entry is filled from the first inline version seen + * for that bare name, preserving any other fields already on the entry. + * - Two different inline versions for the same bare name, with no bundle + * pin to arbitrate, are ambiguous — throw naming both steps' versions + * rather than silently picking one. + * - Identical inline versions across steps are fine (no-op). + * Alias/git/file suffixes are not special-cased: `parsePackageSpec` splits on + * the last `@` and the resolver handles or rejects the rest downstream. + */ +export function applyStepPackages( + flowSettings: Flow, + packages: Record, + logger: Logger.Instance, +): void { + const stepPackages = collectAllStepPackages(flowSettings); + // Bundle-pinned version per bare name, captured the first time each name is + // encountered — i.e. before this function's own fill-ins can be mistaken + // for a real `config.bundle.packages` pin on a later iteration. + const originalPins = new Map(); + // Inline version already seen per bare name (with no real bundle pin), + // used to detect a second, different inline version for the same name. + const inlineSeen = new Map(); + + const rewriteSteps = (from: string, to: string): void => { + for (const section of [ + 'sources', + 'destinations', + 'transformers', + 'stores', + ] as const) { + const steps = getFlowSection(flowSettings, section); + if (!steps) continue; + for (const step of Object.values(steps)) { + if (step.package === from) { + step.package = to; + } + } + } + }; + + for (const pkg of stepPackages) { + const isLocalPath = pkg.startsWith('.') || pkg.startsWith('/'); + + if (isLocalPath) { + // Normalize: convert path-based package: to packages section entry. + // The synthetic key acts as the package name for downstream codegen, + // so the regular default-import flow wires it up automatically. + const varName = packageNameToVariable(pkg); + if (!packages[varName]) { + packages[varName] = { + path: pkg, + }; + } + + // Rewrite all components that reference the raw path to point at the + // synthetic packages-section key instead. + rewriteSteps(pkg, varName); + continue; + } + + const { name, version } = parsePackageSpec(pkg); + + if (!originalPins.has(name)) { + const entry = packages[name]; + // A `path` entry is a pin too: local paths win at resolution, so the + // inline versions the steps declare can never be used. + originalPins.set( + name, + entry?.version !== undefined || entry?.path !== undefined, + ); + } + const hasBundlePin = originalPins.get(name) === true; + + if (name !== pkg) { + // Rewrite every step that declared the versioned spec to the bare + // name, mirroring the local-path rewrite above. + rewriteSteps(pkg, name); + } + + if (version && !hasBundlePin) { + const seen = inlineSeen.get(name); + if (seen !== undefined && seen !== version) { + throw new Error( + `Conflicting inline versions for ${name}: "${seen}" and "${version}" are ` + + `declared by different steps. Pin one version in config.bundle.packages.`, + ); + } + inlineSeen.set(name, version); + } + + const existing = packages[name]; + if (!existing) { + packages[name] = version ? { version } : {}; + } else if (version && !existing.path) { + // A path entry decides on its own; stamping a version onto it would + // record a version that resolution never consults. + if (!existing.version) { + existing.version = version; // fill an unversioned bundle entry + } else if (existing.version !== version) { + logger.warn( + `Package ${name}: config.bundle.packages pins ${existing.version}; ` + + `a step declares ${version} inline. Using the bundle pin.`, + ); + } + } + } +} diff --git a/packages/core/src/schemas/__tests__/source.test.ts b/packages/core/src/schemas/__tests__/source.test.ts index 44ea85111..99e4976d5 100644 --- a/packages/core/src/schemas/__tests__/source.test.ts +++ b/packages/core/src/schemas/__tests__/source.test.ts @@ -1,4 +1,8 @@ -import { InitSourceSchema, initSourceJsonSchema } from '../source'; +import { + InitSourceSchema, + initSourceJsonSchema, + PartialConfigSchema, +} from '../source'; interface JsonNode { properties?: Record; @@ -42,3 +46,22 @@ describe('InitSourceSchema', () => { expect(def?.properties?.state).toBeDefined(); }); }); + +describe('Source.Config async', () => { + it.each([[true], [false]])('accepts boolean async %p', (value) => { + const result = PartialConfigSchema.safeParse({ async: value }); + expect(result.success).toBe(true); + }); + + it('accepts a per-method record', () => { + const result = PartialConfigSchema.safeParse({ + async: { GET: false, POST: true }, + }); + expect(result.success).toBe(true); + }); + + it('rejects a non-boolean record value', () => { + const result = PartialConfigSchema.safeParse({ async: { GET: 'yes' } }); + expect(result.success).toBe(false); + }); +}); diff --git a/packages/core/src/schemas/source.ts b/packages/core/src/schemas/source.ts index 7ff9baa9b..72f70ba01 100644 --- a/packages/core/src/schemas/source.ts +++ b/packages/core/src/schemas/source.ts @@ -155,10 +155,10 @@ export const ConfigSchema = MappingConfigSchema.extend({ 'Logger configuration (level, handler) to override the collector defaults', ), async: z - .boolean() + .union([z.boolean(), z.record(z.string(), z.boolean())]) .optional() .describe( - 'Respond-first acknowledgement for response-producing server sources (express today; future fetch/lambda). When true (the default for such sources) the source responds 2xx ("accepted") before the event is delivered; when false it waits for delivery to settle. Browser/dataLayer sources have no HTTP response to defer and ignore it. A 2xx means accepted, not delivered. Default is per source type.', + 'Respond-first acknowledgement for response-producing server sources. When true the source responds 2xx ("accepted") before the event is delivered; when false it waits for delivery to settle and the response reflects the outcome. A record configures this per source-defined key: the express source keys it by HTTP method (GET/POST) and defaults to { GET: false, POST: true }. Browser/dataLayer sources have no HTTP response to defer and ignore it. A 2xx means accepted, not delivered. Defaults are per source type.', ), setup: z .union([z.boolean(), z.record(z.string(), z.unknown())]) diff --git a/packages/core/src/types/source.ts b/packages/core/src/types/source.ts index d0e4903f8..0e3bce595 100644 --- a/packages/core/src/types/source.ts +++ b/packages/core/src/types/source.ts @@ -110,17 +110,18 @@ export interface Config< /** * Respond-first acknowledgement for response-producing server sources. * - * When a source produces an HTTP response (express today; future fetch / - * lambda), `async: true` (the default for such sources) responds 2xx - * ("accepted") before the event is delivered to the collector, so the - * client is not blocked on backend delivery. `async: false` waits for - * delivery to settle before responding. A 2xx means "accepted", not - * "delivered". + * `true` responds 2xx ("accepted") before the event is delivered to the + * collector, so the client is not blocked on backend delivery; `false` + * waits for delivery to settle before responding and lets the response + * reflect the outcome. A record configures this per source-defined key: + * the express source keys it by HTTP method (`GET`/`POST`) and defaults + * to `{ GET: false, POST: true }`, so a step can serve real content on + * GET while POST acks fast. A 2xx means "accepted", not "delivered". * * Browser and dataLayer sources have no HTTP response to defer and ignore - * this flag. The default is per source type. + * this flag. Defaults are per source type. */ - async?: boolean; + async?: boolean | Record; /** Mark as primary source; its push function becomes the exported `elb` from startFlow. */ primary?: boolean; /** Defer source initialization until these collector events fire (e.g., `['consent']`). */ diff --git a/packages/mcps/mcp/README.md b/packages/mcps/mcp/README.md new file mode 100644 index 000000000..b2e6c6fc0 --- /dev/null +++ b/packages/mcps/mcp/README.md @@ -0,0 +1,181 @@ +

+ + walkerOS + +

+ +# @walkeros/mcp + +Model Context Protocol server for walkerOS flow development. Gives an AI +assistant the tools to discover packages, build a flow configuration, validate +it, simulate events through it, bundle it, and deploy it. + +[Documentation](https://www.walkeros.io/docs/apps/mcp) • +[NPM Package](https://www.npmjs.com/package/@walkeros/mcp) • +[Source Code](https://github.com/elbwalker/walkerOS/tree/main/packages/mcps/mcp) + +## Installation + +The server runs over stdio and is started by your MCP client. Add it to the +client's configuration: + +```json +{ + "mcpServers": { + "walkeros-flow": { + "command": "npx", + "args": ["@walkeros/mcp"] + } + } +} +``` + +| Client | File | +| -------------- | ---------------------------------------------------------- | +| Claude Code | `.mcp.json` in the project root | +| Cursor | `.cursor/mcp.json` in the project root | +| Claude Desktop | `claude_desktop_config.json` | +| VS Code | `.vscode/mcp.json`, with `servers` instead of `mcpServers` | + +In Claude Code you can also install the walkerOS plugin, which registers this +server, the tagging server, and the walkerOS skills in one step: + +``` +/plugin marketplace add elbwalker/walkerOS +/plugin install walkeros@elbwalker +``` + +To install the binary directly instead of running it through `npx`: + +```bash +npm install @walkeros/mcp +``` + +## No account required + +The server starts, registers all tools, and runs the whole local loop without +any credentials. `auth` reports `{ "authenticated": false }` and the local tools +work regardless. Only the walkerOS cloud tools need a login, either through the +`auth` tool's device code flow or a `WALKEROS_TOKEN` environment variable. + +## Quick start + +Five tools cover building a flow from nothing to a tested configuration, and all +five run locally: + +1. **`flow_load`** creates an empty flow for a platform, or loads an existing + one from a file, URL, or inline JSON. +2. **`package_search`** finds the exact package names for the sources, + destinations, transformers, and stores the flow needs. Never guess a package + name, search for it. +3. **`package_get`** returns that package's configuration schema, hints, and + worked examples, so the config is written against the real shape. +4. **`flow_validate`** checks the result. Fix and re-validate until it passes. +5. **`flow_simulate`** pushes an event through the flow with vendor calls mocked + and shows what each step produced. + +## Tools + +The server registers 17 tools. + +### Local, no account + +| Tool | Description | +| ---------------- | ---------------------------------------------------------------------------------- | +| `flow_load` | Load a flow from a file path, URL, inline JSON, or flow ID, or create an empty one | +| `flow_validate` | Validate an event, flow config, mapping rule, or data contract | +| `flow_simulate` | Run an event through a flow with mocked vendor calls and inspect each step | +| `flow_bundle` | Compile a flow into a tree-shaken, deployable JavaScript bundle | +| `flow_push` | Push a real event through a flow to real destinations, making real API calls | +| `flow_examples` | List the step examples in a flow, the fixtures simulation can replay | +| `package_search` | Find packages by name, type, or platform. The entry point for package discovery | +| `package_get` | Read one package's schemas, configuration hints, and examples by exact name | +| `diagnostics` | Report MCP and CLI versions, app URL, backend, and whether the app is reachable | + +### walkerOS cloud + +| Tool | Description | +| ------------------ | ----------------------------------------------------------------------------------------- | +| `auth` | Check login status, log in through the device code flow, or log out | +| `project_manage` | List, create, update, or delete projects, and set the default one | +| `flow_manage` | List, create, update, delete, or duplicate saved flows, and manage their previews | +| `deploy_manage` | Deploy a flow and list, inspect, or delete its deployments | +| `secret_manage` | Manage a flow's `$secret.` values. Write-mostly, values are never returned | +| `observe_session` | Start, inspect, or stop an Observe session, a time-boxed window on one running flow | +| `observe_journeys` | Read the assembled journeys for an observed flow, each event traced across web and server | +| `feedback` | Send feedback about walkerOS | + +## Resources + +| URI | Content | +| ---------------------------------- | --------------------------------------------------------------- | +| `walkeros://reference/flow-schema` | Flow configuration structure and connection rules | +| `walkeros://reference/event-model` | Event naming, properties, and auto-populated fields | +| `walkeros://reference/mapping` | Mapping syntax: data, map, loop, set, condition, policy | +| `walkeros://reference/consent` | The consent model at destination, rule, and field level | +| `walkeros://reference/variables` | Variable patterns: `$var`, `$env`, `$secret`, `$code`, `$store` | +| `walkeros://reference/contract` | Event schemas, wildcards, and inheritance | +| `walkeros://reference/openapi` | OpenAPI 3.1 specification for the walkerOS API | +| `walkeros://reference/packages` | The full package catalog | +| `walkeros://schema/{packageName}` | JSON schema for one package | + +Read these before writing a configuration by hand. + +## Prompts + +| Prompt | Purpose | +| ----------------- | ------------------------------------------------------------------ | +| `add-step` | Add a source, destination, transformer, or store to a flow | +| `setup-mapping` | Configure event mapping for a step | +| `manage-contract` | Create or update event contracts, in both directions with mappings | + +## Environment variables + +| Variable | Required | Default | Purpose | +| --------------------- | -------- | ------------------------- | ----------------------------------------------------- | +| `WALKEROS_TOKEN` | No | none | Bearer token, an alternative to the `auth` tool login | +| `WALKEROS_PROJECT_ID` | No | none | Active project ID (`proj_...`) | +| `WALKEROS_APP_URL` | No | `https://app.walkeros.io` | Base URL override | + +## Programmatic usage + +The package exports a transport-agnostic server factory, so a host application +can mount the protocol over HTTP instead of running the stdio binary: + +```typescript +import { + createWalkerOSMcpServer, + HttpToolClient, + createStreamableHttpHandler, +} from '@walkeros/mcp'; + +const server = createWalkerOSMcpServer({ + client: new HttpToolClient(), + version: '1.0.0', +}); + +export const POST = createStreamableHttpHandler(server, { + sessionIdGenerator: () => crypto.randomUUID(), +}); +``` + +To use the tool registry without the MCP protocol, for example with the Vercel +AI SDK, import `TOOL_DEFINITIONS` and supply your own `ToolClient`. The stdio +binary stays available as `@walkeros/mcp/stdio` and the `walkeros-mcp` bin +entry. + +## Documentation + +Full parameter tables, workflows, and examples live in the docs: +**https://www.walkeros.io/docs/apps/mcp** + +## Contribute + +Feel free to contribute by submitting an +[issue](https://github.com/elbwalker/walkerOS/issues), starting a +[discussion](https://github.com/elbwalker/walkerOS/discussions), or getting in +[contact](https://calendly.com/elb-alexander/30min). + +## License + +MIT diff --git a/packages/mcps/mcp/package.json b/packages/mcps/mcp/package.json index 6b5a738f1..8703155c5 100644 --- a/packages/mcps/mcp/package.json +++ b/packages/mcps/mcp/package.json @@ -57,11 +57,19 @@ "engines": { "node": ">=20.0.0" }, + "mcpName": "io.walkeros/flow", "keywords": [ "walkerOS", "mcp", "model-context-protocol", + "mcp-server", + "ai-agents", + "llm", "analytics", + "event-tracking", + "data-collection", + "tag-management", + "privacy-first", "cli" ], "publishConfig": { diff --git a/packages/mcps/mcp/server.json b/packages/mcps/mcp/server.json new file mode 100644 index 000000000..ad39cb626 --- /dev/null +++ b/packages/mcps/mcp/server.json @@ -0,0 +1,44 @@ +{ + "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json", + "name": "io.walkeros/flow", + "title": "walkerOS Flow", + "description": "Build, validate, simulate, and deploy walkerOS event pipelines", + "version": "4.4.0", + "websiteUrl": "https://www.walkeros.io/docs/apps/mcp", + "repository": { + "url": "https://github.com/elbwalker/walkerOS", + "source": "github", + "subfolder": "packages/mcps/mcp" + }, + "packages": [ + { + "registryType": "npm", + "registryBaseUrl": "https://registry.npmjs.org", + "identifier": "@walkeros/mcp", + "version": "4.4.0", + "transport": { + "type": "stdio" + }, + "environmentVariables": [ + { + "name": "WALKEROS_TOKEN", + "description": "walkerOS API token. Optional, the local tools work without it and the auth tool can log in instead.", + "isRequired": false, + "isSecret": true + }, + { + "name": "WALKEROS_PROJECT_ID", + "description": "Default walkerOS project id (proj_...) used by the cloud tools.", + "isRequired": false, + "isSecret": false + }, + { + "name": "WALKEROS_APP_URL", + "description": "Base URL override for the walkerOS app. Defaults to https://app.walkeros.io.", + "isRequired": false, + "isSecret": false + } + ] + } + ] +} diff --git a/packages/mcps/source-browser/README.md b/packages/mcps/source-browser/README.md new file mode 100644 index 000000000..e06dad927 --- /dev/null +++ b/packages/mcps/source-browser/README.md @@ -0,0 +1,85 @@ +

+ + walkerOS + +

+ +# @walkeros/mcp-source-browser + +Model Context Protocol server for walkerOS HTML tagging. Generates, parses, and +validates `data-elb` attributes using real DOM parsing, so an AI assistant can +tag a page and check its own work. Runs locally, no account or API token. + +[Documentation](https://www.walkeros.io/docs/apps/mcp) • +[NPM Package](https://www.npmjs.com/package/@walkeros/mcp-source-browser) • +[Source Code](https://github.com/elbwalker/walkerOS/tree/main/packages/mcps/source-browser) + +## Installation + +The server runs over stdio and is started by your MCP client. Add it to the +client's configuration: + +```json +{ + "mcpServers": { + "walkeros-source-browser": { + "command": "npx", + "args": ["@walkeros/mcp-source-browser"] + } + } +} +``` + +In Claude Code you can install the walkerOS plugin instead, which registers this +server along with the flow development server and the walkerOS skills: + +``` +/plugin marketplace add elbwalker/walkerOS +/plugin install walkeros@elbwalker +``` + +To install the binary directly: + +```bash +npm install @walkeros/mcp-source-browser +``` + +## Quick start + +Ask your assistant to tag a page, for example: "Generate walkerOS tagging for a +product card with name and price". It calls `generate_tagging` and returns +ready-to-paste `data-elb` attributes. + +## Tools + +| Tool | Description | +| ------------------ | -------------------------------------------------------------------------------------------------- | +| `generate_tagging` | Turn structured input into `data-elb` attributes and an example HTML snippet | +| `parse_tagging` | Parse an HTML snippet and extract the walkerOS events and globals it produces | +| `validate_tagging` | Check tagging for orphan actions, missing entities, unknown triggers, and entities without actions | + +All three accept an optional `prefix` to match a custom attribute prefix, which +defaults to `data-elb`. + +## Resources + +| URI | Content | +| ----------------------------------------- | ------------------------------------------ | +| `walkeros://docs/tagging/html-attributes` | Guide to `data-elb` HTML attribute tagging | +| `walkeros://docs/tagging/tagger` | `createTagger()` fluent API reference | + +## Documentation + +Full parameter tables and examples live in the docs: +**https://www.walkeros.io/docs/apps/mcp** + +## Contribute + +Feel free to contribute by submitting an +[issue](https://github.com/elbwalker/walkerOS/issues), starting a +[discussion](https://github.com/elbwalker/walkerOS/discussions), or getting in +[contact](https://calendly.com/elb-alexander/30min). + +## License + +MIT diff --git a/packages/mcps/source-browser/package.json b/packages/mcps/source-browser/package.json index bf452fd9d..a3f2f27aa 100644 --- a/packages/mcps/source-browser/package.json +++ b/packages/mcps/source-browser/package.json @@ -48,12 +48,18 @@ "engines": { "node": ">=18.0.0" }, + "mcpName": "io.walkeros/source-browser", "keywords": [ "walkerOS", "mcp", + "model-context-protocol", + "mcp-server", + "ai-agents", + "llm", "tagging", "data-elb", "analytics", + "event-tracking", "html-attributes" ], "publishConfig": { diff --git a/packages/mcps/source-browser/server.json b/packages/mcps/source-browser/server.json new file mode 100644 index 000000000..2572d9d10 --- /dev/null +++ b/packages/mcps/source-browser/server.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json", + "name": "io.walkeros/source-browser", + "title": "walkerOS Source Browser", + "description": "Generate, parse, and validate walkerOS data-elb HTML tagging attributes", + "version": "4.4.0", + "websiteUrl": "https://www.walkeros.io/docs/apps/mcp", + "repository": { + "url": "https://github.com/elbwalker/walkerOS", + "source": "github", + "subfolder": "packages/mcps/source-browser" + }, + "packages": [ + { + "registryType": "npm", + "registryBaseUrl": "https://registry.npmjs.org", + "identifier": "@walkeros/mcp-source-browser", + "version": "4.4.0", + "transport": { + "type": "stdio" + } + } + ] +} diff --git a/packages/server/sources/express/src/__tests__/async-per-method.test.ts b/packages/server/sources/express/src/__tests__/async-per-method.test.ts new file mode 100644 index 000000000..d3d650c28 --- /dev/null +++ b/packages/server/sources/express/src/__tests__/async-per-method.test.ts @@ -0,0 +1,172 @@ +import { startFlow } from '@walkeros/collector'; +import { Source } from '@walkeros/core'; +import type { Destination, RespondFn } from '@walkeros/core'; +import type { Request, Response } from 'express'; +import { sourceExpress } from '../index'; +import type { Types as ExpressTypes } from '../types'; + +/** + * Per-method `async` resolution at the express boundary, proven through a + * real `startFlow`: GET defaults to synchronous so a step's respond wins + * over the GIF fallback, POST defaults to respond-first, and a + * `{ GET?, POST? }` record overrides one method without touching the other. + */ +type ResponderTypes = Destination.Types< + unknown, + unknown, + { respond?: RespondFn } +>; + +const ASSET_EVENT = 'asset get'; + +function createHarness(asyncConfig?: boolean | Record) { + const order: string[] = []; + + // Serves real content for the asset event only; records every delivery. + const responder: Destination.Instance = { + type: 'responder', + config: {}, + push: async (event, ctx) => { + order.push(`delivered:${event.name}`); + if (event.name === ASSET_EVENT) { + ctx.env?.respond?.({ + body: 'REAL', + status: 200, + headers: { 'Content-Type': 'application/javascript' }, + }); + } + }, + }; + + const flow = startFlow({ + consent: { functional: true }, + sources: { + express: { + code: sourceExpress, + config: { + settings: { paths: ['/asset'] }, + ...(asyncConfig === undefined ? {} : { async: asyncConfig }), + }, + }, + }, + destinations: { + responder: { code: responder }, + }, + }); + + const mockGet = (): Request => + ({ + method: 'GET', + url: `/asset?name=${encodeURIComponent(ASSET_EVENT)}`, + headers: {}, + get: () => undefined, + }) as unknown as Request; + + const mockPost = (): Request => + ({ + method: 'POST', + url: '/asset', + body: { name: 'page view' }, + headers: {}, + get: () => undefined, + }) as unknown as Request; + + const mockResponse = () => { + const calls: { method: 'send' | 'json'; body: unknown; status: number }[] = + []; + const headers: Record = {}; + let status = 200; + const res = { + status: (code: number) => { + status = code; + return res; + }, + set: (key: string, value: string) => { + headers[key] = value; + return res; + }, + send: (body?: unknown) => { + order.push('respond'); + calls.push({ method: 'send', body, status }); + return res; + }, + json: (body: unknown) => { + order.push('respond'); + calls.push({ method: 'json', body, status }); + return res; + }, + }; + return { res: res as unknown as Response, calls, headers }; + }; + + return { flow, order, mockGet, mockPost, mockResponse }; +} + +const flushDelivery = () => + new Promise((resolve) => setImmediate(resolve)); + +describe('Express per-method async', () => { + it('GET default awaits the push so a step response wins over the GIF', async () => { + const harness = createHarness(); + const { collector } = await harness.flow; + const source = Source.getSource(collector, 'express'); + + const { res, calls, headers } = harness.mockResponse(); + await source.push(harness.mockGet(), res); + + expect(calls).toHaveLength(1); + expect(calls[0]).toEqual({ method: 'send', body: 'REAL', status: 200 }); + expect(headers['Content-Type']).toBe('application/javascript'); + }); + + it('POST default responds before delivery settles', async () => { + const harness = createHarness(); + const { collector } = await harness.flow; + const source = Source.getSource(collector, 'express'); + + const { res, calls } = harness.mockResponse(); + await source.push(harness.mockPost(), res); + + expect(calls).toHaveLength(1); + expect(calls[0].method).toBe('json'); + expect(calls[0].body).toMatchObject({ success: true }); + expect(harness.order[0]).toBe('respond'); + + await flushDelivery(); + expect(harness.order).toContain('delivered:page view'); + }); + + it('async {POST:false} makes POST synchronous while GET keeps its sync default', async () => { + const harness = createHarness({ POST: false }); + const { collector } = await harness.flow; + const source = Source.getSource(collector, 'express'); + + const post = harness.mockResponse(); + await source.push(harness.mockPost(), post.res); + expect(post.calls).toHaveLength(1); + expect(post.calls[0].body).toMatchObject({ success: true }); + expect(harness.order.indexOf('delivered:page view')).toBeLessThan( + harness.order.indexOf('respond'), + ); + + const get = harness.mockResponse(); + await source.push(harness.mockGet(), get.res); + expect(get.calls).toHaveLength(1); + expect(get.calls[0]).toEqual({ method: 'send', body: 'REAL', status: 200 }); + }); + + it('async {GET:true} keeps GET respond-first: the GIF wins over the step', async () => { + const harness = createHarness({ GET: true }); + const { collector } = await harness.flow; + const source = Source.getSource(collector, 'express'); + + const { res, calls, headers } = harness.mockResponse(); + await source.push(harness.mockGet(), res); + await flushDelivery(); + + expect(calls).toHaveLength(1); + expect(calls[0].method).toBe('send'); + expect(Buffer.isBuffer(calls[0].body)).toBe(true); + expect(headers['Content-Type']).toBe('image/gif'); + }); +}); diff --git a/packages/server/sources/express/src/__tests__/concurrent-requests.test.ts b/packages/server/sources/express/src/__tests__/concurrent-requests.test.ts index 7bcb81edf..6944f5832 100644 --- a/packages/server/sources/express/src/__tests__/concurrent-requests.test.ts +++ b/packages/server/sources/express/src/__tests__/concurrent-requests.test.ts @@ -1,6 +1,6 @@ import { startFlow } from '@walkeros/collector'; import { Source } from '@walkeros/core'; -import type { Destination, Ingest, WalkerOS } from '@walkeros/core'; +import type { Destination, Ingest, RespondFn, WalkerOS } from '@walkeros/core'; import type { Request, Response } from 'express'; import { sourceExpress } from '../index'; import type { Types as ExpressTypes } from '../types'; @@ -121,4 +121,110 @@ describe('Express concurrent requests', () => { expect(body.success).toBe(true); } }); + + it('keeps sync GETs and respond-first POSTs isolated under the method defaults', async () => { + type ResponderTypes = Destination.Types< + unknown, + unknown, + { respond?: RespondFn } + >; + + // Serves per-request content for asset events; ignores ingestion events. + const responder: Destination.Instance = { + type: 'responder', + config: {}, + push: async (event, ctx) => { + if (event.name === 'asset get') { + ctx.env?.respond?.({ + body: `FILE:${String(event.data?.id)}`, + status: 200, + headers: { 'Content-Type': 'application/javascript' }, + }); + } + }, + }; + + const { collector } = await startFlow({ + consent: { functional: true }, + sources: { + express: { + code: sourceExpress, + config: { + settings: { paths: ['/collect'] }, + }, + }, + }, + destinations: { + responder: { code: responder }, + }, + }); + + const expressSource = Source.getSource(collector, 'express'); + + const mockGet = (id: number): Request => + ({ + method: 'GET', + url: `/collect?name=asset%20get&data[id]=${id}`, + headers: {}, + get: () => undefined, + }) as unknown as Request; + + const mockPost = (id: number): Request => + ({ + method: 'POST', + url: '/collect', + headers: { 'content-type': 'application/json' }, + body: { name: 'page view', data: { id } }, + }) as unknown as Request; + + const mockResponse = () => { + const captures: { status: number; body: unknown }[] = []; + let currentStatus = 200; + const res = { + status: (code: number) => { + currentStatus = code; + return res; + }, + set: () => res, + send: (body?: unknown) => { + captures.push({ status: currentStatus, body }); + return res; + }, + json: (body: unknown) => { + captures.push({ status: currentStatus, body }); + return res; + }, + }; + return { res: res as unknown as Response, captures }; + }; + + const N = 10; + const gets = Array.from({ length: N }, (_, id) => { + const { res, captures } = mockResponse(); + return { id, req: mockGet(id), res, captures }; + }); + const posts = Array.from({ length: N }, (_, id) => { + const { res, captures } = mockResponse(); + return { id, req: mockPost(id), res, captures }; + }); + + await Promise.all( + [...gets, ...posts].map(({ req, res }) => expressSource.push(req, res)), + ); + + // Each GET got its own served bytes, never the GIF, never a sibling's. + for (const { id, captures } of gets) { + expect(captures).toHaveLength(1); + expect(captures[0].status).toBe(200); + expect(captures[0].body).toBe(`FILE:${id}`); + } + + // Each POST got the respond-first ack, never a served file body. + for (const { captures } of posts) { + expect(captures).toHaveLength(1); + expect(captures[0].status).toBe(200); + const body = captures[0].body as { success: boolean }; + expect(body.success).toBe(true); + } + }); }); diff --git a/packages/server/sources/express/src/__tests__/index.test.ts b/packages/server/sources/express/src/__tests__/index.test.ts index 67987bdbc..8db5cf12f 100644 --- a/packages/server/sources/express/src/__tests__/index.test.ts +++ b/packages/server/sources/express/src/__tests__/index.test.ts @@ -514,6 +514,39 @@ describe('sourceExpress', () => { expect(res.responseHeaders?.['Content-Type']).toBe('image/gif'); expect(Buffer.isBuffer(res.responseBody)).toBe(true); }); + + it('GET sync (default) answers 500 when the push rejects', async () => { + const logger = createMockLogger(); + const error = new Error('delivery failed'); + const rejectingPush = jest.fn().mockRejectedValue(error); + const source = await sourceExpress( + createSourceContext( + {}, + { + push: rejectingPush as never, + command: mockCommand as never, + elb: jest.fn() as never, + logger, + }, + ), + ); + + const req = createMockRequest({ + method: 'GET', + url: '/collect.gif?event=page%20view', + }); + const res = createMockResponse(); + + await source.push(req, res); + + // A rejected push is a server fault: it surfaces as 500, not the GIF. + expect(logger.error).toHaveBeenCalledWith(error); + expect(res.statusCode).toBe(500); + expect(res.responseBody).toEqual({ + success: false, + error: 'Internal server error', + }); + }); }); describe('OPTIONS request handling (CORS)', () => { @@ -804,11 +837,11 @@ describe('sourceExpress', () => { return { push, resolve, reject }; } - it('GET returns the GIF even if push never resolves', async () => { + it('GET async {GET:true} returns the GIF even if push never resolves', async () => { const { push } = createDeferredPush(); const source = await sourceExpress( createSourceContext( - {}, + { async: { GET: true } }, { push: push as never, command: mockCommand as never, @@ -833,12 +866,12 @@ describe('sourceExpress', () => { // Push is still pending — we did not block on it. }); - it('GET logs a rejected push and does not throw out of the handler', async () => { + it('GET async {GET:true} logs a rejected push and does not throw out of the handler', async () => { const { push, reject } = createDeferredPush(); const logger = createMockLogger(); const source = await sourceExpress( createSourceContext( - {}, + { async: { GET: true } }, { push: push as never, command: mockCommand as never, @@ -1127,12 +1160,12 @@ describe('sourceExpress', () => { expect(mockLogger.error).not.toHaveBeenCalled(); }); - it('GET async warns when the push settles ok:false after the GIF', async () => { + it('GET async {GET:true} warns when the push settles ok:false after the GIF', async () => { const mockLogger = createMockLogger(); const okFalsePush = jest.fn().mockResolvedValue({ ok: false }); const source = await sourceExpress( createSourceContext( - {}, + { async: { GET: true } }, { push: okFalsePush as never, command: mockCommand as never, diff --git a/packages/server/sources/express/src/__tests__/respond-mode.test.ts b/packages/server/sources/express/src/__tests__/respond-mode.test.ts new file mode 100644 index 000000000..b8ebda370 --- /dev/null +++ b/packages/server/sources/express/src/__tests__/respond-mode.test.ts @@ -0,0 +1,23 @@ +import { resolveRespondFirst } from '../respond-mode'; +import type { RouteMethod } from '../types'; + +describe('resolveRespondFirst', () => { + it.each< + [boolean | Record | undefined, RouteMethod, boolean] + >([ + [undefined, 'GET', false], + [undefined, 'POST', true], + [true, 'GET', true], + [true, 'POST', true], + [false, 'GET', false], + [false, 'POST', false], + [{ GET: false, POST: true }, 'GET', false], + [{ GET: false, POST: true }, 'POST', true], + [{ GET: true }, 'GET', true], + [{ GET: true }, 'POST', true], + [{ POST: false }, 'GET', false], + [{ POST: false }, 'POST', false], + ])('resolves %j for %s to %p', (asyncConfig, method, expected) => { + expect(resolveRespondFirst(asyncConfig, method)).toBe(expected); + }); +}); diff --git a/packages/server/sources/express/src/index.ts b/packages/server/sources/express/src/index.ts index 713db23fb..392ee9d1e 100644 --- a/packages/server/sources/express/src/index.ts +++ b/packages/server/sources/express/src/index.ts @@ -16,6 +16,7 @@ import type { Elb, Logger, Source } from '@walkeros/core'; import type { ExpressSource, Types, EventRequest } from './types'; import { setCorsHeaders, TRANSPARENT_GIF } from './utils'; import { buildScope } from './scope'; +import { resolveRespondFirst } from './respond-mode'; /** * Normalize an unknown rejection reason into an Error for the logger. @@ -127,9 +128,6 @@ export const sourceExpress = async ( maxBatchSize: userSettings.maxBatchSize ?? 100, }; - // Respond-first by default: a 2xx means "accepted", not "delivered". - // Standardized on the source config (Source.Config.async), not settings. - const respondFirst = config.async ?? true; const maxBatchSize = settings.maxBatchSize; // Rejection volume belongs in collector.status, not in per-request logs. @@ -234,7 +232,7 @@ export const sourceExpress = async ( }); if (parsedData && typeof parsedData === 'object') { - if (respondFirst) { + if (resolveRespondFirst(config.async, 'GET')) { // Respond-first: the tracking pixel must return instantly and // never block on backend delivery. Fire the push without // awaiting; a rejected push and a settled non-invalid decline @@ -272,7 +270,7 @@ export const sourceExpress = async ( return; } - if (respondFirst) { + if (resolveRespondFirst(config.async, 'POST')) { // Respond-first ("accepted"), then deliver asynchronously. A // rejected push and a settled non-invalid decline are logged, not // surfaced to the client and not left unhandled (destination @@ -452,6 +450,7 @@ export type { Settings, RouteConfig, RouteMethod, + AsyncByMethod, } from './types'; // Export utils diff --git a/packages/server/sources/express/src/respond-mode.ts b/packages/server/sources/express/src/respond-mode.ts new file mode 100644 index 000000000..8b2f67e8c --- /dev/null +++ b/packages/server/sources/express/src/respond-mode.ts @@ -0,0 +1,31 @@ +import type { RouteMethod } from './types'; + +/** + * Per-method respond-first defaults. GET is synchronous so a step (a file + * transformer, a cache destination) can respond with real content before + * the GIF fallback applies; POST is respond-first so ingestion acks fast + * and delivers asynchronously. + */ +const METHOD_DEFAULTS: Record = { + GET: false, + POST: true, +}; + +/** + * Resolves the effective respond-first flag for one request method from + * `config.async`. A boolean applies to both methods; a record overrides per + * method, with unset methods falling back to the method default; undefined + * yields the method default. + * + * @param asyncConfig The source's `config.async` value. + * @param method The request method to resolve for. + * @returns Whether the handler responds before the push settles. + */ +export function resolveRespondFirst( + asyncConfig: boolean | Record | undefined, + method: RouteMethod, +): boolean { + if (asyncConfig === undefined) return METHOD_DEFAULTS[method]; + if (typeof asyncConfig === 'boolean') return asyncConfig; + return asyncConfig[method] ?? METHOD_DEFAULTS[method]; +} diff --git a/packages/server/sources/express/src/types.ts b/packages/server/sources/express/src/types.ts index e1ee8cfb2..e4599bfe9 100644 --- a/packages/server/sources/express/src/types.ts +++ b/packages/server/sources/express/src/types.ts @@ -20,6 +20,7 @@ export type Settings = z.infer; export type CorsOptions = z.infer; export type RouteConfig = z.infer; export type RouteMethod = 'GET' | 'POST'; +export type AsyncByMethod = Partial>; export type InitSettings = Partial; export interface Mapping { @@ -29,11 +30,13 @@ export interface Mapping { // Express-specific push type (uses Express Request/Response types) // // Ack contract: a 2xx response means the event was *accepted*, not that it was -// *delivered*. With `config.async` (the default, `true`) the handler responds -// first and pushes to the collector without blocking the response; rejected -// pushes are logged and destination errors are DLQ'd inside the collector. GET -// (the tracking pixel) and POST both honor the flag: set `config.async: false` -// to make the response wait for delivery to settle before replying. +// *delivered*. `config.async` resolves per method (boolean for both methods, a +// `{ GET?, POST? }` record per method, defaults `{ GET: false, POST: true }`). +// Respond-first (`true`) answers before the push settles; rejected pushes are +// logged and destination errors are DLQ'd inside the collector. Synchronous +// (`false`) awaits delivery: a GET lets a step respond with real content +// before the GIF fallback, a POST reflects the outcome (400/500, batch +// outcomes) instead of the blind accepted ack. export type Push = (req: Request, res: Response) => Promise; export interface Env extends CoreSource.Env { diff --git a/packages/server/transformers/fingerprint/README.md b/packages/server/transformers/fingerprint/README.md new file mode 100644 index 000000000..250b6764f --- /dev/null +++ b/packages/server/transformers/fingerprint/README.md @@ -0,0 +1,90 @@ +

+ + walkerOS + +

+ +# @walkeros/server-transformer-fingerprint + +Server-side user identification for walkerOS without cookies. Hashes +configurable request fields into a deterministic identifier and stores it on the +event. No cookies, no PII stored: the same inputs always produce the same hash, +which gives session continuity and cross-domain stitching without a client-side +ID. + +[Documentation](https://www.walkeros.io/docs/transformers/fingerprint) • +[NPM Package](https://www.npmjs.com/package/@walkeros/server-transformer-fingerprint) +• +[Source Code](https://github.com/elbwalker/walkerOS/tree/main/packages/server/transformers/fingerprint) + +## Installation + +```bash +npm install @walkeros/server-transformer-fingerprint +``` + +## Quick start + +```typescript +import { startFlow } from '@walkeros/collector'; +import { transformerFingerprint } from '@walkeros/server-transformer-fingerprint'; + +await startFlow({ + transformers: { + fingerprint: { + code: transformerFingerprint, + config: { + settings: { + fields: ['ingest.ip', 'ingest.userAgent'], + output: 'user.hash', + length: 16, + }, + }, + }, + }, +}); +``` + +The event then carries the hash at the configured `output` path: + +```json +{ "name": "page view", "user": { "hash": "158f99cc06e33fd6" } } +``` + +Fields resolve from `{ event, ingest }` using walkerOS mapping. Strings use dot +notation, and function values compute dynamically. A missing field is treated as +an empty string, and the transformer never throws. + +## Daily rotation + +Without rotation the same IP and user agent produce the same hash indefinitely. +Add a date field to reset it each day, which limits cross-day tracking while +keeping session continuity within a day: + +```typescript +settings: { + fields: [ + 'ingest.ip', + 'ingest.userAgent', + { fn: () => new Date().toISOString().slice(0, 10) }, + ], + output: 'user.hash', + length: 16, +} +``` + +## Documentation + +Full configuration, IP anonymization, and examples live in the docs: +**https://www.walkeros.io/docs/transformers/fingerprint** + +## Contribute + +Feel free to contribute by submitting an +[issue](https://github.com/elbwalker/walkerOS/issues), starting a +[discussion](https://github.com/elbwalker/walkerOS/discussions), or getting in +[contact](https://calendly.com/elb-alexander/30min). + +## License + +MIT diff --git a/packages/transformers/validate/README.md b/packages/transformers/validate/README.md new file mode 100644 index 000000000..3d53a2319 --- /dev/null +++ b/packages/transformers/validate/README.md @@ -0,0 +1,100 @@ +

+ + walkerOS + +

+ +# @walkeros/transformer-validate + +Enforce JSON Schema contracts on walkerOS events at runtime. Checks each event +against its contract and records a verdict, either annotating the event and +continuing or dropping it. Runs on both web and server. + +[Documentation](https://www.walkeros.io/docs/transformers/validate) • +[NPM Package](https://www.npmjs.com/package/@walkeros/transformer-validate) +• +[Source Code](https://github.com/elbwalker/walkerOS/tree/main/packages/transformers/validate) + +## Installation + +```bash +npm install @walkeros/transformer-validate +``` + +## Quick start + +```typescript +import { startFlow } from '@walkeros/collector'; +import { transformerValidate } from '@walkeros/transformer-validate'; + +await startFlow({ + transformers: { + validate: { + code: transformerValidate, + config: { settings: { contract: [contractWeb], mode: 'strict' } }, + }, + }, +}); +``` + +The same step in a bundled flow, referencing a named contract from the top-level +`contract` block: + +```json +{ + "version": 4, + "contract": { + "web": { + "events": { + "order": { + "complete": { + "properties": { "data": { "required": ["total", "currency"] } } + } + } + } + } + }, + "flows": { + "default": { + "transformers": { + "validate": { + "package": "@walkeros/transformer-validate", + "config": { + "settings": { "contract": ["$contract.web"], "mode": "strict" } + }, + "next": "ga4" + } + } + } + } +} +``` + +## Modes + +`mode` decides what happens to an invalid event: + +- **`pass`** (default) writes the verdict to the event and continues, so a + downstream step can route on `event.source.valid`. +- **`strict`** records the errors and stops the chain, so the event never + reaches downstream steps. + +The boolean verdict goes onto the event as data that travels with it. The issue +list goes onto the ingest as diagnostics, so it survives even a strict-mode +drop. + +## Documentation + +Full configuration, contract references, and examples live in the docs: +**https://www.walkeros.io/docs/transformers/validate** + +## Contribute + +Feel free to contribute by submitting an +[issue](https://github.com/elbwalker/walkerOS/issues), starting a +[discussion](https://github.com/elbwalker/walkerOS/discussions), or getting in +[contact](https://calendly.com/elb-alexander/30min). + +## License + +MIT diff --git a/packages/web/destinations/gtag/src/__tests__/gtm.test.ts b/packages/web/destinations/gtag/src/__tests__/gtm.test.ts index 73c2762d6..a8e4f4469 100644 --- a/packages/web/destinations/gtag/src/__tests__/gtm.test.ts +++ b/packages/web/destinations/gtag/src/__tests__/gtm.test.ts @@ -47,16 +47,72 @@ describe('GTM Implementation', () => { }); }); - it('should still push init event when loadScript is false', () => { + const falsyLoadScript: Array<[string, boolean | undefined]> = [ + ['false', false], + ['undefined', undefined], + ]; + + it.each(falsyLoadScript)( + 'should not push the init event when loadScript is %s', + (_label, loadScript) => { + const settings: GTMSettings = { containerId: 'GTM-XXXXXXX' }; + + initGTM(settings, loadScript, mockEnv); + + expect(mockDataLayer).toHaveLength(0); + }, + ); + + it('should leave an existing dataLayer untouched when loadScript is off', () => { const settings: GTMSettings = { containerId: 'GTM-XXXXXXX' }; + mockDataLayer.push({ event: 'existing' }); initGTM(settings, false, mockEnv); - expect(mockDataLayer).toHaveLength(1); - expect(mockDataLayer[0]).toEqual({ - 'gtm.start': expect.any(Number), - event: 'gtm.js', - }); + expect(mockEnv.window.dataLayer).toBe(mockDataLayer); + expect(mockDataLayer).toEqual([{ event: 'existing' }]); + }); + + it('should adopt the dataLayer without pushing when loadScript is off', () => { + const customEnv = clone(examples.env.push); + const settings: GTMSettings = { + containerId: 'GTM-XXXXXXX', + dataLayer: 'customDataLayer', + }; + + initGTM(settings, false, customEnv); + + expect(customEnv.window).toHaveProperty('customDataLayer', []); + }); + + it('should push the init event and load the container when loadScript is true', () => { + const script = { + src: '', + setAttribute: () => {}, + removeAttribute: () => {}, + }; + const appended: unknown[] = []; + const env: Env = { + window: { dataLayer: mockDataLayer }, + document: { + createElement: () => script, + head: { + appendChild: (node) => { + appended.push(node); + }, + }, + }, + }; + + initGTM({ containerId: 'GTM-XXXXXXX' }, true, env); + + expect(mockDataLayer).toEqual([ + { 'gtm.start': expect.any(Number), event: 'gtm.js' }, + ]); + expect(appended).toHaveLength(1); + expect(script.src).toBe( + 'https://www.googletagmanager.com/gtm.js?id=GTM-XXXXXXX', + ); }); }); diff --git a/packages/web/destinations/gtag/src/__tests__/stepExamples.test.ts b/packages/web/destinations/gtag/src/__tests__/stepExamples.test.ts index 1eff580ce..40b1d9bd4 100644 --- a/packages/web/destinations/gtag/src/__tests__/stepExamples.test.ts +++ b/packages/web/destinations/gtag/src/__tests__/stepExamples.test.ts @@ -155,6 +155,7 @@ describe('gtag web destination -- step examples', () => { // GA4 is active. Build a composite bootstrap if ads/gtm also needed. if (mappingSettings.ads || mappingSettings.gtm) { const composite: Destination.Config = { + loadScript: true, settings: { ...((ga4InitIn.settings || {}) as Record), ...(mappingSettings.ads @@ -209,6 +210,7 @@ describe('gtag web destination -- step examples', () => { } else { // ads + gtm together bootstrapConfig = { + loadScript: true, settings: { ads: { conversionId: 'AW-123456789', currency: 'EUR' }, gtm: { containerId: 'GTM-XXXXXXX' }, diff --git a/packages/web/destinations/gtag/src/examples/step.ts b/packages/web/destinations/gtag/src/examples/step.ts index 09ae31cac..89f333ad7 100644 --- a/packages/web/destinations/gtag/src/examples/step.ts +++ b/packages/web/destinations/gtag/src/examples/step.ts @@ -48,13 +48,14 @@ export const adsInit: Flow.StepExample = { /** * GTM destination bootstrap. - * Initializes the dataLayer and pushes the gtm.js start event. + * Loads the container and pushes the gtm.js start event. */ export const gtmInit: Flow.StepExample = { title: 'GTM init', description: - 'The destination initializes the dataLayer and pushes the gtm.js start event for a GTM container.', + 'With script loading enabled the destination initializes the dataLayer, pushes the gtm.js start event, and loads the GTM container.', in: { + loadScript: true, settings: { gtm: { containerId: 'GTM-XXXXXXX' }, }, diff --git a/packages/web/destinations/gtag/src/gtm/config.ts b/packages/web/destinations/gtag/src/gtm/config.ts index 96e5b334c..4d6752870 100644 --- a/packages/web/destinations/gtag/src/gtm/config.ts +++ b/packages/web/destinations/gtag/src/gtm/config.ts @@ -22,14 +22,17 @@ export function initGTM( const dataLayerArray: unknown[] = isArray(existing) ? existing : []; window[dataLayerName] = dataLayerArray; - dataLayerArray.push({ - 'gtm.start': new Date().getTime(), - event: 'gtm.js', - }); + // Load the gtm script and container. The `gtm.js` bootstrap object is part of + // that snippet and belongs to whoever loads the container: a container loaded + // elsewhere pushes its own, and a second one re-fires its Page View triggers. + if (loadScript && containerId) { + dataLayerArray.push({ + 'gtm.start': new Date().getTime(), + event: 'gtm.js', + }); - // Load the gtm script and container - if (loadScript && containerId) addScript(containerId, domain || defaultDomain, dataLayerName, document); + } } function addScript( diff --git a/skills/walkeros-create-destination/SKILL.md b/skills/walkeros-create-destination/SKILL.md index 1fc12bb67..e3992c0bd 100644 --- a/skills/walkeros-create-destination/SKILL.md +++ b/skills/walkeros-create-destination/SKILL.md @@ -1,8 +1,8 @@ --- name: walkeros-create-destination -description: - Use when creating a new walkerOS destination to send events to a vendor or - API (GA4/gtag, Meta/Facebook Pixel, Mixpanel, Amplitude, a custom HTTP API, +description: >- + Use when creating a new walkerOS destination to send events to a vendor or API + (GA4/gtag, Meta/Facebook Pixel, Mixpanel, Amplitude, a custom HTTP API, Measurement Protocol), web or server-side. Example-driven workflow: research the vendor SDK and define step examples before implementing the destination interface, env pattern, and mappings. diff --git a/skills/walkeros-create-source/SKILL.md b/skills/walkeros-create-source/SKILL.md index 7d1f80d94..d17216c08 100644 --- a/skills/walkeros-create-source/SKILL.md +++ b/skills/walkeros-create-source/SKILL.md @@ -1,6 +1,6 @@ --- name: walkeros-create-source -description: +description: >- Use when creating a new walkerOS source to capture events (browser source, dataLayer interception, server/HTTP source, webhook receiver, event capture), web or server-side. Example-driven workflow: research the input format and diff --git a/skills/walkeros-using-cli/SKILL.md b/skills/walkeros-using-cli/SKILL.md index db5a32d8f..718ae35dd 100644 --- a/skills/walkeros-using-cli/SKILL.md +++ b/skills/walkeros-using-cli/SKILL.md @@ -36,16 +36,17 @@ walkeros push flow.json -e '{"entity":"page","action":"view"}' ## Commands Overview -| Command | Purpose | Safe? | -| ---------- | -------------------------------------------------------- | ----- | -| `bundle` | Generate JS bundle from config | ✅ | -| `push` | Execute with real API calls (or `--simulate` for mocked) | ⚠️ | -| `run` | Local HTTP event collection | ✅ | -| `deploy` | Deploy flows to cloud | ⚠️ | -| `previews` | Manage preview bundles for testing on live sites | ⚠️ | -| `observe` | Start a live observation session for a flow (app login) | ⚠️ | -| `validate` | Validate configs/events | ✅ | -| `cache` | Manage caching | ✅ | +| Command | Purpose | Safe? | +| ---------- | ----------------------------------------------------------- | ----- | +| `bundle` | Generate JS bundle from config | ✅ | +| `push` | Execute with real API calls (or `--simulate` for mocked) | ⚠️ | +| `run` | Local HTTP event collection | ✅ | +| `setup` | Run a component's `setup()` to provision external resources | ⚠️ | +| `deploy` | Deploy flows to cloud | ⚠️ | +| `previews` | Manage preview bundles for testing on live sites | ⚠️ | +| `observe` | Start a live observation session for a flow (app login) | ⚠️ | +| `validate` | Validate configs/events | ✅ | +| `cache` | Manage caching | ✅ | For detailed command reference, see [commands-reference.md](commands-reference.md). @@ -358,6 +359,22 @@ walkeros run flow.tar.gz --port 8080 walkeros run https://example.com/flow.tar.gz ``` +### Setup Command + +```bash +# Provision external resources for one component (explicit only, never +# triggered by push, simulate, deploy, or the runtime) +walkeros setup . [-c ./flow.json] [-f ] [--json] [--verbose] [--silent] +``` + +The target uses the same `.` syntax as `walkeros push --simulate` +(`source`, `destination`, or `store`). Package resolution follows the flow's +pins exactly like `bundle`: the version from `config.bundle.packages` (or inline +`@scope/pkg@x.y.z`, bundle pin wins) is downloaded through the shared pacote +pipeline and cache, then imported from the extracted tree. Setup works via npx +without a local install; `path:` entries in `config.bundle.packages` support +local development with a built package directory. + --- ## Bundler Gotchas diff --git a/website/docs/apps/cli.mdx b/website/docs/apps/cli.mdx index b6f56fb54..c1649154c 100644 --- a/website/docs/apps/cli.mdx +++ b/website/docs/apps/cli.mdx @@ -704,6 +704,37 @@ The target uses the same `.` syntax as `walkeros push --simulate`. V | `source.events-in` | The source named `events-in` | | `store.session` | The store named `session` | +### Package resolution + +Setup resolves the component's package exactly like `walkeros bundle`: the version pinned in `config.bundle.packages` (or inline on the step as `"@scope/pkg@x.y.z"`, with the bundle pin winning on disagreement) is downloaded from the npm registry into a temporary tree and imported from there. + + + +What this means in practice: + +- **Works from any install.** npx, devDependency, or global: no local `npm install` of the component package is needed or consulted. +- **Shared cache with bundle.** Exact version pins keep working offline once cached; `latest` and ranges re-resolve against the registry. +- **Local packages via `path`.** A `path` entry in `config.bundle.packages` is supported for local development; the target must be a built package directory (package.json plus dist output), since setup imports with Node at runtime. +- **No version drift with exact pins.** Setup and bundle resolve through the same pipeline and cache, so an exact version pin guarantees both act on the same package release and provisioning (for example a BigQuery schema) cannot drift from what the bundle writes. Mutable specifications (`latest`, ranges, `path` targets) can resolve differently between separate runs, so pin exact versions when drift matters. + ### Options +Set `loadScript: true` when walkerOS should install the container for you. It +loads `gtm.js` and pushes the `gtm.js` start event into the dataLayer, which is +what Google's own snippet does. + +Leave `loadScript` unset when the container is already on the page, for example +when you are feeding an existing GTM setup while you migrate. walkerOS then +neither loads the container nor pushes the `gtm.js` bootstrap event: it adopts +the configured dataLayer (creating it if it does not exist yet) and pushes your +mapped events into it, so your existing triggers keep firing exactly as often +as before. + +`loadScript` is set once per destination and applies to every flavor configured +on it. On a destination that also configures `ga4` or `ads`, the same flag +governs whether those scripts are loaded too. + diff --git a/website/docs/sources/envelope.mdx b/website/docs/sources/envelope.mdx index 9e6be26a1..2427885b5 100644 --- a/website/docs/sources/envelope.mdx +++ b/website/docs/sources/envelope.mdx @@ -47,11 +47,12 @@ Single event requests keep their existing per source response bodies. ### 207 requires synchronous mode -The Express source acknowledges before delivery by default (`config.async` is -`true`), so it cannot report per index outcomes: an accepted batch answers 200 -meaning "accepted", not "delivered", and failures surface through -`collector.status` and the logs. Set `config.async: false` to have the response -reflect the delivery outcome. +The Express source acknowledges POST requests before delivery by default +(`async` resolves to `true` for POST), so it cannot report per-index outcomes: +an accepted batch answers 200 meaning "accepted", not "delivered", and +failures surface through `collector.status` and the logs. Set +`config.async: false` or `config.async: { "POST": false }` to have the +response reflect the delivery outcome. ## GET is one event, deliberately diff --git a/website/docs/sources/server/express.mdx b/website/docs/sources/server/express.mdx index c077c71ca..ca25998e7 100644 --- a/website/docs/sources/server/express.mdx +++ b/website/docs/sources/server/express.mdx @@ -140,17 +140,50 @@ Server sources require platform-specific handlers. For containerized deployments Health check endpoints (`/health` and `/ready`) are provided by the [runner](/docs/apps/runner), not by individual sources. This means health checks work regardless of which source type you use. ::: +### Response modes (`async`) + +`config.async` controls whether a request is acknowledged before or after its +event settles in the pipeline, resolved per HTTP method: + + + +| Value | GET | POST | +|---|---|---| +| unset (default) | synchronous | respond-first | +| `true` | respond-first | respond-first | +| `false` | synchronous | synchronous | +| `{ "GET": true }` | respond-first | respond-first (its default) | +| `{ "POST": false }` | synchronous (its default) | synchronous | + +**Synchronous** awaits delivery before answering. On GET, a step (the file +transformer, a cache destination) can respond with real content before the +pixel GIF fallback applies, so asset serving works out of the box. On POST, +the response reflects the outcome: 400 for invalid input, 500 for a processing +failure, and batch envelopes answer per-index outcomes. + +**Respond-first** answers immediately: a 2xx means "accepted", not +"delivered". Delivery continues in the background and failures surface through +`collector.status` and the logs. This is the POST default (fast ingestion) and +opt-in for GET, for hot pixel endpoints that must not wait for the in-process +push. + ### Responses | Status | Meaning | |--------|---------| -| 200 | Accepted (default respond-first mode: accepted for processing, not proof of delivery) or, with `async: false`, processed | -| 400 | Rejected client input: unparseable JSON (body echoes the parser message) or, with `async: false`, an event the pipeline declared invalid (body echoes the validation message, e.g. `Event name is required`) | +| 200 | Accepted (respond-first mode: accepted for processing, not proof of delivery) or, in synchronous mode, processed | +| 400 | Rejected client input: unparseable JSON (body echoes the parser message) or, on synchronous POST, an event the pipeline declared invalid (body echoes the validation message, e.g. `Event name is required`) | | 404 | Path not configured in `paths` | | 405 | Method not allowed on the path | | 413 | Body exceeds the 1mb limit | | 415 | Unsupported charset | -| 500 | With `async: false`: the pipeline failed to process a valid event. Otherwise: unexpected server fault | +| 500 | On synchronous POST: the pipeline failed to process a valid event. Otherwise: unexpected server fault | GET pixel requests always answer the GIF with 200 in every mode, because an image endpoint cannot carry a useful 400. Rejected pixel input is visible through the diff --git a/website/docusaurus.config.ts b/website/docusaurus.config.ts index c91ee6188..948febba6 100644 --- a/website/docusaurus.config.ts +++ b/website/docusaurus.config.ts @@ -243,31 +243,418 @@ const config: Config = { [ '@docusaurus/plugin-client-redirects', { + // Inbound links from the walker.js era still arrive under the old + // `clients/`, `collectors/`, `stacks/`, `utils/` and `consent_management/` + // trees, forwarded here by www.elbwalker.com and docs.elbwalker.com. + // The plugin fails the build when a `to` is not a real route, so these + // stay honest as the docs move. redirects: [ { from: '/docs/sources/web/session/detection', to: '/docs/sources/web/session', }, { - from: '/docs/sources/walkerjs/installation/package', + from: [ + '/docs/sources/walkerjs/installation/package', + '/docs/clients/walkerjs/installation/npm', + '/docs/collectors/web/installation/package', + ], to: '/docs/sources/web/browser/commands#run', }, { - from: '/docs/guides/consent/examples/cookiefirst', + from: [ + '/docs/guides/consent/examples/cookiefirst', + '/docs/consent_management/cookiefirst', + '/docs/guides/consent_management/cookiefirst', + ], to: '/docs/sources/web/cmps/cookiefirst', }, { - from: '/docs/guides/consent/examples/cookiepro', + from: [ + '/docs/guides/consent/examples/cookiepro', + '/docs/consent_management/cookiepro', + '/docs/guides/consent_management/cookiepro', + ], to: '/docs/sources/web/cmps/cookiepro', }, { - from: '/docs/guides/consent/examples/usercentrics', + from: [ + '/docs/guides/consent/examples/usercentrics', + '/docs/consent_management/usercentrics', + '/docs/guides/consent_management/usercentrics', + ], to: '/docs/sources/web/cmps/usercentrics', }, { - from: '/docs/getting-started/stores', + from: [ + '/docs/getting-started/stores', + '/docs/stores/memory', + '/docs/utils/storage', + ], to: '/docs/stores', }, + { + from: [ + '/docs/clients/walkerjs', + '/docs/clients/walkerjs/installation', + '/docs/clients/walkerjs/installation/cdn', + '/docs/clients/walkerjs/installation/gtm', + '/docs/clients/walkerjs/installation/script', + '/docs/clients/walkerjs/versions', + '/docs/clients/walkerjs/versions/2.1.3', + '/docs/clients/walkerjs/versions/2.1.3/installation', + '/docs/clients/walkerjs/versions/2.1.3/installation/gtm', + '/docs/clients/walkerjs/versions/2.1.3/installation/npm', + '/docs/clients/walkerjs/versions/2.1.3/installation/script', + '/docs/sources/walkerjs', + '/docs/sources/walkerjs/configuration', + '/docs/sources/walkerjs/installation', + '/docs/sources/walkerjs/installation/gtm', + '/docs/sources/walkerjs/installation/script', + '/docs/sources/walkerjs/versions', + '/docs/sources/web/browser/configuration', + '/docs/sources/web/browser/installation', + ], + to: '/docs/sources/web/browser', + }, + { + from: [ + '/docs/clients/walkerjs/commands', + '/docs/clients/walkerjs/versions/2.1.3/commands', + '/docs/sources/walkerjs/commands', + '/docs/web/walkerjs/api', + ], + to: '/docs/sources/web/browser/commands', + }, + { + from: [ + '/docs/clients/walkerjs/tagging', + '/docs/clients/walkerjs/trigger', + '/docs/clients/walkerjs/versions/2.1.3/tagging', + '/docs/sources/walkerjs/tagging', + '/docs/sources/html-tagging', + '/docs/sources/web/browser/tagging', + ], + to: '/docs/sources/web/browser/tagging/html-attributes', + }, + { + from: [ + '/docs/clients/walkerjs/using-javascript', + '/docs/clients/walkerjs/versions/2.1.3/using-javascript', + '/docs/sources/walkerjs/using-javascript', + '/docs/sources/javascript-elb', + ], + to: '/docs/sources/web/browser/tagging/javascript', + }, + { + from: '/docs/utils/tagger', + to: '/docs/sources/web/browser/tagger', + }, + { + from: [ + '/docs/sources/dataLayer', + '/docs/sources/dataLayer/configration', + '/docs/sources/dataLayer/configuration', + '/docs/sources/dataLayer/consent_mode', + '/docs/sources/dataLayer/installation', + '/docs/sources/dataLayer/testing', + '/docs/sources/datalayer', + ], + to: '/docs/sources/web/dataLayer', + }, + { + from: [ + '/docs/clients', + '/docs/sources/overview', + '/docs/sources/sources', + '/docs/getting_started/sources', + ], + to: '/docs/sources', + }, + { + from: [ + '/docs/clients/node', + '/docs/clients/node/commands', + '/docs/clients/node/installation', + '/docs/sources/node', + '/docs/sources/node/commands', + '/docs/sources/node/configuration', + '/docs/sources/node/installation', + '/docs/sources/node/versions', + '/docs/stacks', + '/docs/stacks/custom', + '/docs/stacks/gtm', + '/docs/stacks/gtm/tag_template', + ], + to: '/docs/sources/server', + }, + { + from: '/docs/stacks/firebase', + to: '/docs/sources/server/gcp', + }, + { + from: [ + '/docs/collector/configuration', + '/docs/collectors', + '/docs/collectors/node-collector', + '/docs/collectors/server', + '/docs/collectors/server-collector', + '/docs/collectors/server-collector/configuration', + '/docs/collectors/server-collector/installation', + '/docs/collectors/server/configuration', + '/docs/collectors/server/installation', + '/docs/collectors/web', + '/docs/collectors/web-collector', + '/docs/collectors/web-collector/configuration', + '/docs/collectors/web-collector/installation/gtm', + '/docs/collectors/web/configuration', + '/docs/collectors/web/installation', + '/docs/collectors/web/installation/gtm', + '/docs/collectors/web/installation/script', + ], + to: '/docs/collector', + }, + { + from: [ + '/docs/collectors/server/commands', + '/docs/collectors/web/commands', + '/docs/consent_management/commands', + '/docs/clients/walkerjs/hooks', + '/docs/utils/hooks', + ], + to: '/docs/collector/commands', + }, + { + from: [ + '/docs/clients/walkerjs/debugging', + '/docs/clients/walkerjs/testing', + '/docs/collector/testing', + '/docs/collectors/web/testing', + '/docs/sources/node/testing', + '/docs/sources/walkerjs/testing', + ], + to: '/docs/guides/debugging', + }, + { + from: [ + '/docs/clients/walkerjs/user-identification', + '/docs/guides/user_stitching', + '/docs/user_stitching', + ], + to: '/docs/guides/user-stitching', + }, + { + from: '/docs/utils/session', + to: '/docs/guides/session', + }, + { + from: [ + '/docs/consent_management', + '/docs/consent_management/configuration', + '/docs/consent_management/overview', + '/docs/guides/consent_management', + '/docs/getting-started/quickstart/consent-management', + ], + to: '/docs/guides/consent', + }, + { + from: [ + '/docs/destinations/configuration', + '/docs/destinations/overview', + '/docs/destinations/node', + '/docs/destinations/web', + '/docs/getting_started/destinations', + ], + to: '/docs/destinations', + }, + { + from: [ + '/docs/destinations/custom', + '/docs/destinations/web/custom', + '/docs/getting-started/quickstart/custom-destination', + ], + to: '/docs/destinations/create-your-own', + }, + { + from: '/docs/destinations/web/api', + to: '/docs/destinations/api/web', + }, + { + from: '/docs/destinations/node/api', + to: '/docs/destinations/api/server', + }, + { + from: '/docs/web/google', + to: '/docs/destinations/web/gtag', + }, + { + from: [ + '/docs/destinations/google-ga4', + '/docs/destinations/web/ga4', + '/docs/destinations/web/google-ga4', + '/docs/web/google/ga4', + ], + to: '/docs/destinations/web/gtag/ga4', + }, + { + from: [ + '/docs/destinations/google-ads', + '/docs/destinations/web/gads', + '/docs/destinations/web/google-ads', + '/docs/destinations/web/google_ads', + '/docs/web/google/ads', + ], + to: '/docs/destinations/web/gtag/ads', + }, + { + from: [ + '/docs/destinations/google-gtm', + '/docs/destinations/web/google-gtm', + '/docs/destinations/web/gtm', + '/docs/web/google/gtm', + ], + to: '/docs/destinations/web/gtag/gtm', + }, + { + from: [ + '/docs/destinations/meta', + '/docs/destinations/meta-pixel', + '/docs/destinations/web/meta', + '/docs/web/meta', + ], + to: '/docs/destinations/web/meta-pixel', + }, + { + from: [ + '/docs/destinations/piwikpro', + '/docs/destinations/web/piwik', + '/docs/web/piwikpro', + ], + to: '/docs/destinations/web/piwikpro', + }, + { + from: ['/docs/destinations/plausible', '/docs/web/plausible'], + to: '/docs/destinations/web/plausible', + }, + { + from: ['/docs/destinations/aws', '/docs/destinations/node/aws'], + to: '/docs/destinations/server/aws', + }, + { + from: [ + '/docs/destinations/bigquery', + '/docs/destinations/node/bigquery', + '/docs/destinations/server/gcp/bigquery', + '/docs/node/google', + ], + to: '/docs/destinations/server/gcp', + }, + { + from: [ + '/docs/destinations/node/meta', + '/docs/destinations/server/meta', + '/docs/node/meta', + ], + to: '/docs/destinations/server/meta-capi', + }, + { + from: [ + '/docs/destinations/event-mapping', + '/docs/destinations/event_mapping', + '/docs/destinations/mapping', + '/docs/guides/event-mapping', + '/docs/utils/mapping', + '/docs/getting-started/quickstart/custom-mapping-functions', + ], + to: '/docs/mapping', + }, + { + from: '/docs/transformers/cache', + to: '/docs/stores/cache', + }, + { + from: '/docs/transformers/router', + to: '/docs/transformers', + }, + { + from: [ + '/docs/transformers/validator', + '/docs/utils/validate', + '/docs/utils/validation', + ], + to: '/docs/transformers/validate', + }, + { + from: [ + '/docs/utils', + '/docs/utils/helper', + '/docs/utils/installation', + '/docs/utils/versions', + ], + to: '/docs/core', + }, + { + from: [ + '/docs/getting-started/quick-start', + '/docs/getting-started/quickstart/cdn', + '/docs/getting-started/quickstart/walker-with-sources', + '/docs/getting_started/quick_start', + ], + to: '/docs/getting-started/quickstart', + }, + { + from: [ + '/docs/getting_started/event-model', + '/docs/walkeros/event-model', + ], + to: '/docs/getting-started/event-model', + }, + { + from: [ + '/docs/getting-started/step-examples', + '/docs/guides/interactive-examples', + ], + to: '/docs/getting-started/flow/step-examples', + }, + { + from: '/docs/contract', + to: '/docs/getting-started/flow/contract', + }, + { + from: ['/docs/guides/migrations', '/docs/migrations'], + to: '/docs/guides/migration', + }, + { + from: '/docs/migrations/elbwalker_to_walkeros', + to: '/docs/migrating/v3-to-v4', + }, + { + from: '/docs/comparisons/comparisons', + to: '/docs/comparisons', + }, + { + from: '/docs/comparisons/dataLayerGTM', + to: '/docs/comparisons/dataLayer', + }, + { + from: '/docs/guides/gtm', + to: '/docs/comparisons/gtm', + }, + { + from: '/docs/apps', + to: '/docs/apps/walkerjs', + }, + { + from: [ + '/docs/intro', + '/docs/walkeros', + '/docs/walkeros/getting-started', + '/docs/walkeros/privacy', + '/docs/getting-started/what_is_walkeros', + '/docs/getting_started/what_is_walkeros', + ], + to: '/docs/', + }, ], }, ], @@ -296,6 +683,14 @@ const config: Config = { content: { enableMarkdownFiles: true, enableLlmsFullTxt: true, + // llms.txt is read detached from the site: pasted into an agent's + // context, chunked, or fetched on its own. A root-relative target has + // nothing to resolve against there, so the production export is fully + // qualified with the site url. Preview builds stay relative on + // purpose: the plugin appends the baseUrl to the site url while + // route paths already carry it, which doubles the prefix on any + // non-root baseUrl. + relativePaths: Boolean(process.env.DOCUSAURUS_BASEURL), excludeRoutes: ['/search', '/404', '/tags/**'], // The export appends `.md` to the route path, so a trailing-slash // route yields `/docs/mapping/.md` while the page is written to diff --git a/website/scripts/deploy-bunny.mjs b/website/scripts/deploy-bunny.mjs index cc758aafe..66153d88d 100644 --- a/website/scripts/deploy-bunny.mjs +++ b/website/scripts/deploy-bunny.mjs @@ -7,10 +7,25 @@ const API_KEY = process.env.BUNNY_API_KEY; const PULLZONE_URL = process.env.BUNNY_PULLZONE_URL; const DEPLOY_PATH = process.env.DEPLOY_PATH || ''; // e.g., 'preview/pr-123' for PR previews const STORAGE_URL = `https://storage.bunnycdn.com/${STORAGE_ZONE}`; +// Prune scope prefix: '' for production (zone root), '/' for previews. +const SCOPE = DEPLOY_PATH ? `${DEPLOY_PATH}/` : ''; + +const PRUNE_DRY_RUN = process.env.PRUNE_DRY_RUN === '1'; +const PRUNE_FORCE = process.env.PRUNE_FORCE === '1'; +// A build below this size is treated as broken rather than as a site that +// shrank; prune refuses to run so a failed build can never empty the zone. +const PRUNE_MIN_LOCAL_FILES = Number(process.env.PRUNE_MIN_LOCAL_FILES || 50); +// Extra scope-relative prefixes to protect, comma-separated. +const PRUNE_EXCLUDE = (process.env.PRUNE_EXCLUDE || '') + .split(',') + .map((p) => p.trim().replace(/\/+$/, '')) + .filter(Boolean); async function getFiles(dir) { const entries = await readdir(dir, { withFileTypes: true, recursive: true }); - return entries.filter((e) => e.isFile()).map((e) => join(e.parentPath, e.name)); + return entries + .filter((e) => e.isFile()) + .map((e) => join(e.parentPath, e.name)); } async function uploadFile(localPath, remotePath) { @@ -18,15 +33,118 @@ async function uploadFile(localPath, remotePath) { const fullPath = DEPLOY_PATH ? `${DEPLOY_PATH}/${remotePath}` : remotePath; const res = await fetch(`${STORAGE_URL}/${fullPath}`, { method: 'PUT', - headers: { AccessKey: STORAGE_PASSWORD, 'Content-Type': 'application/octet-stream' }, + headers: { + AccessKey: STORAGE_PASSWORD, + 'Content-Type': 'application/octet-stream', + }, body: content, }); if (!res.ok) throw new Error(`Upload failed: ${fullPath} (${res.status})`); console.log(`✓ ${fullPath}`); } +// Retry transient storage-API failures (5xx, 429, network); return definitive +// responses (2xx and other 4xx) to the caller. +async function storageFetch(url, options) { + const maxAttempts = 3; + let lastError; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + const res = await fetch(url, options); + if (res.ok || (res.status < 500 && res.status !== 429)) return res; + lastError = new Error(`HTTP ${res.status}`); + } catch (err) { + lastError = err; + } + if (attempt < maxAttempts) { + await new Promise((r) => setTimeout(r, 1000 * 2 ** (attempt - 1))); + } + } + throw lastError; +} + +// A production deploy owns the zone root EXCEPT PR previews (deployed under +// preview/ by this same script) and Bunny-internal __*__ folders. A preview +// deploy owns its own prefix entirely. +function isProtected(relPath) { + if (PRUNE_EXCLUDE.some((p) => relPath === p || relPath.startsWith(`${p}/`))) { + return true; + } + if (DEPLOY_PATH) return false; + return ( + relPath === 'preview' || + relPath.startsWith('preview/') || + /^__[^/]*__(\/|$)/.test(relPath) + ); +} + +// Recursively list files under the deploy scope, as scope-relative paths. +async function listRemoteFiles(dir = '') { + const res = await storageFetch(`${STORAGE_URL}/${SCOPE}${dir}`, { + headers: { AccessKey: STORAGE_PASSWORD, Accept: 'application/json' }, + }); + if (res.status === 404) return []; // scope does not exist yet + if (!res.ok) throw new Error(`List failed: /${SCOPE}${dir} (${res.status})`); + const entries = await res.json(); + const files = []; + for (const entry of entries) { + const rel = `${dir}${entry.ObjectName}`; + if (entry.IsDirectory) { + if (!isProtected(rel)) files.push(...(await listRemoteFiles(`${rel}/`))); + } else if (!isProtected(rel)) { + files.push(rel); + } + } + return files; +} + +async function deleteRemoteFile(rel) { + const res = await storageFetch(`${STORAGE_URL}/${SCOPE}${rel}`, { + method: 'DELETE', + headers: { AccessKey: STORAGE_PASSWORD }, + }); + if (!res.ok && res.status !== 404) { + throw new Error(`Delete failed: ${SCOPE}${rel} (${res.status})`); + } + console.log(`✗ ${SCOPE}${rel}`); +} + +// The storage zone only ever loses a file through this prune (uploads never +// delete), so removed pages otherwise stay live forever as stale "zombies". +async function pruneRemote(localRelPaths) { + if (localRelPaths.length < PRUNE_MIN_LOCAL_FILES) { + console.warn( + `::warning::Prune skipped: only ${localRelPaths.length} local files (floor ${PRUNE_MIN_LOCAL_FILES}) — build output looks incomplete.`, + ); + return; + } + const local = new Set(localRelPaths); + const remote = await listRemoteFiles(); + const stale = remote.filter((p) => !local.has(p)); + if (stale.length === 0) { + console.log('Prune: no stale remote files.'); + return; + } + const cap = Math.max(200, Math.ceil(remote.length * 0.25)); + if (stale.length > cap && !PRUNE_FORCE) { + console.warn( + `::warning::Prune skipped: ${stale.length} stale of ${remote.length} remote files exceeds the safety cap (${cap}). Inspect with PRUNE_DRY_RUN=1, then set PRUNE_FORCE=1 to prune.`, + ); + return; + } + console.log( + `Pruning ${stale.length} stale remote file(s)${PRUNE_DRY_RUN ? ' (dry run)' : ''}...`, + ); + for (const rel of stale) { + if (PRUNE_DRY_RUN) console.log(`would delete ${SCOPE}${rel}`); + else await deleteRemoteFile(rel); + } +} + async function purgeCache() { - const purgePath = DEPLOY_PATH ? `${PULLZONE_URL}/${DEPLOY_PATH}/*` : `${PULLZONE_URL}/*`; + const purgePath = DEPLOY_PATH + ? `${PULLZONE_URL}/${DEPLOY_PATH}/*` + : `${PULLZONE_URL}/*`; const url = `https://api.bunny.net/purge?url=${encodeURIComponent(purgePath)}`; const maxAttempts = 5; @@ -51,10 +169,14 @@ async function purgeCache() { const text = await res.text(); throw new Error(`Purge failed: ${res.status} - ${text.slice(0, 200)}`); } - console.warn(`Purge attempt ${attempt}/${maxAttempts} failed: ${res.status}`); + console.warn( + `Purge attempt ${attempt}/${maxAttempts} failed: ${res.status}`, + ); } catch (err) { if (err.message?.startsWith('Purge failed:')) throw err; - console.warn(`Purge attempt ${attempt}/${maxAttempts} error: ${err.message}`); + console.warn( + `Purge attempt ${attempt}/${maxAttempts} error: ${err.message}`, + ); } if (attempt < maxAttempts) { @@ -65,19 +187,33 @@ async function purgeCache() { // Upload already succeeded; a failed purge only delays cache eviction. // Don't fail the deploy — warn loudly so the job stays green. - console.warn('::warning::Cache purge failed after retries — new content will propagate as edge TTLs expire.'); + console.warn( + '::warning::Cache purge failed after retries — new content will propagate as edge TTLs expire.', + ); } async function deploy() { const outDir = 'build'; const files = await getFiles(outDir); - console.log(`Uploading ${files.length} files${DEPLOY_PATH ? ` to /${DEPLOY_PATH}` : ''}...`); + console.log( + `Uploading ${files.length} files${DEPLOY_PATH ? ` to /${DEPLOY_PATH}` : ''}...`, + ); for (const file of files) { const remotePath = relative(outDir, file); await uploadFile(file, remotePath); } + try { + // Before purge, so edge caches also drop what was deleted. + await pruneRemote(files.map((file) => relative(outDir, file))); + } catch (e) { + // Uploads succeeded; stale files remaining is the pre-prune status quo. + console.warn( + `::warning::Prune failed: ${e.message} — stale files remain until the next deploy.`, + ); + } + await purgeCache(); console.log('Deploy complete!');