From 8c0fca249e8fafdc3b421e6941eb45ec1932c9b6 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 19 Aug 2026 10:21:50 -0700 Subject: [PATCH 1/7] feat(setup): publish standalone self-hosting package --- .agents/skills/add-block/SKILL.md | 34 +- .agents/skills/add-integration/SKILL.md | 31 +- .agents/skills/validate-integration/SKILL.md | 33 +- .github/workflows/publish-sim-setup.yml | 172 ++ README.md | 58 +- .../docs/en/platform/self-hosting/docker.mdx | 16 +- .../self-hosting/environment-variables.mdx | 2 +- .../en/platform/self-hosting/upgrades.mdx | 4 +- .../(shell)/[slug]/opengraph-image.tsx | 2 +- .../integrations/(shell)/opengraph-image.tsx | 2 +- apps/sim/blocks/registry-maps.ts | 2 +- .../config/env-capabilities.server.test.ts | 6 +- .../core/config/env-capabilities.server.ts | 1 + .../lib/core/config/env-capabilities.test.ts | 2 +- apps/sim/lib/core/config/env-capabilities.ts | 1495 +---------------- apps/sim/lib/core/config/env-flags.ts | 4 +- .../integrations/availability.server.test.ts | 14 +- apps/sim/lib/integrations/availability.ts | 168 +- .../integrations/credential-display.test.ts | 3 +- .../lib/integrations/credential-display.ts | 2 +- .../credential-visibility.server.test.ts | 1 + apps/sim/lib/integrations/index.ts | 2 +- .../lib/integrations/oauth-service.test.ts | 3 +- apps/sim/lib/integrations/oauth-service.ts | 2 +- .../integrations/service-account-metadata.ts | 63 +- apps/sim/lib/oauth/oauth.test.ts | 2 +- apps/sim/package.json | 1 + .../scripts/canvas-sentence-audit-corpus.ts | 2 +- apps/sim/scripts/canvas-sentence-spec.ts | 2 +- bun.lock | 36 + docker-compose.prod.yml | 6 - package.json | 13 +- packages/cli/src/index.ts | 2 +- packages/deployment-config/package.json | 37 + .../deployment-config/src/env-capabilities.ts | 1493 ++++++++++++++++ .../src/integration-availability.ts | 142 ++ .../deployment-config/src}/integrations.json | 0 .../src/service-account-metadata.ts | 48 + .../service-account-providers.generated.ts | 39 + packages/deployment-config/tsconfig.json | 5 + packages/sim-setup/LICENSE | 202 +++ packages/sim-setup/README.md | 14 + packages/sim-setup/THIRD_PARTY_LICENSES | 55 + packages/sim-setup/package.json | 63 + .../sim-setup/src}/banner.ts | 4 +- packages/sim-setup/src/build-assets.ts | 18 + .../sim-setup/src}/capability-config.test.ts | 8 +- .../sim-setup/src}/capability-config.ts | 4 +- .../sim-setup/src}/capability-setup.test.ts | 6 +- .../sim-setup/src}/capability-setup.ts | 6 +- .../sim-setup/src}/capability-status.test.ts | 8 +- .../sim-setup/src}/capability-status.ts | 8 +- .../sim-setup/src}/checks.ts | 27 +- .../sim-setup/src}/cli-auth.ts | 4 +- packages/sim-setup/src/compose-asset.test.ts | 73 + packages/sim-setup/src/compose-asset.ts | 84 + .../src}/configuration-sources.test.ts | 4 +- .../sim-setup/src}/configuration-sources.ts | 4 +- packages/sim-setup/src/context.test.ts | 96 ++ packages/sim-setup/src/context.ts | 112 ++ .../setup => packages/sim-setup/src}/db.ts | 14 +- .../sim-setup/src}/detect.ts | 9 +- .../sim-setup/src}/docker.ts | 12 +- .../sim-setup/src}/doctor.ts | 4 +- .../sim-setup/src}/env-files.test.ts | 4 +- .../sim-setup/src}/env-files.ts | 13 +- .../sim-setup/src}/errors.ts | 0 packages/sim-setup/src/executables.ts | 8 + .../sim-setup/src}/feature-setup.test.ts | 12 +- .../sim-setup/src}/feature-setup.ts | 26 +- packages/sim-setup/src/index.ts | 185 ++ .../sim-setup/src}/lifecycle.test.ts | 4 +- .../sim-setup/src}/lifecycle.ts | 67 +- .../sim-setup/src}/modes/compose.ts | 79 +- .../sim-setup/src}/modes/dev.ts | 24 +- .../sim-setup/src}/modes/k8s.ts | 24 +- .../setup => packages/sim-setup/src}/ports.ts | 8 +- .../sim-setup/src}/probes.ts | 9 +- .../sim-setup/src}/prompter.ts | 42 +- .../setup => packages/sim-setup/src}/redis.ts | 14 +- .../sim-setup/src}/setup-status.test.ts | 4 +- .../sim-setup/src}/setup-status.ts | 22 +- .../sim-setup/src}/steps.test.ts | 8 +- .../setup => packages/sim-setup/src}/steps.ts | 23 +- .../sim-setup/src}/terminal.ts | 0 .../setup => packages/sim-setup/src}/theme.ts | 0 .../setup => packages/sim-setup/src}/twins.ts | 5 +- .../setup => packages/sim-setup/src}/urls.ts | 0 .../sim-setup/src}/wizard.ts | 38 +- packages/sim-setup/tsconfig.json | 5 + packages/sim-setup/vitest.config.ts | 7 + scripts/check-integration-catalog.ts | 2 +- scripts/generate-deployment-config.ts | 96 ++ scripts/generate-docs.ts | 3 +- scripts/run-audits.ts | 1 + scripts/setup/index.ts | 110 -- scripts/setup/launcher.test.ts | 33 - scripts/setup/launcher.ts | 52 - 98 files changed, 3453 insertions(+), 2264 deletions(-) create mode 100644 .github/workflows/publish-sim-setup.yml create mode 100644 packages/deployment-config/package.json create mode 100644 packages/deployment-config/src/env-capabilities.ts create mode 100644 packages/deployment-config/src/integration-availability.ts rename {apps/sim/lib/integrations => packages/deployment-config/src}/integrations.json (100%) create mode 100644 packages/deployment-config/src/service-account-metadata.ts create mode 100644 packages/deployment-config/src/service-account-providers.generated.ts create mode 100644 packages/deployment-config/tsconfig.json create mode 100644 packages/sim-setup/LICENSE create mode 100644 packages/sim-setup/README.md create mode 100644 packages/sim-setup/THIRD_PARTY_LICENSES create mode 100644 packages/sim-setup/package.json rename {scripts/setup => packages/sim-setup/src}/banner.ts (95%) create mode 100644 packages/sim-setup/src/build-assets.ts rename {scripts/setup => packages/sim-setup/src}/capability-config.test.ts (92%) rename {scripts/setup => packages/sim-setup/src}/capability-config.ts (99%) rename {scripts/setup => packages/sim-setup/src}/capability-setup.test.ts (86%) rename {scripts/setup => packages/sim-setup/src}/capability-setup.ts (99%) rename {scripts/setup => packages/sim-setup/src}/capability-status.test.ts (97%) rename {scripts/setup => packages/sim-setup/src}/capability-status.ts (99%) rename {scripts/setup => packages/sim-setup/src}/checks.ts (97%) rename {scripts/setup => packages/sim-setup/src}/cli-auth.ts (98%) create mode 100644 packages/sim-setup/src/compose-asset.test.ts create mode 100644 packages/sim-setup/src/compose-asset.ts rename {scripts/setup => packages/sim-setup/src}/configuration-sources.test.ts (99%) rename {scripts/setup => packages/sim-setup/src}/configuration-sources.ts (99%) create mode 100644 packages/sim-setup/src/context.test.ts create mode 100644 packages/sim-setup/src/context.ts rename {scripts/setup => packages/sim-setup/src}/db.ts (97%) rename {scripts/setup => packages/sim-setup/src}/detect.ts (95%) rename {scripts/setup => packages/sim-setup/src}/docker.ts (95%) rename {scripts/setup => packages/sim-setup/src}/doctor.ts (97%) rename {scripts/setup => packages/sim-setup/src}/env-files.test.ts (96%) rename {scripts/setup => packages/sim-setup/src}/env-files.ts (94%) rename {scripts/setup => packages/sim-setup/src}/errors.ts (100%) create mode 100644 packages/sim-setup/src/executables.ts rename {scripts/setup => packages/sim-setup/src}/feature-setup.test.ts (95%) rename {scripts/setup => packages/sim-setup/src}/feature-setup.ts (88%) create mode 100644 packages/sim-setup/src/index.ts rename {scripts/setup => packages/sim-setup/src}/lifecycle.test.ts (90%) rename {scripts/setup => packages/sim-setup/src}/lifecycle.ts (91%) rename {scripts/setup => packages/sim-setup/src}/modes/compose.ts (81%) rename {scripts/setup => packages/sim-setup/src}/modes/dev.ts (93%) rename {scripts/setup => packages/sim-setup/src}/modes/k8s.ts (97%) rename {scripts/setup => packages/sim-setup/src}/ports.ts (96%) rename {scripts/setup => packages/sim-setup/src}/probes.ts (93%) rename {scripts/setup => packages/sim-setup/src}/prompter.ts (60%) rename {scripts/setup => packages/sim-setup/src}/redis.ts (95%) rename {scripts/setup => packages/sim-setup/src}/setup-status.test.ts (97%) rename {scripts/setup => packages/sim-setup/src}/setup-status.ts (95%) rename {scripts/setup => packages/sim-setup/src}/steps.test.ts (97%) rename {scripts/setup => packages/sim-setup/src}/steps.ts (94%) rename {scripts/setup => packages/sim-setup/src}/terminal.ts (100%) rename {scripts/setup => packages/sim-setup/src}/theme.ts (100%) rename {scripts/setup => packages/sim-setup/src}/twins.ts (97%) rename {scripts/setup => packages/sim-setup/src}/urls.ts (100%) rename {scripts/setup => packages/sim-setup/src}/wizard.ts (86%) create mode 100644 packages/sim-setup/tsconfig.json create mode 100644 packages/sim-setup/vitest.config.ts create mode 100644 scripts/generate-deployment-config.ts delete mode 100755 scripts/setup/index.ts delete mode 100644 scripts/setup/launcher.test.ts delete mode 100755 scripts/setup/launcher.ts diff --git a/.agents/skills/add-block/SKILL.md b/.agents/skills/add-block/SKILL.md index 091a8307c31..05412f3f29a 100644 --- a/.agents/skills/add-block/SKILL.md +++ b/.agents/skills/add-block/SKILL.md @@ -172,8 +172,8 @@ Optional companions: `credentialLabels` (override the picker's section/connect-r ### OAuth deployment availability (required for integration blocks) A visible tools-category block with OAuth is deployment-gated. Its `oauth-input.serviceId` is -projected into `apps/sim/lib/integrations/integrations.json`, then resolved through -`resolveOAuthClientCapabilityId()` in `apps/sim/lib/core/config/env-capabilities.ts`. +projected into `packages/deployment-config/src/integrations.json`, then resolved through +`resolveOAuthClientCapabilityId()` in `packages/deployment-config/src/env-capabilities.ts`. When adding or changing an OAuth integration block: @@ -184,13 +184,14 @@ When adding or changing an OAuth integration block: 3. For a new capability, add its required client fields to `OAUTH_CLIENT_CAPABILITIES` and ensure every referenced field exists in the env schema in `apps/sim/lib/core/config/env.ts`. Then add the matching `text` or `secret` input modes to `OAUTH_CLIENT_SETUP_FIELDS` in - `scripts/setup/capability-config.ts`. The CLI catalog is exhaustively typed and checked against - the runtime field list; do not infer secrecy from the field name. -4. If the canonical OAuth service declares `serviceAccountProviderId`, keep - `SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID` in - `apps/sim/lib/integrations/service-account-metadata.ts` aligned. Set - `deploymentRequirement` only when the service-account path is preview-gated or depends on the - OAuth client fields; otherwise omit it. + `packages/sim-setup/src/capability-config.ts`. The CLI catalog is exhaustively typed and checked + against the runtime field list; do not infer secrecy from the field name. +4. If the canonical OAuth service declares `serviceAccountProviderId`, run + `bun run deployment-config:generate`; this regenerates the provider-ID facts in + `packages/deployment-config/src/service-account-providers.generated.ts`. Never hand-edit that + generated map. Add `deploymentRequirement` policy in + `packages/deployment-config/src/service-account-metadata.ts` only when the service-account path + is preview-gated or depends on the OAuth client fields; otherwise omit it. Missing capability metadata is a runtime configuration error, not a reason to make the integration silently available. @@ -992,16 +993,21 @@ After adding or changing one, run: ```bash bun run scripts/generate-docs.ts +bun run deployment-config:generate bun run integration-catalog:check +bun run deployment-config:check bun run docs:check ``` The catalog check independently derives deployment metadata from the executable block registry and -compares it with the committed `apps/sim/lib/integrations/integrations.json`. `docs:check` re-renders -every generated docs artifact in memory and fails on any committed file that differs — it runs in CI -via `check:audits`, so commit the full generator output. If the generator also trues up pages an -earlier PR left stale, commit that catch-up too; reverting it as "unrelated drift" makes `docs:check` -fail. +compares it with the committed `packages/deployment-config/src/integrations.json`. The deployment +config check verifies the generated service-account facts against the canonical OAuth registry and +catalog. `docs:check` re-renders every generated docs artifact in memory and fails on any committed +file that differs — it runs in CI via `check:audits`, so commit the full generator output. If the +generator also trues up pages an earlier PR left stale, commit that catch-up too; reverting it as +"unrelated drift" makes `docs:check` fail. Review the generated diff and keep only intentional +changes. + ## Checklist Before Finishing - [ ] `integrationType` is set to the correct `IntegrationType` enum value diff --git a/.agents/skills/add-integration/SKILL.md b/.agents/skills/add-integration/SKILL.md index 22ed2ef4195..3f3bbaccf9a 100644 --- a/.agents/skills/add-integration/SKILL.md +++ b/.agents/skills/add-integration/SKILL.md @@ -538,16 +538,18 @@ the OAuth service configuration, deployment availability, and the setup CLI. 1. Ensure the block has exactly one distinct OAuth `serviceId` and that it matches the canonical service entry in `apps/sim/lib/oauth/oauth.ts`. 2. Confirm `resolveOAuthClientCapabilityId(serviceId)` resolves to the intended provider entry in - `OAUTH_CLIENT_CAPABILITIES` in `apps/sim/lib/core/config/env-capabilities.ts`. Google and + `OAUTH_CLIENT_CAPABILITIES` in `packages/deployment-config/src/env-capabilities.ts`. Google and Microsoft service IDs deliberately share provider-level capabilities. 3. For a new OAuth provider, add the required client fields to `OAUTH_CLIENT_CAPABILITIES`, add every referenced field to the env schema in `apps/sim/lib/core/config/env.ts`, and add the matching `text` or `secret` entries to `OAUTH_CLIENT_SETUP_FIELDS` in - `scripts/setup/capability-config.ts`. Do not create integration-specific setup logic or infer - secret fields from naming; the CLI mapping is exhaustively checked against the runtime fields. -4. If the canonical OAuth service has `serviceAccountProviderId`, add the matching projection to - `SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID` in - `apps/sim/lib/integrations/service-account-metadata.ts`. Use: + `packages/sim-setup/src/capability-config.ts`. Do not create integration-specific setup logic or + infer secret fields from naming; the CLI mapping is exhaustively checked against the runtime + fields. +4. If the canonical OAuth service has `serviceAccountProviderId`, run + `bun run deployment-config:generate` to refresh + `packages/deployment-config/src/service-account-providers.generated.ts`; never hand-edit the + generated provider-ID map. In `packages/deployment-config/src/service-account-metadata.ts`, use: - no `deploymentRequirement` when the service-account path works independently of OAuth client fields; - `'oauth-client'` when it requires the same deployment OAuth client fields; - `'preview-gated'` when availability is controlled by the service-account preview block. @@ -560,15 +562,18 @@ a resolvable capability must fail validation. Run the documentation generator: ```bash bun run scripts/generate-docs.ts +bun run deployment-config:generate bun run integration-catalog:check +bun run deployment-config:check bun run docs:check ``` This creates `apps/docs/content/docs/en/integrations/{service}.mdx` — one page per service carrying the block's Actions and, if it has one, its Triggers section. Never hand-edit generated pages; the only editable region is the `{/* MANUAL-CONTENT */}` block (see `scripts/README.md`). -The same generator refreshes `apps/sim/lib/integrations/integrations.json`. The catalog check then -derives the deployment-relevant fields from the executable block registry and compares them with the -committed projection. Review the generated diff and keep only intentional changes. +The docs generator refreshes `packages/deployment-config/src/integrations.json`, and the deployment +config generator projects service-account provider IDs from that catalog plus the canonical OAuth +registry. The checks compare both committed projections with their sources. Review the generated +diff and keep only intentional changes. ## V2 Integration Pattern @@ -647,14 +652,16 @@ If creating V2 versions (API-aligned outputs): - [ ] Created `index.ts` barrel export - [ ] Registered all triggers in `triggers/registry.ts` -### Docs +### Docs and deployment metadata - [ ] Ran `bun run scripts/generate-docs.ts` +- [ ] Ran `bun run deployment-config:generate` for OAuth or service-account changes - [ ] Verified docs file created -- [ ] Reviewed and committed the generated `apps/sim/lib/integrations/integrations.json` change +- [ ] Reviewed and committed the generated `packages/deployment-config/src/integrations.json` change - [ ] `bun run integration-catalog:check` passes - [ ] `bun run docs:check` passes — CI fails on stale generated docs, so commit the full generator output, including catch-up regeneration for pages another PR left stale (never revert it as "unrelated drift") +- [ ] `bun run deployment-config:check` passes ### Final Validation (Required) - [ ] Read every tool file and cross-referenced inputs/outputs against the API docs @@ -1002,4 +1009,4 @@ requiredScopes: getScopesForService('{service}'), 11. **Never hardcode scopes** - Use `getScopesForService()` in blocks and `getCanonicalScopesForProvider()` in auth.ts 12. **Always add scope descriptions** - New scopes must have entries in `SCOPE_DESCRIPTIONS` within `lib/oauth/utils.ts` 13. **OAuth service IDs need deployment capabilities** - Every visible OAuth integration must resolve through `OAUTH_CLIENT_CAPABILITIES`; shared Google/Microsoft aliases map to their provider capability -14. **Keep runtime and presentation separate** - Runtime OAuth fields live in `env-capabilities.ts`; CLI input modes live in the exhaustively checked `scripts/setup/capability-config.ts` mapping +14. **Keep runtime and presentation separate** - Runtime OAuth fields live in `packages/deployment-config/src/env-capabilities.ts`; CLI input modes live in the exhaustively checked `packages/sim-setup/src/capability-config.ts` mapping diff --git a/.agents/skills/validate-integration/SKILL.md b/.agents/skills/validate-integration/SKILL.md index 68ed9ee7620..4ca991588cc 100644 --- a/.agents/skills/validate-integration/SKILL.md +++ b/.agents/skills/validate-integration/SKILL.md @@ -29,12 +29,13 @@ apps/sim/blocks/registry-maps.ts # Block + meta registry entry (BLOCK_REGISTR apps/sim/components/icons.tsx # Icon definition apps/sim/lib/auth/auth.ts # OAuth config — should use getCanonicalScopesForProvider() apps/sim/lib/oauth/oauth.ts # OAuth provider config — single source of truth for scopes -apps/sim/lib/oauth/utils.ts # Scope utilities, SCOPE_DESCRIPTIONS for modal UI -apps/sim/lib/core/config/env-capabilities.ts # OAuth client runtime capability source of truth +apps/sim/lib/oauth/utils.ts # Scope utilities, SCOPE_DESCRIPTIONS for modal UI +packages/deployment-config/src/env-capabilities.ts # OAuth client runtime capability source of truth apps/sim/lib/core/config/env.ts # Runtime env schema for capability fields -scripts/setup/capability-config.ts # Exhaustive CLI input-mode mapping for OAuth fields -apps/sim/lib/integrations/integrations.json # Generated client-safe integration catalog -apps/sim/lib/integrations/service-account-metadata.ts # Lightweight service-account projection +packages/sim-setup/src/capability-config.ts # Exhaustive CLI input-mode mapping for OAuth fields +packages/deployment-config/src/integrations.json # Generated client-safe integration catalog +packages/deployment-config/src/service-account-providers.generated.ts # Generated provider-ID facts +packages/deployment-config/src/service-account-metadata.ts # Handwritten deployment policy ``` ## Step 2: Pull API Documentation @@ -291,7 +292,7 @@ Scopes are centralized — the single source of truth is `OAUTH_PROVIDERS` in `l ## Step 6: Validate Deployment Availability (if OAuth service) The deployment UI and setup CLI do not infer OAuth client fields from scopes. They resolve the -block's generated `oauthServiceId` through the application-owned capability catalog. +block's generated `oauthServiceId` through the shared deployment capability catalog. - [ ] The visible integration block has exactly one distinct `oauth-input.serviceId` - [ ] `resolveOAuthClientCapabilityId(serviceId)` returns the intended provider capability @@ -299,9 +300,9 @@ block's generated `oauthServiceId` through the application-owned capability cata - [ ] Every field listed by that capability exists in `apps/sim/lib/core/config/env.ts` - [ ] Every capability field has the correct `text` or `secret` entry in `OAUTH_CLIENT_SETUP_FIELDS`; no CLI naming heuristic is required - [ ] Shared Google/Microsoft service IDs resolve to their provider capability rather than duplicate entries -- [ ] `bun run setup integration ` is the command emitted by availability; the CLI has only the exhaustive input-mode projection, not a second runtime provider definition +- [ ] `npx @sim/setup add integration ` is the command emitted by availability; the CLI has only the exhaustive input-mode projection, not a second runtime provider definition - [ ] If the canonical OAuth service declares `serviceAccountProviderId`, - `SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID[serviceId]` has the same provider ID + the generated `SERVICE_ACCOUNT_PROVIDER_BY_OAUTH_SERVICE_ID[serviceId]` has the same provider ID - [ ] The service-account `deploymentRequirement` matches how that credential actually works: omitted for an independent path, `'oauth-client'` when it needs the OAuth client fields, or `'preview-gated'` when controlled by a preview block @@ -386,13 +387,16 @@ Several files are generated from tool and block definitions. Editing a tool or b ```bash bun run tool-metadata:generate # repo root — apps/sim/tools/generated/* -bun run scripts/generate-docs.ts # docs .mdx + lib/integrations/integrations.json + docs icons +bun run scripts/generate-docs.ts # docs .mdx + deployment-config/integrations.json + docs icons +bun run deployment-config:generate # canonical OAuth registry + catalog → provider-ID facts bun run integration-catalog:check # registry ↔ committed deployment metadata drift bun run docs:check # committed docs ↔ what the generator renders today +bun run deployment-config:check # OAuth registry/catalog ↔ provider-ID fact drift ``` - **`tool-metadata:generate`** — required whenever a tool's `outputs`, `params`, or descriptions change. CI enforces this with `bun run tool-metadata:check`, which fails with *"Generated tool metadata is stale"*. This is the easiest gate to miss, because nothing in the tool file hints that a generated artifact mirrors it. -- **`generate-docs`** — required whenever block metadata changes (`bgColor`, `name`, `description`, operations, outputs). Regenerates the integration `.mdx`, `integrations.json`, and the docs copy of `components/icons.tsx`. +- **`generate-docs`** — required whenever block metadata changes (`bgColor`, `name`, `description`, operations, outputs). Regenerates the integration `.mdx`, `packages/deployment-config/src/integrations.json`, and the docs copy of `components/icons.tsx`. +- **`deployment-config:generate`** — required for OAuth or service-account changes. Regenerates provider-ID facts from the canonical OAuth registry and integration catalog; special deployment requirements remain handwritten policy. - **`integration-catalog:check`** — loads the executable block registry, derives visible integration deployment fields, and compares them with the committed catalog. It catches missing/unexpected entries and stale auth/service IDs without loading the executable registry in client code. @@ -419,9 +423,10 @@ After fixing, confirm: 4. Derived artifacts regenerated and their diffs reviewed (see above) 5. `bun run integration-catalog:check` passes 6. `bun run docs:check` passes -7. For OAuth or service-account changes, `bun test apps/sim/lib/integrations/availability.server.test.ts` passes -8. Re-read all modified files to verify fixes are correct -9. Any remaining unknown response schemas were explicitly reported to the user instead of guessed +7. For OAuth or service-account changes, `bun run deployment-config:check` passes +8. For OAuth or service-account changes, `bun run --cwd apps/sim test lib/integrations/availability.server.test.ts` passes +9. Re-read all modified files to verify fixes are correct +10. Any remaining unknown response schemas were explicitly reported to the user instead of guessed ## Checklist Summary @@ -437,7 +442,7 @@ After fixing, confirm: - [ ] Validated scope descriptions exist in `SCOPE_DESCRIPTIONS` within `lib/oauth/utils.ts` for all scopes - [ ] Validated OAuth `serviceId` resolves to the intended `OAUTH_CLIENT_CAPABILITIES` entry and all capability fields exist in the env schema - [ ] Validated service-account projection and deployment requirement against the canonical OAuth service config -- [ ] Regenerated `integrations.json` when block metadata changed and ran `bun run integration-catalog:check` +- [ ] Regenerated deployment config when block/OAuth metadata changed and ran both catalog checks - [ ] Validated pagination consistency across tools and block - [ ] Validated memory load safety using `.agents/skills/memory-load-check/SKILL.md` when tools list/search/download/import/export/batch data - [ ] Validated error handling (error checks, meaningful messages) diff --git a/.github/workflows/publish-sim-setup.yml b/.github/workflows/publish-sim-setup.yml new file mode 100644 index 00000000000..0d98c2cd5f4 --- /dev/null +++ b/.github/workflows/publish-sim-setup.yml @@ -0,0 +1,172 @@ +name: Publish Sim Setup Package + +on: + push: + branches: [main, staging, dev] + paths: + - 'packages/sim-setup/**' + - 'packages/deployment-config/**' + - 'packages/security/**' + - 'packages/utils/**' + - 'docker-compose.prod.yml' + - 'bun.lock' + +permissions: + contents: read + +concurrency: + group: publish-sim-setup-${{ github.ref }} + cancel-in-progress: true + +jobs: + publish-npm: + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} + timeout-minutes: 15 + steps: + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.14 + + - name: Setup Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version: '20' + + - name: Cache Bun dependencies + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: | + ~/.bun/install/cache + node_modules + **/node_modules + key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }} + restore-keys: | + ${{ runner.os }}-bun- + + - name: Install dependencies + run: bun install --frozen-lockfile --ignore-scripts + + - name: Verify npm authentication + env: + NPM_CONFIG_TOKEN: ${{ secrets.NPM_TOKEN }} + run: bun pm whoami + + - name: Check generated deployment config + run: bun run deployment-config:check + + - name: Run tests + working-directory: packages/sim-setup + run: bun run test + + - name: Type-check packages + run: | + bun run --cwd packages/deployment-config type-check + bun run --cwd packages/sim-setup type-check + + - name: Build package + working-directory: packages/sim-setup + run: bun run build + + - name: Resolve release channel + id: release + working-directory: packages/sim-setup + env: + BRANCH: ${{ github.ref_name }} + run: | + BASE_VERSION="$(bun -p "require('./package.json').version")" + if [[ ! "$BASE_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Package version must be a stable X.Y.Z base, got '$BASE_VERSION'." >&2 + exit 1 + fi + + case "$BRANCH" in + dev) + VERSION="${BASE_VERSION}-dev.${GITHUB_RUN_NUMBER}.${GITHUB_RUN_ATTEMPT}" + TAG="dev" + ;; + staging) + VERSION="${BASE_VERSION}-preview.${GITHUB_RUN_NUMBER}.${GITHUB_RUN_ATTEMPT}" + TAG="staging" + ;; + main) + VERSION="$BASE_VERSION" + TAG="latest" + ;; + *) + echo "Unsupported release branch '$BRANCH'." >&2 + exit 1 + ;; + esac + + bun pm pkg set "version=$VERSION" + RESOLVED_VERSION="$(bun -p "require('./package.json').version")" + if [ "$RESOLVED_VERSION" != "$VERSION" ]; then + echo "Version injection mismatch: wanted '$VERSION', got '$RESOLVED_VERSION'." >&2 + exit 1 + fi + + { + echo "version=$VERSION" + echo "tag=$TAG" + } >> "$GITHUB_OUTPUT" + + - name: Smoke-test packed Node bundle + working-directory: packages/sim-setup + run: | + set -euo pipefail + SMOKE_DIR="$(mktemp -d "$RUNNER_TEMP/sim-setup-smoke.XXXXXX")" + PACKAGE_PATH="$SMOKE_DIR/sim-setup.tgz" + bun pm pack --ignore-scripts --filename "$PACKAGE_PATH" --quiet + tar -xzf "$PACKAGE_PATH" -C "$SMOKE_DIR" + + if tar -tzf "$PACKAGE_PATH" | grep -Eq '^package/(apps|docker|helm|packages|scripts)/'; then + echo 'Packed setup package contains repository source directories.' >&2 + exit 1 + fi + test -x "$SMOKE_DIR/package/dist/index.js" + test -f "$SMOKE_DIR/package/dist/docker-compose.prod.yml" + if grep -Eq '^[[:space:]]+(build|context):' "$SMOKE_DIR/package/dist/docker-compose.prod.yml"; then + echo 'Production Compose asset contains a local build dependency.' >&2 + exit 1 + fi + + cd "$SMOKE_DIR" + node package/dist/index.js --version + node package/dist/index.js --help + + - name: Check if version already exists + id: version_check + working-directory: packages/sim-setup + env: + VERSION: ${{ steps.release.outputs.version }} + run: | + if bun pm view "@sim/setup@$VERSION" version > /dev/null 2>&1; then + echo "exists=true" >> "$GITHUB_OUTPUT" + else + echo "exists=false" >> "$GITHUB_OUTPUT" + fi + + - name: Publish to npm + if: steps.version_check.outputs.exists == 'false' + working-directory: packages/sim-setup + env: + NPM_CONFIG_TOKEN: ${{ secrets.NPM_TOKEN }} + NPM_TAG: ${{ steps.release.outputs.tag }} + run: bun publish --access public --tag "$NPM_TAG" --no-save + + - name: Summarize release + if: steps.version_check.outputs.exists == 'false' + env: + VERSION: ${{ steps.release.outputs.version }} + NPM_TAG: ${{ steps.release.outputs.tag }} + run: echo "Published @sim/setup@$VERSION with the '$NPM_TAG' tag." + + - name: Summarize skipped release + if: steps.version_check.outputs.exists == 'true' + env: + VERSION: ${{ steps.release.outputs.version }} + run: echo "Skipped @sim/setup@$VERSION because that version is already published." diff --git a/README.md b/README.md index 5967bfa7c12..0e506fb3128 100644 --- a/README.md +++ b/README.md @@ -27,9 +27,7 @@ ### Self-hosted ```bash -git clone https://github.com/simstudioai/sim.git && cd sim -bun install -bun run setup +npx @sim/setup ``` Open [http://localhost:3000](http://localhost:3000) @@ -72,56 +70,52 @@ Open [http://localhost:3000](http://localhost:3000) ## Self-hosting -**Requirements:** [Bun](https://bun.sh/) and [Docker](https://www.docker.com/). +**Requirements:** [Node.js 20+](https://nodejs.org/) and [Docker](https://www.docker.com/). -`bun run setup` is an interactive wizard: it provisions the database, generates secrets, writes your `.env` files, connects a Chat API key, and starts Sim the way you choose: - -- **Local dev** — run from source to contribute or hack on Sim -- **Docker Compose** — a self-contained instance for testing self-hosting -- **Kubernetes (Helm)** — deploy to a local cluster +`npx @sim/setup` is an interactive wizard that creates a small `sim/` deployment directory, provisions the database, generates secrets, writes `.env`, connects a Chat API key, and starts the published Sim images with Docker Compose. It does not clone the repository. When it finishes, open [http://localhost:3000](http://localhost:3000). +Run the same command inside a cloned Sim repository to unlock the source-only local development and Kubernetes modes. The existing `bun run setup` and `bun run sim` commands remain available to contributors. + Reconfigure an optional capability without rerunning the full wizard: ```bash -bun run setup status -bun run setup email -bun run setup storage -bun run setup sandbox -bun run setup jobs -bun run setup cache -bun run setup knowledge -bun run setup llm -bun run setup integration slack +npx @sim/setup config +npx @sim/setup add email +npx @sim/setup add storage +npx @sim/setup add sandbox +npx @sim/setup add jobs +npx @sim/setup add cache +npx @sim/setup add knowledge +npx @sim/setup add llm +npx @sim/setup add integration slack ``` -`bun run setup status` detects the effective local-dev, Docker Compose, or current-context +`npx @sim/setup config` detects the effective local-dev, Docker Compose, or current-context Helm configuration and reports configured, missing, or invalid capabilities and OAuth -integrations without printing credential values. This is separate from `bun run sim status`, +integrations without printing credential values. This is separate from `npx @sim/setup status`, which reports whether installed services are running and healthy. -Manage your install with `bun run sim`: +Manage your install from its directory: ```bash -bun run sim start | stop | restart # bring your install up / down / cycle -bun run sim update # pull/rebuild and apply Compose images -bun run sim status # what's installed and healthy -bun run sim logs # follow logs -bun run sim doctor # diagnose configuration problems -bun run sim down # remove containers (data kept) -bun run sim reset # archive .env and wipe managed data +npx @sim/setup start | stop | restart # bring your install up / down / cycle +npx @sim/setup update # pull and apply Compose images +npx @sim/setup status # what's installed and healthy +npx @sim/setup logs # follow logs +npx @sim/setup doctor # diagnose configuration problems +npx @sim/setup down # remove containers (data kept) +npx @sim/setup reset # archive .env and wipe managed data ``` -`sim` detects how you're running (Docker Compose, local dev, or Kubernetes) and acts accordingly. - -Prefer a bare `sim`? Run `bun link` once — but note `sim` lands in `~/.bun/bin`, which Homebrew's bun doesn't add to your PATH, so you may need `export PATH="$HOME/.bun/bin:$PATH"` in your shell profile. +The setup package detects how you're running and acts accordingly. Use `--dir ` to create or manage a deployment somewhere other than `./sim`. Sim also supports local models via [Ollama](https://ollama.ai) and [vLLM](https://docs.vllm.ai/). See the [self-hosting docs](https://docs.sim.ai/self-hosting/docker) for details. ## Chat API Keys -Chat is a Sim-managed service. `bun run setup` connects a Chat API key for you — sign in when it opens your browser and the key is stored automatically. To view, create, or revoke keys later, go to [sim.ai/selfhost/settings/chat-keys](https://sim.ai/selfhost/settings/chat-keys). +Chat is a Sim-managed service. `npx @sim/setup` connects a Chat API key for you — sign in when it opens your browser and the key is stored automatically. To view, create, or revoke keys later, go to [sim.ai/selfhost/settings/chat-keys](https://sim.ai/selfhost/settings/chat-keys). ## Environment Variables diff --git a/apps/docs/content/docs/en/platform/self-hosting/docker.mdx b/apps/docs/content/docs/en/platform/self-hosting/docker.mdx index 1b911191227..2e54fc2f609 100644 --- a/apps/docs/content/docs/en/platform/self-hosting/docker.mdx +++ b/apps/docs/content/docs/en/platform/self-hosting/docker.mdx @@ -10,20 +10,14 @@ import { FAQ } from '@/components/ui/faq' ## Quick Start ```bash -git clone https://github.com/simstudioai/sim.git && cd sim - -cat > .env << EOF -BETTER_AUTH_SECRET=$(openssl rand -hex 32) -ENCRYPTION_KEY=$(openssl rand -hex 32) -INTERNAL_API_SECRET=$(openssl rand -hex 32) -CRON_SECRET=$(openssl rand -hex 32) -EOF - -docker compose -f docker-compose.prod.yml up -d +npx @sim/setup ``` Open [http://localhost:3000](http://localhost:3000) +The setup package creates `./sim` with a generated `.env` and the production Compose file, then +starts published container images. Pass `--dir ` to choose another directory. + ## Production Setup ### 1. Configure Environment @@ -150,7 +144,7 @@ docker compose -f docker-compose.prod.yml logs migrations docker compose -f docker-compose.prod.yml logs -f cron # Upgrade: bump SIM_VERSION in .env when pinned, then -bun run sim update +npx @sim/setup update ``` ```bash -bun run sim update +npx @sim/setup update docker compose -f docker-compose.prod.yml logs migrations ``` -`bun run sim update` pulls the versions configured by `SIM_VERSION` (or `latest` when it is +`npx @sim/setup update` pulls the versions configured by `SIM_VERSION` (or `latest` when it is unset), recreates the changed services, and keeps data volumes. It is equivalent to running `docker compose pull` followed by `docker compose up -d`. diff --git a/apps/sim/app/(landing)/integrations/(shell)/[slug]/opengraph-image.tsx b/apps/sim/app/(landing)/integrations/(shell)/[slug]/opengraph-image.tsx index 9a7f50a2039..6d1b584db17 100644 --- a/apps/sim/app/(landing)/integrations/(shell)/[slug]/opengraph-image.tsx +++ b/apps/sim/app/(landing)/integrations/(shell)/[slug]/opengraph-image.tsx @@ -1,5 +1,5 @@ +import integrationsJson from '@sim/deployment-config/integrations.json' import { notFound } from 'next/navigation' -import integrationsJson from '@/lib/integrations/integrations.json' import type { AuthType, Integration } from '@/lib/integrations/types' import { createLandingOgImage } from '@/app/(landing)/og-utils' diff --git a/apps/sim/app/(landing)/integrations/(shell)/opengraph-image.tsx b/apps/sim/app/(landing)/integrations/(shell)/opengraph-image.tsx index 8d33c0043be..3695ffdf289 100644 --- a/apps/sim/app/(landing)/integrations/(shell)/opengraph-image.tsx +++ b/apps/sim/app/(landing)/integrations/(shell)/opengraph-image.tsx @@ -1,4 +1,4 @@ -import integrationsJson from '@/lib/integrations/integrations.json' +import integrationsJson from '@sim/deployment-config/integrations.json' import type { Integration } from '@/lib/integrations/types' import { createLandingOgImage } from '@/app/(landing)/og-utils' diff --git a/apps/sim/blocks/registry-maps.ts b/apps/sim/blocks/registry-maps.ts index 2f54c9453e2..6228fc79854 100644 --- a/apps/sim/blocks/registry-maps.ts +++ b/apps/sim/blocks/registry-maps.ts @@ -698,7 +698,7 @@ export const BLOCK_REGISTRY: Record = { * two stay in lockstep without a separate registry to maintain. * * `BlockMeta` exists only for catalog-visible integrations — every key here - * has a corresponding entry in `lib/integrations/integrations.json`. Blocks + * has a corresponding entry in `packages/deployment-config/src/integrations.json`. Blocks * absent from the catalog (core blocks like `agent`/`api`, superseded base * versions, and hidden tools) carry no meta because the only consumers are * integration surfaces: `getTemplatesForBlock` (the two integration detail diff --git a/apps/sim/lib/core/config/env-capabilities.server.test.ts b/apps/sim/lib/core/config/env-capabilities.server.test.ts index f4ad969bded..182ee4e8d8d 100644 --- a/apps/sim/lib/core/config/env-capabilities.server.test.ts +++ b/apps/sim/lib/core/config/env-capabilities.server.test.ts @@ -31,13 +31,13 @@ describe('server environment capabilities', () => { expect(inspectConfiguredOAuthClient('slack')).toEqual({ state: 'partial', missingFields: ['SLACK_CLIENT_SECRET'], - setupCommand: 'bun run setup integration slack', + setupCommand: 'npx @sim/setup add integration slack', }) }) it('fails fast when an OAuth client is absent', () => { expect(() => requireConfiguredOAuthClient('shopify')).toThrow( - 'OAuth client shopify is not configured. Run bun run setup integration shopify.' + 'OAuth client shopify is not configured. Run npx @sim/setup add integration shopify.' ) }) @@ -45,7 +45,7 @@ describe('server environment capabilities', () => { setEnv({ SLACK_CLIENT_ID: 'slack-client' }) expect(() => requireConfiguredOAuthClient('slack')).toThrow( - 'OAuth client slack is partially configured — missing SLACK_CLIENT_SECRET. Run bun run setup integration slack.' + 'OAuth client slack is partially configured — missing SLACK_CLIENT_SECRET. Run npx @sim/setup add integration slack.' ) }) diff --git a/apps/sim/lib/core/config/env-capabilities.server.ts b/apps/sim/lib/core/config/env-capabilities.server.ts index 9c80f33961b..698a44dca1b 100644 --- a/apps/sim/lib/core/config/env-capabilities.server.ts +++ b/apps/sim/lib/core/config/env-capabilities.server.ts @@ -3,6 +3,7 @@ * * @packageDocumentation */ + import { env } from '@/lib/core/config/env' import { ASYNC_JOBS_CAPABILITY, diff --git a/apps/sim/lib/core/config/env-capabilities.test.ts b/apps/sim/lib/core/config/env-capabilities.test.ts index 216dbbda791..2be795b62a1 100644 --- a/apps/sim/lib/core/config/env-capabilities.test.ts +++ b/apps/sim/lib/core/config/env-capabilities.test.ts @@ -1,3 +1,4 @@ +import integrationsJson from '@sim/deployment-config/integrations.json' import { describe, expect, it, vi } from 'vitest' import { ASYNC_JOBS_CAPABILITY, @@ -20,7 +21,6 @@ import { validateCapabilityFieldInput, wireFallback, } from '@/lib/core/config/env-capabilities' -import integrationsJson from '@/lib/integrations/integrations.json' import type { Integration } from '@/lib/integrations/types' import { getServiceConfigByServiceId } from '@/lib/oauth/utils' diff --git a/apps/sim/lib/core/config/env-capabilities.ts b/apps/sim/lib/core/config/env-capabilities.ts index 830838fecf3..4a36493790b 100644 --- a/apps/sim/lib/core/config/env-capabilities.ts +++ b/apps/sim/lib/core/config/env-capabilities.ts @@ -1,1493 +1,2 @@ -/** - * Canonical runtime deployment-capability definitions. Keep this module free of application - * runtime dependencies so setup and diagnostics can consume the rules the app enforces. - * - * @packageDocumentation - */ -import { - IMMUTABLE_DAYTONA_SNAPSHOT_REF_ERROR, - IMMUTABLE_E2B_TEMPLATE_REF_ERROR, - isImmutableDaytonaSnapshotRef, - isImmutableE2BTemplateRef, - isValidSandboxReleaseGeneration, - SANDBOX_RELEASE_GENERATION_ERROR, -} from '@sim/utils/sandbox-references' - -export type EnvCapabilityValue = string | number | boolean | null | undefined - -export const CORE_CONFIGURATION_KEYS = [ - 'DATABASE_URL', - 'BETTER_AUTH_SECRET', - 'BETTER_AUTH_URL', - 'NEXT_PUBLIC_APP_URL', - 'ENCRYPTION_KEY', - 'INTERNAL_API_SECRET', -] as const - -export type EnvCapabilityValues = - | ReadonlyMap - | Readonly> - -export type EnvValueValidation = - | { - kind: 'integer' - min?: number - max?: number - message: string - } - | { - kind: 'json-object' - requiredStringFields?: readonly string[] - message: string - } - | { - kind: 'pattern' - pattern: RegExp - message: string - } - | { - kind: 'immutable-e2b-template-ref' - message: string - } - | { - kind: 'immutable-daytona-snapshot-ref' - message: string - } - | { - kind: 'sandbox-release-generation' - message: string - } - | { - kind: 'url' - protocols?: readonly string[] - message: string - } - -export interface EnvFieldRequirement { - type: 'field' - key: string - validation?: EnvValueValidation -} - -export interface AllOfRequirement { - type: 'allOf' - requirements: readonly EnvRequirement[] -} - -export interface AnyOfRequirement { - type: 'anyOf' - requirements: readonly EnvRequirement[] -} - -export type EnvRequirement = EnvFieldRequirement | AllOfRequirement | AnyOfRequirement - -export type EnvProviderActivation = - | { mode: 'any-present'; keys: readonly string[] } - | { mode: 'enabled'; key: string } - -export interface EnvProviderValidationIssue { - kind: 'missing' | 'invalid' - fields: readonly string[] - message: string -} - -export interface EnvProviderDefinition { - id: TId - label: string - activation: EnvProviderActivation - requires: EnvRequirement - pairedFields?: readonly (readonly [string, string])[] - optionalFields?: readonly EnvFieldRequirement[] - validate?: (values: EnvCapabilityValues) => readonly EnvProviderValidationIssue[] -} - -export interface FallbackCapabilityDefinition< - TId extends string = string, - TProvider extends EnvProviderDefinition = EnvProviderDefinition, -> { - strategy: 'fallback' - id: TId - label: string - providers: readonly TProvider[] -} - -export type EnvDefaultProviderDefinition = - | { id: string; kind: 'built-in'; label: string } - | { id: string; kind: 'provider' } - -export interface SelectedCapabilityDefinition< - TId extends string = string, - TProvider extends EnvProviderDefinition = EnvProviderDefinition, -> { - strategy: 'selected' - id: TId - label: string - selectorKey?: string - whenUnset: 'default' | 'first-ready' - defaultProvider: EnvDefaultProviderDefinition - providers: readonly TProvider[] -} - -export type CapabilityDefinition = FallbackCapabilityDefinition | SelectedCapabilityDefinition - -export type DeclaredProviderId = - TDefinition['providers'][number]['id'] - -export type ProviderId = - TDefinition extends SelectedCapabilityDefinition - ? DeclaredProviderId | TDefinition['defaultProvider']['id'] - : DeclaredProviderId - -export type FallbackFactories = { - [TId in DeclaredProviderId]: () => TProvider | null -} - -export type ProviderConfigurationState = 'absent' | 'partial' | 'ready' | 'invalid' - -export interface ProviderInspection { - id: TId - label: string - active: boolean - state: ProviderConfigurationState - missingFields: readonly string[] - invalidFields: readonly string[] - invalidDetails: readonly string[] -} - -export interface FallbackCapabilityInspection { - strategy: 'fallback' - configured: boolean - providerIds: readonly TId[] - providers: readonly ProviderInspection[] - error: EnvCapabilityConfigurationError | null -} - -export interface SelectedCapabilityInspection< - TProviderId extends string = string, - TDeclaredProviderId extends string = TProviderId, -> { - strategy: 'selected' - providerId: TProviderId | null - providers: readonly ProviderInspection[] - error: EnvCapabilityConfigurationError | null -} - -export type CapabilityInspection = - TDefinition extends SelectedCapabilityDefinition - ? SelectedCapabilityInspection, DeclaredProviderId> - : FallbackCapabilityInspection> - -export class EnvCapabilityConfigurationError extends Error { - constructor( - readonly capabilityId: string, - message: string - ) { - super(message) - this.name = 'EnvCapabilityConfigurationError' - } -} - -function readValue(values: EnvCapabilityValues, key: string): EnvCapabilityValue { - if (values instanceof Map) return values.get(key) - return (values as Readonly>)[key] -} - -function hasValue(values: EnvCapabilityValues, key: string): boolean { - const value = readValue(values, key) - if (value === undefined || value === null || value === false) return false - if (typeof value !== 'string') return true - const normalized = value.trim().toLowerCase() - return normalized !== '' && normalized !== 'placeholder' -} - -function isTruthyValue(values: EnvCapabilityValues, key: string): boolean { - const value = readValue(values, key) - if (value === true || value === 1) return true - if (typeof value !== 'string') return false - const normalized = value.toLowerCase() - return normalized === 'true' || normalized === '1' -} - -/** Returns whether an environment field contains a usable configuration value. */ -export function hasEnvCapabilityValue(values: EnvCapabilityValues, key: string): boolean { - return hasValue(values, key) -} - -/** Resolves the boolean semantics shared by capability selectors and status reporting. */ -export function isTruthyEnvCapabilityValue(values: EnvCapabilityValues, key: string): boolean { - return isTruthyValue(values, key) -} - -function unique(values: readonly string[]): string[] { - return [...new Set(values)] -} - -export function envField( - key: string, - options: Pick = {} -): EnvFieldRequirement { - return { type: 'field', key, ...options } -} - -export function allOf(...requirements: readonly EnvRequirement[]): AllOfRequirement { - return { type: 'allOf', requirements } -} - -export function anyOf(...requirements: readonly EnvRequirement[]): AnyOfRequirement { - return { type: 'anyOf', requirements } -} - -function requirementKeys(requirement: EnvRequirement): string[] { - return requirement.type === 'field' - ? [requirement.key] - : requirement.requirements.flatMap(requirementKeys) -} - -function activationKeys(activation: EnvProviderActivation): readonly string[] { - return activation.mode === 'enabled' ? [activation.key] : activation.keys -} - -function providerKeys(provider: EnvProviderDefinition): string[] { - return [ - ...activationKeys(provider.activation), - ...requirementKeys(provider.requires), - ...(provider.pairedFields ?? []).flat(), - ...(provider.optionalFields ?? []).map((field) => field.key), - ] -} - -/** Returns every environment field that can affect one provider at runtime. */ -export function getProviderFields(provider: EnvProviderDefinition): readonly string[] { - return unique(providerKeys(provider)) -} - -function providerIsActive(provider: EnvProviderDefinition, values: EnvCapabilityValues): boolean { - return provider.activation.mode === 'enabled' - ? isTruthyValue(values, provider.activation.key) - : provider.activation.keys.some((key) => hasValue(values, key)) -} - -function capabilityKeys(definition: CapabilityDefinition): string[] { - return [ - ...(definition.strategy === 'selected' && definition.selectorKey - ? [definition.selectorKey] - : []), - ...definition.providers.flatMap(providerKeys), - ] -} - -/** Returns every environment field that can affect a capability at runtime. */ -export function getCapabilityFields(definition: CapabilityDefinition): readonly string[] { - return unique(capabilityKeys(definition)) -} - -function assertRequirementDefinition( - capabilityId: string, - providerId: string, - requirement: EnvRequirement -): void { - if (requirement.type === 'field') { - if (!requirement.key) { - throw new Error(`Capability ${capabilityId} provider ${providerId} has an empty field key`) - } - return - } - if (requirement.requirements.length === 0) { - throw new Error( - `Capability ${capabilityId} provider ${providerId} has an empty ${requirement.type}` - ) - } - for (const child of requirement.requirements) { - assertRequirementDefinition(capabilityId, providerId, child) - } -} - -function assertCapabilityDefinition(definition: CapabilityDefinition): void { - if (definition.providers.length === 0) { - throw new Error(`Capability ${definition.id} must declare at least one provider`) - } - - const providerIds = definition.providers.map((provider) => provider.id) - if (new Set(providerIds).size !== providerIds.length) { - throw new Error(`Capability ${definition.id} has duplicate provider ids`) - } - - if (definition.strategy === 'selected') { - const defaultIsDeclared = providerIds.includes(definition.defaultProvider.id) - if (definition.defaultProvider.kind === 'provider' && !defaultIsDeclared) { - throw new Error( - `Capability ${definition.id} default provider ${definition.defaultProvider.id} is not declared` - ) - } - if (definition.defaultProvider.kind === 'built-in' && defaultIsDeclared) { - throw new Error( - `Capability ${definition.id} built-in default ${definition.defaultProvider.id} also appears in providers` - ) - } - } - - for (const provider of definition.providers) { - if (provider.activation.mode === 'any-present' && provider.activation.keys.length === 0) { - throw new Error(`Capability ${definition.id} provider ${provider.id} has no activation keys`) - } - assertRequirementDefinition(definition.id, provider.id, provider.requires) - } -} - -export function defineCapability( - definition: TDefinition -): TDefinition { - assertCapabilityDefinition(definition) - return definition -} - -/** Returns the canonical command for configuring a runtime capability. */ -export function getCapabilitySetupCommand(definition: CapabilityDefinition): string { - return `bun run setup ${definition.id}` -} - -interface RequirementInspection { - ready: boolean - missingFields: readonly string[] - invalidFields: readonly string[] - invalidDetails: readonly string[] -} - -function isValidEnvCapabilityFieldValue( - validation: EnvValueValidation, - value: EnvCapabilityValue -): boolean { - const serialized = String(value) - if (validation.kind === 'integer') { - const number = Number(serialized) - return ( - Number.isInteger(number) && - (validation.min === undefined || number >= validation.min) && - (validation.max === undefined || number <= validation.max) - ) - } - if (validation.kind === 'json-object') { - try { - const parsed: unknown = JSON.parse(serialized) - if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) return false - return (validation.requiredStringFields ?? []).every( - (field) => - field in parsed && - typeof (parsed as Record)[field] === 'string' && - ((parsed as Record)[field] as string).length > 0 - ) - } catch { - return false - } - } - if (validation.kind === 'pattern') { - validation.pattern.lastIndex = 0 - return validation.pattern.test(serialized) - } - if (validation.kind === 'immutable-e2b-template-ref') { - return isImmutableE2BTemplateRef(serialized) - } - if (validation.kind === 'immutable-daytona-snapshot-ref') { - return isImmutableDaytonaSnapshotRef(serialized) - } - if (validation.kind === 'sandbox-release-generation') { - return isValidSandboxReleaseGeneration(serialized) - } - try { - const parsed = new URL(serialized) - return !validation.protocols || validation.protocols.includes(parsed.protocol) - } catch { - return false - } -} - -function inspectField( - requirement: EnvFieldRequirement, - values: EnvCapabilityValues -): RequirementInspection { - if (!hasValue(values, requirement.key)) { - return { - ready: false, - missingFields: [requirement.key], - invalidFields: [], - invalidDetails: [], - } - } - - const value = readValue(values, requirement.key) - const valid = requirement.validation - ? isValidEnvCapabilityFieldValue(requirement.validation, value) - : true - - return valid - ? { ready: true, missingFields: [], invalidFields: [], invalidDetails: [] } - : { - ready: false, - missingFields: [], - invalidFields: [requirement.key], - invalidDetails: [`${requirement.key} ${requirement.validation?.message ?? 'is invalid'}`], - } -} - -function inspectRequirement( - requirement: EnvRequirement, - values: EnvCapabilityValues -): RequirementInspection { - if (requirement.type === 'field') return inspectField(requirement, values) - - const inspections = requirement.requirements.map((child) => inspectRequirement(child, values)) - if (requirement.type === 'anyOf') { - const ready = inspections.find((inspection) => inspection.ready) - if (ready) return ready - return inspections.reduce((best, candidate) => { - const bestIssueCount = best.missingFields.length + best.invalidFields.length - const candidateIssueCount = candidate.missingFields.length + candidate.invalidFields.length - return candidateIssueCount < bestIssueCount ? candidate : best - }) - } - - return { - ready: inspections.every((inspection) => inspection.ready), - missingFields: unique(inspections.flatMap((inspection) => inspection.missingFields)), - invalidFields: unique(inspections.flatMap((inspection) => inspection.invalidFields)), - invalidDetails: unique(inspections.flatMap((inspection) => inspection.invalidDetails)), - } -} - -function inspectProviderRequirements( - provider: TProvider, - values: EnvCapabilityValues, - active = true -): ProviderInspection { - const inspection = inspectRequirement(provider.requires, values) - const invalidOptionalFields = (provider.optionalFields ?? []).flatMap((field) => { - if (!hasValue(values, field.key)) return [] - return inspectField(field, values).invalidFields - }) - const invalidPairs = (provider.pairedFields ?? []).flatMap(([left, right]) => - hasValue(values, left) === hasValue(values, right) ? [] : [left, right] - ) - const customIssues = provider.validate?.(values) ?? [] - const missingFields = unique([ - ...inspection.missingFields, - ...customIssues - .filter((customIssue) => customIssue.kind === 'missing') - .flatMap((customIssue) => customIssue.fields), - ]) - const invalidFields = unique([ - ...inspection.invalidFields, - ...invalidOptionalFields, - ...invalidPairs, - ...customIssues - .filter((customIssue) => customIssue.kind === 'invalid') - .flatMap((customIssue) => customIssue.fields), - ]) - const invalidDetails = unique([ - ...inspection.invalidDetails, - ...(provider.optionalFields ?? []).flatMap((field) => { - if (!hasValue(values, field.key)) return [] - return inspectField(field, values).invalidDetails - }), - ...(provider.pairedFields ?? []).flatMap(([left, right]) => - hasValue(values, left) === hasValue(values, right) - ? [] - : [`${left} and ${right} must be set together`] - ), - ...customIssues.map((customIssue) => customIssue.message), - ]) - return { - id: provider.id, - label: provider.label, - active, - state: - invalidFields.length > 0 - ? 'invalid' - : missingFields.length > 0 || !inspection.ready - ? 'partial' - : 'ready', - missingFields, - invalidFields, - invalidDetails, - } -} - -function inspectRequiredProvider( - provider: TProvider, - values: EnvCapabilityValues -): ProviderInspection { - const active = providerIsActive(provider, values) - const inspection = inspectProviderRequirements(provider, values, active) - if (active || provider.activation.mode !== 'enabled') return inspection - - return { - ...inspection, - state: 'invalid', - invalidFields: unique([...inspection.invalidFields, provider.activation.key]), - invalidDetails: unique([ - ...inspection.invalidDetails, - `${provider.activation.key} must be enabled`, - ]), - } -} - -export function inspectProvider( - provider: TProvider, - values: EnvCapabilityValues -): ProviderInspection { - const active = providerIsActive(provider, values) - return active - ? inspectProviderRequirements(provider, values, true) - : { - id: provider.id, - label: provider.label, - active: false, - state: 'absent', - missingFields: [], - invalidFields: [], - invalidDetails: [], - } -} - -function providerProblems(inspection: ProviderInspection): string { - return [ - inspection.missingFields.length > 0 ? `missing ${inspection.missingFields.join(', ')}` : null, - inspection.invalidDetails.length > 0 - ? inspection.invalidDetails.join(', ') - : inspection.invalidFields.length > 0 - ? `invalid ${inspection.invalidFields.join(', ')}` - : null, - ] - .filter(Boolean) - .join('; ') -} - -export function getCapabilityConfigurationError( - definition: CapabilityDefinition, - inspections: readonly ProviderInspection[] -): EnvCapabilityConfigurationError | null { - const broken = inspections.filter( - (inspection) => inspection.state === 'partial' || inspection.state === 'invalid' - ) - if (broken.length === 0) return null - - const details = broken.map((inspection) => `${inspection.label}: ${providerProblems(inspection)}`) - - return new EnvCapabilityConfigurationError( - definition.id, - `${definition.label} is partially or incorrectly configured (${details.join(' | ')}). Run ${getCapabilitySetupCommand(definition)}.` - ) -} - -function replaceProviderInspection( - providers: readonly ProviderInspection[], - replacement: ProviderInspection -): ProviderInspection[] { - return providers.map((provider) => (provider.id === replacement.id ? replacement : provider)) -} - -function inspectSelectedCapability( - definition: TDefinition, - values: EnvCapabilityValues -): SelectedCapabilityInspection, DeclaredProviderId> { - const rawSelector = definition.selectorKey ? readValue(values, definition.selectorKey) : undefined - const selector = - definition.selectorKey && hasValue(values, definition.selectorKey) - ? String(rawSelector).trim().toLowerCase() - : null - let providers = definition.providers.map((provider) => inspectProvider(provider, values)) - const known = new Set([ - definition.defaultProvider.id, - ...definition.providers.map((provider) => provider.id), - ]) - - if (selector && !known.has(selector)) { - const error = new EnvCapabilityConfigurationError( - definition.id, - `Unknown ${definition.selectorKey} "${rawSelector}". Expected one of: ${[...known].join(', ')}` - ) - return { strategy: 'selected', providerId: null, providers, error } - } - - if (selector) { - const selectedDefinition = definition.providers.find((provider) => provider.id === selector) - if (!selectedDefinition) { - return { - strategy: 'selected', - providerId: definition.defaultProvider.id as ProviderId, - providers, - error: null, - } - } - const active = providerIsActive(selectedDefinition, values) - const selected = - !active && selectedDefinition.activation.mode === 'enabled' - ? inspectProvider(selectedDefinition, values) - : inspectProviderRequirements(selectedDefinition, values, active) - providers = replaceProviderInspection(providers, selected) - const error = - selected.state === 'ready' || - (selected.state === 'absent' && selectedDefinition.activation.mode === 'enabled') - ? null - : new EnvCapabilityConfigurationError( - definition.id, - `${definition.label} selects ${selector}, but that provider is not configured (${providerProblems(selected)}). Run ${getCapabilitySetupCommand(definition)}.` - ) - return { - strategy: 'selected', - providerId: selector as ProviderId, - providers, - error, - } - } - - if (definition.whenUnset === 'default') { - const defaultDefinition = definition.providers.find( - (provider) => provider.id === definition.defaultProvider.id - ) - if (!defaultDefinition) { - return { - strategy: 'selected', - providerId: definition.defaultProvider.id as ProviderId, - providers, - error: null, - } - } - const selected = providers.find((provider) => provider.id === definition.defaultProvider.id) - return { - strategy: 'selected', - providerId: definition.defaultProvider.id as ProviderId, - providers, - error: - !selected || selected.state === 'ready' || selected.state === 'absent' - ? null - : new EnvCapabilityConfigurationError( - definition.id, - `${definition.label} selects ${definition.defaultProvider.id}, but that provider is not configured (${providerProblems(selected)}). Run ${getCapabilitySetupCommand(definition)}.` - ), - } - } - - const candidates = providers.filter((provider) => provider.id !== definition.defaultProvider.id) - for (const candidate of candidates) { - if (candidate.state === 'ready') { - return { - strategy: 'selected', - providerId: candidate.id as ProviderId, - providers, - error: null, - } - } - if ( - candidate.state === 'invalid' && - candidate.missingFields.length === 0 && - candidate.invalidFields.length > 0 - ) { - return { - strategy: 'selected', - providerId: candidate.id as ProviderId, - providers, - error: new EnvCapabilityConfigurationError( - definition.id, - `${candidate.label} is incorrectly configured (${providerProblems(candidate)}). Run ${getCapabilitySetupCommand(definition)}.` - ), - } - } - } - - const error = getCapabilityConfigurationError(definition, candidates) - const broken = candidates.find( - (provider) => provider.state === 'partial' || provider.state === 'invalid' - ) - - return { - strategy: 'selected', - providerId: (broken?.id ?? definition.defaultProvider.id) as ProviderId, - providers, - error, - } -} - -function inspectFallbackCapability( - definition: TDefinition, - values: EnvCapabilityValues -): FallbackCapabilityInspection> { - const providers = definition.providers.map((provider) => inspectProvider(provider, values)) - const providerIds = providers - .filter((provider) => provider.state === 'ready') - .map((provider) => provider.id) as DeclaredProviderId[] - const configurationError = getCapabilityConfigurationError(definition, providers) - - return { - strategy: 'fallback', - configured: providerIds.length > 0, - providerIds, - providers, - error: providerIds.length === 0 ? configurationError : null, - } -} - -export function inspectCapability( - definition: TDefinition, - values: EnvCapabilityValues -): SelectedCapabilityInspection, DeclaredProviderId> -export function inspectCapability( - definition: TDefinition, - values: EnvCapabilityValues -): FallbackCapabilityInspection> -export function inspectCapability( - definition: CapabilityDefinition, - values: EnvCapabilityValues -): SelectedCapabilityInspection | FallbackCapabilityInspection -export function inspectCapability( - definition: CapabilityDefinition, - values: EnvCapabilityValues -): SelectedCapabilityInspection | FallbackCapabilityInspection { - return definition.strategy === 'selected' - ? inspectSelectedCapability(definition, values) - : inspectFallbackCapability(definition, values) -} - -export function requireCapability( - definition: TDefinition, - values: EnvCapabilityValues -): Omit< - SelectedCapabilityInspection, DeclaredProviderId>, - 'error' | 'providerId' -> & { - providerId: ProviderId -} -export function requireCapability( - definition: TDefinition, - values: EnvCapabilityValues -): Omit>, 'error'> -export function requireCapability( - definition: CapabilityDefinition, - values: EnvCapabilityValues -): - | (Omit & { - providerId: string - }) - | Omit { - const inspection = - definition.strategy === 'selected' - ? inspectSelectedCapability(definition, values) - : inspectFallbackCapability(definition, values) - if (inspection.error) throw inspection.error - if (inspection.strategy === 'selected') { - const providerId = inspection.providerId - if (providerId === null) { - throw new EnvCapabilityConfigurationError( - definition.id, - `${definition.label} has no selected provider. Run ${getCapabilitySetupCommand(definition)}.` - ) - } - const selected = inspection.providers.find((provider) => provider.id === providerId) - if (selected && selected.state !== 'ready') { - const selectedDefinition = definition.providers.find((provider) => provider.id === providerId) - const strictInspection = - selected.state === 'absent' && selectedDefinition - ? inspectRequiredProvider(selectedDefinition, values) - : selected - throw new EnvCapabilityConfigurationError( - definition.id, - `${definition.label} selects ${providerId}, but that provider is not configured (${providerProblems(strictInspection)}). Run ${getCapabilitySetupCommand(definition)}.` - ) - } - return { - strategy: 'selected', - providerId, - providers: inspection.providers, - } - } - if (!inspection.configured) { - throw new EnvCapabilityConfigurationError( - definition.id, - `${definition.label} is not configured. Run ${getCapabilitySetupCommand(definition)}.` - ) - } - const { error: _, ...resolution } = inspection - return resolution -} - -function findFieldRequirement( - requirement: EnvRequirement, - key: string -): EnvFieldRequirement | null { - if (requirement.type === 'field') return requirement.key === key ? requirement : null - for (const child of requirement.requirements) { - const field = findFieldRequirement(child, key) - if (field) return field - } - return null -} - -/** Validates a candidate environment value with the runtime field rule. */ -export function validateCapabilityFieldInput( - definition: CapabilityDefinition, - key: string, - value: string -): string | undefined { - if (!value) return 'required' - for (const provider of definition.providers) { - const field = - findFieldRequirement(provider.requires, key) ?? - provider.optionalFields?.find((candidate) => candidate.key === key) - if (!field) continue - if (!field.validation || isValidEnvCapabilityFieldValue(field.validation, value)) { - return undefined - } - return field.validation.message - } - throw new Error(`${definition.label} has no validation definition for ${key}`) -} - -export interface WireFallbackOptions { - definition: TDefinition - values: EnvCapabilityValues - factories: FallbackFactories - shouldFallback?: (error: unknown, providerId: DeclaredProviderId) => boolean - onFailure?: (providerId: DeclaredProviderId, error: unknown) => void -} - -export function wireFallback({ - definition, - values, - factories, - shouldFallback, - onFailure, -}: WireFallbackOptions) { - const resolution = inspectCapability(definition, values) - if (resolution.error) throw resolution.error - const providers = resolution.providerIds.map((providerId) => { - const provider = factories[providerId]() - if (!provider) { - throw new EnvCapabilityConfigurationError( - definition.id, - `${definition.label} provider ${providerId} resolved as ready but its factory returned null` - ) - } - return { id: providerId, provider } - }) - - return { - configured: resolution.configured, - providerIds: resolution.providerIds, - providers: providers.map(({ provider }) => provider), - async execute( - operation: ( - provider: TProvider, - providerId: DeclaredProviderId - ) => Promise - ): Promise { - if (resolution.providerIds.length === 0) { - throw new EnvCapabilityConfigurationError( - definition.id, - `${definition.label} is not configured. Run ${getCapabilitySetupCommand(definition)}.` - ) - } - - const failures: unknown[] = [] - for (const { id: providerId, provider } of providers) { - try { - return await operation(provider, providerId) - } catch (error) { - if (shouldFallback && !shouldFallback(error, providerId)) throw error - failures.push(error) - onFailure?.(providerId, error) - } - } - - throw new AggregateError( - failures, - `All ${definition.label} providers failed: ${resolution.providerIds.join(', ')}` - ) - }, - } -} - -export const EMAIL_CAPABILITY = defineCapability({ - strategy: 'fallback', - id: 'email', - label: 'Email', - providers: [ - { - id: 'resend', - label: 'Resend', - activation: { mode: 'any-present', keys: ['RESEND_API_KEY'] }, - requires: envField('RESEND_API_KEY'), - }, - { - id: 'ses', - label: 'Amazon SES', - activation: { mode: 'any-present', keys: ['AWS_SES_REGION'] }, - requires: envField('AWS_SES_REGION'), - }, - { - id: 'smtp', - label: 'SMTP', - activation: { - mode: 'any-present', - keys: ['SMTP_HOST', 'SMTP_PORT', 'SMTP_USER', 'SMTP_PASS'], - }, - requires: allOf( - envField('SMTP_HOST'), - envField('SMTP_PORT', { - validation: { - kind: 'integer', - min: 1, - max: 65535, - message: 'must be a valid port between 1 and 65535', - }, - }) - ), - optionalFields: [envField('SMTP_USER'), envField('SMTP_PASS')], - }, - { - id: 'azure', - label: 'Azure Communication Services', - activation: { - mode: 'any-present', - keys: ['AZURE_ACS_CONNECTION_STRING'], - }, - requires: envField('AZURE_ACS_CONNECTION_STRING'), - }, - { - id: 'gmail', - label: 'Gmail', - activation: { - mode: 'any-present', - keys: ['GMAIL_CREDENTIALS_JSON', 'GMAIL_SENDER'], - }, - requires: allOf( - envField('GMAIL_CREDENTIALS_JSON', { - validation: { - kind: 'json-object', - requiredStringFields: ['client_email', 'private_key'], - message: 'must be service account JSON with client_email and private_key', - }, - }), - envField('GMAIL_SENDER') - ), - }, - ], -} as const) - -export const STORAGE_CAPABILITY = defineCapability({ - strategy: 'selected', - id: 'storage', - label: 'File storage', - selectorKey: 'STORAGE_PROVIDER', - whenUnset: 'first-ready', - defaultProvider: { id: 'local', kind: 'built-in', label: 'Local disk' }, - providers: [ - { - id: 'azure', - label: 'Azure Blob Storage', - activation: { - mode: 'any-present', - keys: [ - 'AZURE_CONNECTION_STRING', - 'AZURE_ACCOUNT_NAME', - 'AZURE_ACCOUNT_KEY', - 'AZURE_STORAGE_CONTAINER_NAME', - ], - }, - requires: allOf( - envField('AZURE_STORAGE_CONTAINER_NAME'), - anyOf( - envField('AZURE_CONNECTION_STRING'), - allOf(envField('AZURE_ACCOUNT_NAME'), envField('AZURE_ACCOUNT_KEY')) - ) - ), - }, - { - id: 's3', - label: 'S3', - activation: { - mode: 'any-present', - keys: [ - 'S3_BUCKET_NAME', - 'S3_KB_BUCKET_NAME', - 'S3_EXECUTION_FILES_BUCKET_NAME', - 'S3_CHAT_BUCKET_NAME', - 'S3_COPILOT_BUCKET_NAME', - 'S3_PROFILE_PICTURES_BUCKET_NAME', - 'S3_OG_IMAGES_BUCKET_NAME', - 'S3_WORKSPACE_LOGOS_BUCKET_NAME', - 'S3_ENDPOINT', - ], - }, - requires: allOf(envField('AWS_REGION'), envField('S3_BUCKET_NAME')), - pairedFields: [['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']], - optionalFields: [ - envField('S3_ENDPOINT', { - validation: { - kind: 'url', - protocols: ['http:', 'https:'], - message: 'must be a valid http:// or https:// URL', - }, - }), - envField('S3_FORCE_PATH_STYLE'), - ], - }, - { - id: 'gcs', - label: 'Google Cloud Storage', - activation: { mode: 'any-present', keys: ['GCS_BUCKET_NAME'] }, - requires: envField('GCS_BUCKET_NAME'), - optionalFields: [ - envField('GCS_CREDENTIALS_JSON', { - validation: { - kind: 'json-object', - requiredStringFields: ['client_email', 'private_key'], - message: 'must be service account JSON with client_email and private_key', - }, - }), - envField('GCS_PROJECT_ID'), - ], - }, - ], -} as const) - -export const SANDBOX_CAPABILITY = defineCapability({ - strategy: 'selected', - id: 'sandbox', - label: 'Remote sandbox', - selectorKey: 'SANDBOX_PROVIDER', - whenUnset: 'default', - defaultProvider: { id: 'e2b', kind: 'provider' }, - providers: [ - { - id: 'e2b', - label: 'E2B', - activation: { mode: 'enabled', key: 'E2B_ENABLED' }, - requires: allOf( - envField('E2B_API_KEY'), - envField('E2B_FUNCTION_TEMPLATE_ID', { - validation: { - kind: 'immutable-e2b-template-ref', - message: IMMUTABLE_E2B_TEMPLATE_REF_ERROR, - }, - }), - envField('E2B_FUNCTION_TEMPLATE_GENERATION', { - validation: { - kind: 'sandbox-release-generation', - message: SANDBOX_RELEASE_GENERATION_ERROR, - }, - }) - ), - optionalFields: [ - envField('NEXT_PUBLIC_E2B_ENABLED'), - envField('NEXT_PUBLIC_SANDBOXES_ENABLED'), - ], - }, - { - id: 'daytona', - label: 'Daytona', - activation: { - mode: 'any-present', - keys: ['DAYTONA_API_KEY', 'DAYTONA_FUNCTION_SNAPSHOT_ID'], - }, - requires: allOf( - envField('DAYTONA_API_KEY'), - envField('DAYTONA_FUNCTION_SNAPSHOT_ID', { - validation: { - kind: 'immutable-daytona-snapshot-ref', - message: IMMUTABLE_DAYTONA_SNAPSHOT_REF_ERROR, - }, - }) - ), - optionalFields: [ - envField('NEXT_PUBLIC_E2B_ENABLED'), - envField('NEXT_PUBLIC_SANDBOXES_ENABLED'), - ], - }, - ], -} as const) - -export const ASYNC_JOBS_CAPABILITY = defineCapability({ - strategy: 'selected', - id: 'jobs', - label: 'Async jobs', - whenUnset: 'first-ready', - defaultProvider: { - id: 'database', - kind: 'built-in', - label: 'Database queue', - }, - providers: [ - { - id: 'trigger-dev', - label: 'Trigger.dev', - activation: { mode: 'enabled', key: 'TRIGGER_DEV_ENABLED' }, - requires: allOf(envField('TRIGGER_PROJECT_ID'), envField('TRIGGER_SECRET_KEY')), - }, - ], -} as const) - -/** Validates the one provider dependency that cannot be expressed as a field-shape rule. */ -function validateRedisProvider(values: EnvCapabilityValues): readonly EnvProviderValidationIssue[] { - if (!hasValue(values, 'REDIS_URL')) return [] - let redisUrl: URL - try { - redisUrl = new URL(String(readValue(values, 'REDIS_URL'))) - } catch { - return [] - } - if ( - redisUrl.protocol === 'rediss:' && - /^\d+\.\d+\.\d+\.\d+$/.test(redisUrl.hostname) && - !hasValue(values, 'REDIS_TLS_SERVERNAME') - ) { - return [ - { - kind: 'missing', - fields: ['REDIS_TLS_SERVERNAME'], - message: 'REDIS_TLS_SERVERNAME is required for rediss:// IP addresses', - }, - ] - } - return [] -} - -export const CACHE_CAPABILITY = defineCapability({ - strategy: 'selected', - id: 'cache', - label: 'Cache', - whenUnset: 'first-ready', - defaultProvider: { id: 'database', kind: 'built-in', label: 'Postgres' }, - providers: [ - { - id: 'redis', - label: 'Redis', - activation: { mode: 'any-present', keys: ['REDIS_URL'] }, - requires: envField('REDIS_URL', { - validation: { - kind: 'url', - protocols: ['redis:', 'rediss:'], - message: 'must be a valid redis:// or rediss:// URL', - }, - }), - optionalFields: [envField('REDIS_TLS_SERVERNAME')], - validate: validateRedisProvider, - }, - ], -} as const) - -export const OCR_CAPABILITY = defineCapability({ - strategy: 'selected', - id: 'knowledge', - label: 'PDF OCR', - selectorKey: 'OCR_PROVIDER', - whenUnset: 'first-ready', - defaultProvider: { id: 'local', kind: 'built-in', label: 'Local parser' }, - providers: [ - { - id: 'azure-mistral', - label: 'Azure Mistral OCR', - activation: { - mode: 'any-present', - keys: ['OCR_AZURE_API_KEY', 'OCR_AZURE_ENDPOINT', 'OCR_AZURE_MODEL_NAME'], - }, - requires: allOf( - envField('OCR_AZURE_API_KEY'), - envField('OCR_AZURE_ENDPOINT', { - validation: { - kind: 'url', - protocols: ['http:', 'https:'], - message: 'must be a valid HTTP(S) URL', - }, - }), - envField('OCR_AZURE_MODEL_NAME') - ), - }, - { - id: 'mistral', - label: 'Mistral OCR', - activation: { mode: 'any-present', keys: ['MISTRAL_API_KEY'] }, - requires: envField('MISTRAL_API_KEY'), - }, - ], -} as const) - -export const KNOWLEDGE_EMBEDDINGS_CAPABILITY = defineCapability({ - strategy: 'fallback', - id: 'knowledge-embeddings', - label: 'Knowledge embeddings', - providers: [ - { - id: 'azure-openai', - label: 'Azure OpenAI', - activation: { - mode: 'any-present', - keys: ['AZURE_OPENAI_API_KEY', 'AZURE_OPENAI_ENDPOINT', 'AZURE_OPENAI_API_VERSION'], - }, - requires: allOf( - envField('AZURE_OPENAI_API_KEY'), - envField('AZURE_OPENAI_ENDPOINT', { - validation: { - kind: 'url', - protocols: ['http:', 'https:'], - message: 'must be a valid HTTP(S) URL', - }, - }), - envField('AZURE_OPENAI_API_VERSION') - ), - optionalFields: [envField('KB_OPENAI_MODEL_NAME')], - }, - { - id: 'openai', - label: 'OpenAI', - activation: { - mode: 'any-present', - keys: ['OPENAI_API_KEY', 'OPENAI_API_KEY_1', 'OPENAI_API_KEY_2', 'OPENAI_API_KEY_3'], - }, - requires: anyOf( - envField('OPENAI_API_KEY'), - envField('OPENAI_API_KEY_1'), - envField('OPENAI_API_KEY_2'), - envField('OPENAI_API_KEY_3') - ), - }, - { - id: 'openrouter', - label: 'OpenRouter', - activation: { mode: 'any-present', keys: ['OPENROUTER_API_KEY'] }, - requires: envField('OPENROUTER_API_KEY'), - }, - ], -} as const) - -export const OAUTH_CLIENT_CAPABILITIES = { - google: ['GOOGLE_CLIENT_ID', 'GOOGLE_CLIENT_SECRET'], - x: ['X_CLIENT_ID', 'X_CLIENT_SECRET'], - tiktok: ['TIKTOK_CLIENT_ID', 'TIKTOK_CLIENT_SECRET'], - confluence: ['CONFLUENCE_CLIENT_ID', 'CONFLUENCE_CLIENT_SECRET'], - jira: ['JIRA_CLIENT_ID', 'JIRA_CLIENT_SECRET'], - calcom: ['CALCOM_CLIENT_ID'], - airtable: ['AIRTABLE_CLIENT_ID', 'AIRTABLE_CLIENT_SECRET'], - notion: ['NOTION_CLIENT_ID', 'NOTION_CLIENT_SECRET'], - microsoft: ['MICROSOFT_CLIENT_ID', 'MICROSOFT_CLIENT_SECRET'], - clickup: ['CLICKUP_CLIENT_ID', 'CLICKUP_CLIENT_SECRET'], - linear: ['LINEAR_CLIENT_ID', 'LINEAR_CLIENT_SECRET'], - attio: ['ATTIO_CLIENT_ID', 'ATTIO_CLIENT_SECRET'], - box: ['BOX_CLIENT_ID', 'BOX_CLIENT_SECRET'], - docusign: ['DOCUSIGN_CLIENT_ID', 'DOCUSIGN_CLIENT_SECRET'], - dropbox: ['DROPBOX_CLIENT_ID', 'DROPBOX_CLIENT_SECRET'], - slack: ['SLACK_CLIENT_ID', 'SLACK_CLIENT_SECRET'], - reddit: ['REDDIT_CLIENT_ID', 'REDDIT_CLIENT_SECRET'], - wealthbox: ['WEALTHBOX_CLIENT_ID', 'WEALTHBOX_CLIENT_SECRET'], - webflow: ['WEBFLOW_CLIENT_ID', 'WEBFLOW_CLIENT_SECRET'], - asana: ['ASANA_CLIENT_ID', 'ASANA_CLIENT_SECRET'], - pipedrive: ['PIPEDRIVE_CLIENT_ID', 'PIPEDRIVE_CLIENT_SECRET'], - hubspot: ['HUBSPOT_CLIENT_ID', 'HUBSPOT_CLIENT_SECRET'], - linkedin: ['LINKEDIN_CLIENT_ID', 'LINKEDIN_CLIENT_SECRET'], - instagram: ['INSTAGRAM_CLIENT_ID', 'INSTAGRAM_CLIENT_SECRET'], - salesforce: ['SALESFORCE_CLIENT_ID', 'SALESFORCE_CLIENT_SECRET'], - shopify: ['SHOPIFY_CLIENT_ID', 'SHOPIFY_CLIENT_SECRET'], - zoom: ['ZOOM_CLIENT_ID', 'ZOOM_CLIENT_SECRET'], - wordpress: ['WORDPRESS_CLIENT_ID', 'WORDPRESS_CLIENT_SECRET'], - spotify: ['SPOTIFY_CLIENT_ID', 'SPOTIFY_CLIENT_SECRET'], - monday: ['MONDAY_CLIENT_ID', 'MONDAY_CLIENT_SECRET'], - trello: ['TRELLO_API_KEY'], - 'zoho-desk': ['ZOHO_CLIENT_ID', 'ZOHO_CLIENT_SECRET'], -} as const - -/** Single registry consumed by runtime status and environment-source detection. */ -export const ENV_CAPABILITIES = [ - EMAIL_CAPABILITY, - STORAGE_CAPABILITY, - SANDBOX_CAPABILITY, - ASYNC_JOBS_CAPABILITY, - CACHE_CAPABILITY, - OCR_CAPABILITY, - KNOWLEDGE_EMBEDDINGS_CAPABILITY, -] as const - -export const LLM_KEY_POOLS = { - openai: { - keys: ['OPENAI_API_KEY_1', 'OPENAI_API_KEY_2', 'OPENAI_API_KEY_3'], - fallbackKey: 'OPENAI_API_KEY', - }, - anthropic: { - keys: ['ANTHROPIC_API_KEY_1', 'ANTHROPIC_API_KEY_2', 'ANTHROPIC_API_KEY_3'], - }, - gemini: { - keys: ['GEMINI_API_KEY_1', 'GEMINI_API_KEY_2', 'GEMINI_API_KEY_3'], - fallbackKey: 'GEMINI_API_KEY', - }, - cohere: { - keys: ['COHERE_API_KEY_1', 'COHERE_API_KEY_2', 'COHERE_API_KEY_3'], - fallbackKey: 'COHERE_API_KEY', - }, - zai: { keys: ['ZAI_API_KEY_1', 'ZAI_API_KEY_2', 'ZAI_API_KEY_3'] }, - xai: { keys: ['XAI_API_KEY_1', 'XAI_API_KEY_2', 'XAI_API_KEY_3'] }, - kimi: { keys: ['KIMI_API_KEY_1', 'KIMI_API_KEY_2', 'KIMI_API_KEY_3'] }, - fireworks: { - keys: ['FIREWORKS_API_KEY_1', 'FIREWORKS_API_KEY_2', 'FIREWORKS_API_KEY_3'], - fallbackKey: 'FIREWORKS_API_KEY', - }, -} as const - -/** - * Environment keys whose process-level values can change setup status or make a - * setup write ineffective. The setup CLI uses this exact runtime-owned list to - * avoid claiming it manages a development configuration shadowed by the shell. - */ -export const DEPLOYMENT_CONFIGURATION_KEYS: readonly string[] = [ - ...new Set([ - ...CORE_CONFIGURATION_KEYS, - ...ENV_CAPABILITIES.flatMap(capabilityKeys), - 'EMAIL_VERIFICATION_ENABLED', - 'NEXT_PUBLIC_E2B_ENABLED', - 'NEXT_PUBLIC_SANDBOXES_ENABLED', - ...Object.values(LLM_KEY_POOLS).flatMap((pool) => [ - ...pool.keys, - ...('fallbackKey' in pool ? [pool.fallbackKey] : []), - ]), - ...Object.values(OAUTH_CLIENT_CAPABILITIES).flat(), - ]), -] - -export type OAuthClientCapabilityId = keyof typeof OAUTH_CLIENT_CAPABILITIES -export type OAuthClientCapabilityField = - (typeof OAUTH_CLIENT_CAPABILITIES)[TCapabilityId][number] - -export interface ConfiguredOAuthClient { - state: 'ready' - missingFields: readonly [] - setupCommand: string - values: Readonly> -} - -const GOOGLE_OAUTH_SERVICES = new Set([ - 'gmail', - 'google-email', - 'google-drive', - 'google-docs', - 'google-sheets', - 'google-calendar', - 'google-contacts', - 'google-ads', - 'google-bigquery', - 'google-tasks', - 'google-vault', - 'google-forms', - 'google-groups', - 'google-meet', - 'vertex-ai', -]) - -const MICROSOFT_OAUTH_SERVICES = new Set([ - 'microsoft', - 'outlook', - 'onedrive', - 'sharepoint', - 'microsoft-ad', - 'microsoft-dataverse', - 'microsoft-excel', - 'microsoft-teams', - 'microsoft-planner', -]) - -export function resolveOAuthClientCapabilityId(serviceId: string): OAuthClientCapabilityId | null { - const normalized = serviceId.toLowerCase().replace(/_/g, '-') - if (GOOGLE_OAUTH_SERVICES.has(normalized)) return 'google' - if (MICROSOFT_OAUTH_SERVICES.has(normalized)) return 'microsoft' - if (normalized === 'zoho') return 'zoho-desk' - // One consumer key serves both Salesforce login hosts, so the sandbox provider - // is configured by the same env pair — without this alias it is silently dropped. - if (normalized === 'salesforce-sandbox') return 'salesforce' - return normalized in OAUTH_CLIENT_CAPABILITIES ? (normalized as OAuthClientCapabilityId) : null -} - -export function getOAuthClientCapabilityFields(serviceId: string): readonly string[] | null { - const providerId = resolveOAuthClientCapabilityId(serviceId) - return providerId ? OAUTH_CLIENT_CAPABILITIES[providerId] : null -} - -export interface OAuthClientCapabilityInspection { - state: ProviderConfigurationState - missingFields: readonly string[] - setupCommand: string -} - -function readOAuthClientFieldValue(values: EnvCapabilityValues, key: string): string | null { - const value = readValue(values, key) - if (typeof value !== 'string' || !hasValue(values, key)) return null - return value -} - -export function inspectOAuthClientCapability( - providerId: string, - values: EnvCapabilityValues -): OAuthClientCapabilityInspection { - const capabilityId = resolveOAuthClientCapabilityId(providerId) - const fields = capabilityId ? OAUTH_CLIENT_CAPABILITIES[capabilityId] : null - if (!fields) { - return { - state: 'absent', - missingFields: [], - setupCommand: `bun run setup integration ${providerId}`, - } - } - - const present = fields.filter((key) => readOAuthClientFieldValue(values, key) !== null) - return { - state: present.length === 0 ? 'absent' : present.length === fields.length ? 'ready' : 'partial', - missingFields: fields.filter((key) => readOAuthClientFieldValue(values, key) === null), - setupCommand: `bun run setup integration ${providerId}`, - } -} - -export function requireOAuthClientCapability( - providerId: TCapabilityId, - values: EnvCapabilityValues -): ConfiguredOAuthClient> -export function requireOAuthClientCapability( - providerId: string, - values: EnvCapabilityValues -): ConfiguredOAuthClient -export function requireOAuthClientCapability( - providerId: string, - values: EnvCapabilityValues -): ConfiguredOAuthClient { - const inspection = inspectOAuthClientCapability(providerId, values) - if (inspection.state !== 'ready') { - const detail = - inspection.state === 'partial' || inspection.state === 'invalid' - ? ` is partially configured — missing ${inspection.missingFields.join(', ')}` - : ' is not configured' - throw new EnvCapabilityConfigurationError( - 'oauth', - `OAuth client ${providerId}${detail}. Run ${inspection.setupCommand}.` - ) - } - - const fields = getOAuthClientCapabilityFields(providerId) - if (!fields) { - throw new EnvCapabilityConfigurationError( - 'oauth', - `OAuth client ${providerId} has no capability definition. Run ${inspection.setupCommand}.` - ) - } - - const configuredValues: Record = {} - for (const field of fields) { - const value = readOAuthClientFieldValue(values, field) - if (value === null) { - throw new EnvCapabilityConfigurationError( - 'oauth', - `OAuth client ${providerId} has an invalid ${field}. Run ${inspection.setupCommand}.` - ) - } - configuredValues[field] = value - } - - return { - ...inspection, - state: 'ready', - missingFields: [], - values: configuredValues, - } -} +/** Application compatibility surface for shared deployment capability policy. */ +export * from '@sim/deployment-config/env-capabilities' diff --git a/apps/sim/lib/core/config/env-flags.ts b/apps/sim/lib/core/config/env-flags.ts index c479ce4b926..492158254a9 100644 --- a/apps/sim/lib/core/config/env-flags.ts +++ b/apps/sim/lib/core/config/env-flags.ts @@ -1,6 +1,6 @@ /** * Loaded by `next.config.ts` before the `@/` alias is available, so - * config-boundary dependencies in this module must use relative imports. + * config-boundary dependencies use workspace packages or relative imports. */ import { @@ -411,7 +411,7 @@ const sandboxProvider = inspectCapability(SANDBOX_CAPABILITY, env).providerId * * The browser cannot inspect provider credentials, so * `NEXT_PUBLIC_SANDBOXES_ENABLED` is its readiness projection. Set the public - * value only after this server-side check succeeds; `bun run setup --doctor` + * value only after this server-side check succeeds; `npx @sim/setup doctor` * reports mismatches in either direction. */ export const isRemoteSandboxEnabled = diff --git a/apps/sim/lib/integrations/availability.server.test.ts b/apps/sim/lib/integrations/availability.server.test.ts index d3a5a6eff19..18b4f97e5f1 100644 --- a/apps/sim/lib/integrations/availability.server.test.ts +++ b/apps/sim/lib/integrations/availability.server.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it, vi } from 'vitest' vi.mock('@/lib/core/config/env', () => ({ env: {} })) +import integrationsJson from '@sim/deployment-config/integrations.json' import { OAUTH_CLIENT_CAPABILITIES, resolveOAuthClientCapabilityId, @@ -20,7 +21,6 @@ import { isIntegrationDeploymentAvailable, isIntegrationDeploymentAvailableForVisibility, } from '@/lib/integrations/availability.server' -import integrationsJson from '@/lib/integrations/integrations.json' import { SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID } from '@/lib/integrations/service-account-metadata' import type { Integration } from '@/lib/integrations/types' import { getServiceConfigByServiceId } from '@/lib/oauth/utils' @@ -50,7 +50,7 @@ describe('integration availability', () => { oauthAvailable: true, serviceAccountAvailable: false, missingFields: [], - setupCommand: 'bun run setup integration slack', + setupCommand: 'npx @sim/setup add integration slack', }) }) @@ -60,7 +60,7 @@ describe('integration availability', () => { oauthAvailable: false, serviceAccountAvailable: true, missingFields: ['NOTION_CLIENT_ID', 'NOTION_CLIENT_SECRET'], - setupCommand: 'bun run setup integration notion', + setupCommand: 'npx @sim/setup add integration notion', }) }) @@ -77,7 +77,7 @@ describe('integration availability', () => { expect(availabilityFor('x')).toMatchObject({ state: 'unavailable', oauthAvailable: false, - setupCommand: 'bun run setup integration x', + setupCommand: 'npx @sim/setup add integration x', }) }) @@ -87,7 +87,7 @@ describe('integration availability', () => { oauthAvailable: false, serviceAccountAvailable: false, missingFields: ['SLACK_CLIENT_SECRET'], - setupCommand: 'bun run setup integration slack', + setupCommand: 'npx @sim/setup add integration slack', }) }) @@ -158,7 +158,7 @@ describe('integration availability', () => { state: 'unavailable', serviceAccountAvailable: false, missingFields: ['TRELLO_API_KEY'], - setupCommand: 'bun run setup integration trello', + setupCommand: 'npx @sim/setup add integration trello', }) expect(availabilityFor('trello', { TRELLO_API_KEY: 'trello-key' })).toMatchObject({ state: 'ready', @@ -181,7 +181,7 @@ describe('integration availability', () => { for (const integration of availability) { if (!integration.setupCommand) continue - const capabilityId = integration.setupCommand.replace('bun run setup integration ', '') + const capabilityId = integration.setupCommand.replace('npx @sim/setup add integration ', '') expect(Object.hasOwn(OAUTH_CLIENT_CAPABILITIES, capabilityId)).toBe(true) } }) diff --git a/apps/sim/lib/integrations/availability.ts b/apps/sim/lib/integrations/availability.ts index 7c1669ddc3c..6b44338c8e9 100644 --- a/apps/sim/lib/integrations/availability.ts +++ b/apps/sim/lib/integrations/availability.ts @@ -1,82 +1,22 @@ -import type { BlockVisibilityState } from '@/lib/core/config/block-visibility' -import type { EnvCapabilityValues } from '@/lib/core/config/env-capabilities' import { - inspectOAuthClientCapability, - resolveOAuthClientCapabilityId, -} from '@/lib/core/config/env-capabilities' + getPreviewServiceAccountProviderId, + type IntegrationAvailabilityState, +} from '@sim/deployment-config/integration-availability' +import type { BlockVisibilityState } from '@/lib/core/config/block-visibility' import { getServiceAccountGatingBlockType } from '@/lib/credentials/service-account-provider-ids' -import integrationsJson from '@/lib/integrations/integrations.json' -import { getServiceAccountMetadata } from '@/lib/integrations/service-account-metadata' import { isHiddenUnder } from '@/blocks/visibility/context' -export type IntegrationAvailabilityState = 'ready' | 'limited' | 'unavailable' | 'misconfigured' - -export interface IntegrationAvailability { - type: string - slug: string - name: string - state: IntegrationAvailabilityState - oauthAvailable: boolean - serviceAccountAvailable: boolean - missingFields: readonly string[] - setupCommand?: string -} - -interface DeploymentIntegration { - type: string - slug: string - name: string - authType: 'oauth' | 'api-key' | 'none' - oauthServiceId?: string -} - -const integrations = integrationsJson.integrations as readonly DeploymentIntegration[] -const deploymentGatedIntegrationTypes = new Set( - integrations - .filter((integration) => integration.authType === 'oauth') - .map((integration) => integration.type.toLowerCase()) -) -const integrationTypesByOAuthServiceId = new Map() -const previewServiceAccountGatesByIntegrationType = new Map() -for (const integration of integrations) { - if (integration.authType !== 'oauth' || !integration.oauthServiceId) continue - const serviceId = integration.oauthServiceId.toLowerCase() - const current = integrationTypesByOAuthServiceId.get(serviceId) ?? [] - const integrationType = integration.type.toLowerCase() - integrationTypesByOAuthServiceId.set(serviceId, [...current, integrationType]) - - const serviceAccount = getServiceAccountMetadata(serviceId) - if (serviceAccount?.deploymentRequirement !== 'preview-gated') continue - const gatingBlockType = getServiceAccountGatingBlockType(serviceAccount.providerId) - if (!gatingBlockType) { - throw new Error( - `Preview-gated service account ${serviceAccount.providerId} has no gating block type` - ) - } - previewServiceAccountGatesByIntegrationType.set(integrationType, gatingBlockType) -} - -export function isDeploymentGatedIntegrationType(blockType: string): boolean { - return deploymentGatedIntegrationTypes.has(blockType.toLowerCase()) -} - -/** Returns the generated integration block types authenticated by one OAuth service entry. */ -export function getIntegrationTypesForOAuthServiceId(serviceId: string): readonly string[] { - return integrationTypesByOAuthServiceId.get(serviceId.toLowerCase()) ?? [] -} - -/** Applies an integration allowlist to an OAuth service without loading executable registries. */ -export function isOAuthServiceAllowedByIntegrationTypes( - serviceId: string, - allowedIntegrationTypes: ReadonlySet | null -): boolean { - if (allowedIntegrationTypes === null) return true - const integrationTypes = getIntegrationTypesForOAuthServiceId(serviceId) - return ( - integrationTypes.length === 0 || - integrationTypes.some((blockType) => allowedIntegrationTypes.has(blockType)) - ) -} +export type { + IntegrationAvailability, + IntegrationAvailabilityState, +} from '@sim/deployment-config/integration-availability' +/** Application compatibility surface for pure deployment availability helpers. */ +export { + getIntegrationTypesForOAuthServiceId, + isDeploymentGatedIntegrationType, + isOAuthServiceAllowedByIntegrationTypes, + resolveIntegrationAvailability, +} from '@sim/deployment-config/integration-availability' interface IntegrationAvailabilitySummary { type: string @@ -95,81 +35,13 @@ export function resolveIntegrationAvailabilityStateForVisibility( availability: IntegrationAvailabilitySummary, visibility: BlockVisibilityState | null ): IntegrationAvailabilityState { - const gatingBlockType = previewServiceAccountGatesByIntegrationType.get( - availability.type.toLowerCase() - ) + const providerId = getPreviewServiceAccountProviderId(availability.type) + const gatingBlockType = providerId ? getServiceAccountGatingBlockType(providerId) : null + if (providerId && !gatingBlockType) { + throw new Error(`Preview-gated service account ${providerId} has no gating block type`) + } if (!gatingBlockType || isHiddenUnder(visibility, { type: gatingBlockType, preview: true })) { return availability.state } return availability.oauthAvailable ? 'ready' : 'limited' } - -function resolveOAuthIntegrationAvailability( - integration: DeploymentIntegration, - values: EnvCapabilityValues -): IntegrationAvailability { - const { oauthServiceId } = integration - if (!oauthServiceId) { - throw new Error(`OAuth integration ${integration.slug} is missing oauthServiceId`) - } - - const capabilityId = resolveOAuthClientCapabilityId(oauthServiceId) - const serviceAccount = getServiceAccountMetadata(oauthServiceId) - - if (!capabilityId) { - throw new Error( - `OAuth integration ${integration.slug} has no OAuth client capability definition` - ) - } - - const oauth = inspectOAuthClientCapability(capabilityId, values) - const setupCommand = `bun run setup integration ${capabilityId}` - const serviceAccountAvailable = Boolean( - serviceAccount && - serviceAccount.deploymentRequirement !== 'preview-gated' && - (serviceAccount.deploymentRequirement !== 'oauth-client' || oauth.state === 'ready') - ) - const state: IntegrationAvailabilityState = - oauth.state === 'ready' - ? 'ready' - : serviceAccountAvailable - ? 'limited' - : oauth.state === 'partial' || oauth.state === 'invalid' - ? 'misconfigured' - : 'unavailable' - - return { - type: integration.type, - slug: integration.slug, - name: integration.name, - state, - oauthAvailable: oauth.state === 'ready', - serviceAccountAvailable, - missingFields: oauth.missingFields, - setupCommand, - } -} - -/** - * Resolves deployment availability for every integration in the generated - * catalog using only caller-supplied environment values and pure metadata. - */ -export function resolveIntegrationAvailability( - values: EnvCapabilityValues -): readonly IntegrationAvailability[] { - return integrations.map((integration) => { - if (integration.authType === 'oauth') { - return resolveOAuthIntegrationAvailability(integration, values) - } - - return { - type: integration.type, - slug: integration.slug, - name: integration.name, - state: 'ready', - oauthAvailable: false, - serviceAccountAvailable: false, - missingFields: [], - } - }) -} diff --git a/apps/sim/lib/integrations/credential-display.test.ts b/apps/sim/lib/integrations/credential-display.test.ts index 4ae9a683e74..e0991f0aca7 100644 --- a/apps/sim/lib/integrations/credential-display.test.ts +++ b/apps/sim/lib/integrations/credential-display.test.ts @@ -1,6 +1,8 @@ /** * @vitest-environment node */ + +import integrationsJson from '@sim/deployment-config/integrations.json' import { describe, expect, it } from 'vitest' import { getIntegrationsForCredentialProvider, @@ -9,7 +11,6 @@ import { isFamilyServiceAccount, resolveCredentialDisplay, } from '@/lib/integrations/credential-display' -import integrationsJson from '@/lib/integrations/integrations.json' import { resolveOAuthServiceForIntegration } from '@/lib/integrations/oauth-service' import type { Integration } from '@/lib/integrations/types' import { OAUTH_PROVIDERS } from '@/lib/oauth/oauth' diff --git a/apps/sim/lib/integrations/credential-display.ts b/apps/sim/lib/integrations/credential-display.ts index 5bf924a611b..1f8d863bb12 100644 --- a/apps/sim/lib/integrations/credential-display.ts +++ b/apps/sim/lib/integrations/credential-display.ts @@ -11,8 +11,8 @@ */ import type { ComponentType } from 'react' +import integrationsJson from '@sim/deployment-config/integrations.json' import { getServiceAccountConnectNoun } from '@/lib/credentials/service-account-provider-ids' -import integrationsJson from '@/lib/integrations/integrations.json' import { CANONICAL_SERVICE_ACCOUNT_SLUGS } from '@/lib/integrations/oauth-service' import type { Integration } from '@/lib/integrations/types' import { OAUTH_PROVIDERS } from '@/lib/oauth/oauth' diff --git a/apps/sim/lib/integrations/credential-visibility.server.test.ts b/apps/sim/lib/integrations/credential-visibility.server.test.ts index 502adecdc3b..708d9833fc5 100644 --- a/apps/sim/lib/integrations/credential-visibility.server.test.ts +++ b/apps/sim/lib/integrations/credential-visibility.server.test.ts @@ -1,6 +1,7 @@ /** * @vitest-environment node */ + import { beforeEach, describe, expect, it, vi } from 'vitest' import type { IntegrationAvailability } from '@/lib/integrations/availability' import type { OAuthServiceMetadata } from '@/lib/oauth/types' diff --git a/apps/sim/lib/integrations/index.ts b/apps/sim/lib/integrations/index.ts index 97228c7cf69..463096463bc 100644 --- a/apps/sim/lib/integrations/index.ts +++ b/apps/sim/lib/integrations/index.ts @@ -14,7 +14,7 @@ * imports are fine — they erase. */ -import integrationsJson from '@/lib/integrations/integrations.json' +import integrationsJson from '@sim/deployment-config/integrations.json' import type { Integration, IntegrationSummary } from '@/lib/integrations/types' /** All integrations surfaced in the catalog, ordered by `scripts/generate-docs.ts`. */ diff --git a/apps/sim/lib/integrations/oauth-service.test.ts b/apps/sim/lib/integrations/oauth-service.test.ts index dbd70af7c58..f5a5e2dc661 100644 --- a/apps/sim/lib/integrations/oauth-service.test.ts +++ b/apps/sim/lib/integrations/oauth-service.test.ts @@ -1,8 +1,9 @@ /** * @vitest-environment node */ + +import integrationsJson from '@sim/deployment-config/integrations.json' import { describe, expect, it } from 'vitest' -import integrationsJson from '@/lib/integrations/integrations.json' import { resolveOAuthServiceForSlug, resolveServiceAccountIntegration, diff --git a/apps/sim/lib/integrations/oauth-service.ts b/apps/sim/lib/integrations/oauth-service.ts index d952c61f678..977d0845eab 100644 --- a/apps/sim/lib/integrations/oauth-service.ts +++ b/apps/sim/lib/integrations/oauth-service.ts @@ -1,6 +1,6 @@ import type { ComponentType } from 'react' +import integrationsJson from '@sim/deployment-config/integrations.json' import { asServiceAccountProviderId } from '@/lib/credentials/service-account-provider-ids' -import integrationsJson from '@/lib/integrations/integrations.json' import type { Integration } from '@/lib/integrations/types' import { getServiceConfigByServiceId } from '@/lib/oauth' import type { ServiceAccountProviderId } from '@/app/workspace/[workspaceId]/integrations/components/connect-service-account-modal' diff --git a/apps/sim/lib/integrations/service-account-metadata.ts b/apps/sim/lib/integrations/service-account-metadata.ts index a1a90346662..9e13b80eb33 100644 --- a/apps/sim/lib/integrations/service-account-metadata.ts +++ b/apps/sim/lib/integrations/service-account-metadata.ts @@ -1,61 +1,2 @@ -/** - * Lightweight deployment metadata for OAuth services that also accept a - * user-supplied service-account credential. - * - * This projection deliberately contains no icons, scopes, or OAuth runtime - * configuration so deployment tooling can inspect integration availability - * without loading the executable integration graph. Its parity with the - * canonical OAuth service configuration is enforced by an invariant test. - */ - -export interface ServiceAccountMetadata { - providerId: string - deploymentRequirement?: 'preview-gated' | 'oauth-client' -} - -export const SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID: Readonly< - Record -> = { - airtable: { providerId: 'airtable-service-account' }, - asana: { providerId: 'asana-service-account' }, - attio: { providerId: 'attio-service-account' }, - box: { providerId: 'box-service-account' }, - calcom: { providerId: 'calcom-service-account' }, - clickup: { providerId: 'clickup-service-account' }, - confluence: { providerId: 'atlassian-service-account' }, - gmail: { providerId: 'google-service-account' }, - 'google-bigquery': { providerId: 'google-service-account' }, - 'google-calendar': { providerId: 'google-service-account' }, - 'google-contacts': { providerId: 'google-service-account' }, - 'google-docs': { providerId: 'google-service-account' }, - 'google-drive': { providerId: 'google-service-account' }, - 'google-forms': { providerId: 'google-service-account' }, - 'google-groups': { providerId: 'google-service-account' }, - 'google-meet': { providerId: 'google-service-account' }, - 'google-sheets': { providerId: 'google-service-account' }, - 'google-tasks': { providerId: 'google-service-account' }, - 'google-vault': { providerId: 'google-service-account' }, - hubspot: { providerId: 'hubspot-service-account' }, - jira: { providerId: 'atlassian-service-account' }, - linear: { providerId: 'linear-service-account' }, - monday: { providerId: 'monday-service-account' }, - notion: { providerId: 'notion-service-account' }, - pipedrive: { providerId: 'pipedrive-service-account' }, - salesforce: { providerId: 'salesforce-service-account' }, - shopify: { providerId: 'shopify-service-account' }, - slack: { providerId: 'slack-custom-bot', deploymentRequirement: 'preview-gated' }, - trello: { providerId: 'trello-service-account', deploymentRequirement: 'oauth-client' }, - wealthbox: { providerId: 'wealthbox-service-account' }, - webflow: { providerId: 'webflow-service-account' }, - 'zoho-desk': { providerId: 'zoho-desk-service-account' }, - zoom: { providerId: 'zoom-service-account' }, -} as const - -export function getServiceAccountMetadata( - oauthServiceId: string -): ServiceAccountMetadata | undefined { - if (!Object.hasOwn(SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID, oauthServiceId)) { - return undefined - } - return SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID[oauthServiceId] -} +/** Application compatibility surface for shared service-account deployment metadata. */ +export * from '@sim/deployment-config/service-account-metadata' diff --git a/apps/sim/lib/oauth/oauth.test.ts b/apps/sim/lib/oauth/oauth.test.ts index b06cd09aaf0..56898366954 100644 --- a/apps/sim/lib/oauth/oauth.test.ts +++ b/apps/sim/lib/oauth/oauth.test.ts @@ -403,7 +403,7 @@ describe('OAuth Token Refresh', () => { expect(result).toEqual({ ok: false, message: - 'OAuth client monday is partially configured — missing MONDAY_CLIENT_SECRET. Run bun run setup integration monday.', + 'OAuth client monday is partially configured — missing MONDAY_CLIENT_SECRET. Run npx @sim/setup add integration monday.', }) expect(mockFetch).not.toHaveBeenCalled() }) diff --git a/apps/sim/package.json b/apps/sim/package.json index 6e2954f145d..370279e327c 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -109,6 +109,7 @@ "@sim/audit": "workspace:*", "@sim/auth": "workspace:*", "@sim/browser-protocol": "workspace:*", + "@sim/deployment-config": "workspace:*", "@sim/desktop-bridge": "workspace:*", "@sim/emcn": "workspace:*", "@sim/logger": "workspace:*", diff --git a/apps/sim/scripts/canvas-sentence-audit-corpus.ts b/apps/sim/scripts/canvas-sentence-audit-corpus.ts index 3c59e3872b7..500e902736d 100644 --- a/apps/sim/scripts/canvas-sentence-audit-corpus.ts +++ b/apps/sim/scripts/canvas-sentence-audit-corpus.ts @@ -33,7 +33,7 @@ function loadToolDescriptions(): Map> { const byType = new Map>() try { const catalog = JSON.parse( - readFileSync('apps/sim/lib/integrations/integrations.json', 'utf-8') + readFileSync('packages/deployment-config/src/integrations.json', 'utf-8') ) as { integrations: Array<{ type: string diff --git a/apps/sim/scripts/canvas-sentence-spec.ts b/apps/sim/scripts/canvas-sentence-spec.ts index d2b6439d283..4e70d2c5c3b 100644 --- a/apps/sim/scripts/canvas-sentence-spec.ts +++ b/apps/sim/scripts/canvas-sentence-spec.ts @@ -49,7 +49,7 @@ function loadToolDescriptions(type: string): Map { const byLabel = new Map() try { const catalog = JSON.parse( - readFileSync('apps/sim/lib/integrations/integrations.json', 'utf-8') + readFileSync('packages/deployment-config/src/integrations.json', 'utf-8') ) as { integrations: Array<{ type: string diff --git a/bun.lock b/bun.lock index 7d1a7d6e2a0..ccf6d710ff1 100644 --- a/bun.lock +++ b/bun.lock @@ -212,6 +212,7 @@ "@sim/audit": "workspace:*", "@sim/auth": "workspace:*", "@sim/browser-protocol": "workspace:*", + "@sim/deployment-config": "workspace:*", "@sim/desktop-bridge": "workspace:*", "@sim/emcn": "workspace:*", "@sim/logger": "workspace:*", @@ -450,6 +451,17 @@ "typescript": "^7.0.2", }, }, + "packages/deployment-config": { + "name": "@sim/deployment-config", + "version": "0.1.0", + "dependencies": { + "@sim/utils": "workspace:*", + }, + "devDependencies": { + "@sim/tsconfig": "workspace:*", + "typescript": "^7.0.2", + }, + }, "packages/desktop-bridge": { "name": "@sim/desktop-bridge", "version": "0.1.0", @@ -607,6 +619,26 @@ "vitest": "^4.1.0", }, }, + "packages/sim-setup": { + "name": "@sim/setup", + "version": "1.0.0", + "bin": { + "sim-setup": "dist/index.js", + }, + "devDependencies": { + "@clack/prompts": "1.7.0", + "@next/env": "16.2.12", + "@sim/deployment-config": "workspace:*", + "@sim/security": "workspace:*", + "@sim/tsconfig": "workspace:*", + "@sim/utils": "workspace:*", + "@types/node": "24.2.1", + "chalk": "5.6.2", + "postgres": "^3.4.5", + "typescript": "^7.0.2", + "vitest": "^4.1.0", + }, + }, "packages/terminal-protocol": { "name": "@sim/terminal-protocol", "version": "0.1.0", @@ -1785,6 +1817,8 @@ "@sim/db": ["@sim/db@workspace:packages/db"], + "@sim/deployment-config": ["@sim/deployment-config@workspace:packages/deployment-config"], + "@sim/desktop": ["@sim/desktop@workspace:apps/desktop"], "@sim/desktop-bridge": ["@sim/desktop-bridge@workspace:packages/desktop-bridge"], @@ -1805,6 +1839,8 @@ "@sim/security": ["@sim/security@workspace:packages/security"], + "@sim/setup": ["@sim/setup@workspace:packages/sim-setup"], + "@sim/terminal-protocol": ["@sim/terminal-protocol@workspace:packages/terminal-protocol"], "@sim/testing": ["@sim/testing@workspace:packages/testing"], diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index b6742c7b38e..c2c5c920377 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -134,12 +134,6 @@ services: # and shares no database schema with the app, so it does not need to move in # lockstep — and no cron tag exists for releases that predate it. image: ghcr.io/simstudioai/cron:${SIM_CRON_VERSION:-latest} - # Built from the repo when the published image is not available locally, so a - # fresh `git clone && docker compose up -d` works before the first release - # that publishes it. - build: - context: . - dockerfile: docker/cron.Dockerfile # on-failure, not unless-stopped: with no CRON_SECRET the container exits 0 # after explaining why, and stays stopped instead of crash-looping. Upgrades # from a compose file that predates this service therefore still come up. diff --git a/package.json b/package.json index df8cd486af4..99e5f832bcf 100644 --- a/package.json +++ b/package.json @@ -8,9 +8,6 @@ "apps/*", "packages/*" ], - "bin": { - "sim": "./scripts/setup/launcher.ts" - }, "scripts": { "build": "turbo run build", "dev": "turbo run dev", @@ -18,7 +15,7 @@ "dev:full": "bunx concurrently -n \"App,Realtime\" -c \"cyan,magenta\" \"cd apps/sim && bun run dev\" \"cd apps/realtime && bun run dev\"", "dev:full:capped": "bunx concurrently -n \"App,Realtime\" -c \"cyan,magenta\" \"cd apps/sim && bun run dev:capped\" \"cd apps/realtime && bun run dev\"", "test": "bun run test:setup && turbo run test", - "test:setup": "bun test scripts/setup", + "test:setup": "bun run --cwd packages/sim-setup test", "format": "turbo run format", "format:check": "turbo run format:check", "lint": "turbo run lint", @@ -64,6 +61,8 @@ "billing-protocol-contract:check": "bun run scripts/sync-billing-protocol-contract.ts --check", "tool-metadata:generate": "bun run scripts/sync-tool-metadata.ts", "tool-metadata:check": "bun run scripts/sync-tool-metadata.ts --check", + "deployment-config:generate": "bun run scripts/generate-deployment-config.ts", + "deployment-config:check": "bun run scripts/generate-deployment-config.ts --check", "integration-catalog:check": "bun run scripts/check-integration-catalog.ts", "docs:check": "bun run scripts/generate-docs.ts --check", "mship-tools:generate": "bun run scripts/sync-tool-catalog.ts", @@ -85,9 +84,9 @@ "library:covers": "bun run scripts/generate-library-covers.tsx", "library:covers:check": "bun run scripts/generate-library-covers.tsx --check", "skills:sync": "bun run scripts/sync-skills.ts", - "setup": "bun run scripts/setup/launcher.ts setup", - "sim": "bun run scripts/setup/launcher.ts", - "doctor": "bun run scripts/setup/launcher.ts doctor", + "setup": "bun run packages/sim-setup/src/index.ts setup", + "sim": "bun run packages/sim-setup/src/index.ts", + "doctor": "bun run packages/sim-setup/src/index.ts doctor", "agent-stream-docs:generate": "bun run scripts/sync-agent-stream-docs.ts", "agent-stream-docs:check": "bun run scripts/sync-agent-stream-docs.ts --check", "prepare": "bun husky", diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 47a85e9ff61..c862c419e99 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -44,7 +44,7 @@ const SECRET_KEYS = [ * Placeholders are rejected at any length: the repository publishes example values longer * than the 32-character minimum, so a copied `.env.example` would otherwise pass as real. * - * Mirrors `isUsableSecret` in `scripts/setup/env-files.ts`. + * Mirrors `isUsableSecret` in `packages/sim-setup/src/env-files.ts`. */ const AES_KEY_PATTERN = /^[0-9a-f]{64}$/i const AES_SECRET_KEYS = new Set(['ENCRYPTION_KEY', 'API_ENCRYPTION_KEY']) diff --git a/packages/deployment-config/package.json b/packages/deployment-config/package.json new file mode 100644 index 00000000000..f2afe6c03b6 --- /dev/null +++ b/packages/deployment-config/package.json @@ -0,0 +1,37 @@ +{ + "name": "@sim/deployment-config", + "version": "0.1.0", + "private": true, + "sideEffects": false, + "type": "module", + "license": "Apache-2.0", + "exports": { + "./env-capabilities": { + "types": "./src/env-capabilities.ts", + "default": "./src/env-capabilities.ts" + }, + "./integration-availability": { + "types": "./src/integration-availability.ts", + "default": "./src/integration-availability.ts" + }, + "./integrations.json": "./src/integrations.json", + "./service-account-metadata": { + "types": "./src/service-account-metadata.ts", + "default": "./src/service-account-metadata.ts" + } + }, + "scripts": { + "type-check": "tsc --noEmit", + "lint": "biome check --write --unsafe .", + "lint:check": "biome check .", + "format": "biome format --write .", + "format:check": "biome format ." + }, + "dependencies": { + "@sim/utils": "workspace:*" + }, + "devDependencies": { + "@sim/tsconfig": "workspace:*", + "typescript": "^7.0.2" + } +} diff --git a/packages/deployment-config/src/env-capabilities.ts b/packages/deployment-config/src/env-capabilities.ts new file mode 100644 index 00000000000..c58d39d747a --- /dev/null +++ b/packages/deployment-config/src/env-capabilities.ts @@ -0,0 +1,1493 @@ +/** + * Canonical runtime deployment-capability definitions. Keep this module free of application + * runtime dependencies so setup and diagnostics can consume the rules the app enforces. + * + * @packageDocumentation + */ +import { + IMMUTABLE_DAYTONA_SNAPSHOT_REF_ERROR, + IMMUTABLE_E2B_TEMPLATE_REF_ERROR, + isImmutableDaytonaSnapshotRef, + isImmutableE2BTemplateRef, + isValidSandboxReleaseGeneration, + SANDBOX_RELEASE_GENERATION_ERROR, +} from '@sim/utils/sandbox-references' + +export type EnvCapabilityValue = string | number | boolean | null | undefined + +export const CORE_CONFIGURATION_KEYS = [ + 'DATABASE_URL', + 'BETTER_AUTH_SECRET', + 'BETTER_AUTH_URL', + 'NEXT_PUBLIC_APP_URL', + 'ENCRYPTION_KEY', + 'INTERNAL_API_SECRET', +] as const + +export type EnvCapabilityValues = + | ReadonlyMap + | Readonly> + +export type EnvValueValidation = + | { + kind: 'integer' + min?: number + max?: number + message: string + } + | { + kind: 'json-object' + requiredStringFields?: readonly string[] + message: string + } + | { + kind: 'pattern' + pattern: RegExp + message: string + } + | { + kind: 'immutable-e2b-template-ref' + message: string + } + | { + kind: 'immutable-daytona-snapshot-ref' + message: string + } + | { + kind: 'sandbox-release-generation' + message: string + } + | { + kind: 'url' + protocols?: readonly string[] + message: string + } + +export interface EnvFieldRequirement { + type: 'field' + key: string + validation?: EnvValueValidation +} + +export interface AllOfRequirement { + type: 'allOf' + requirements: readonly EnvRequirement[] +} + +export interface AnyOfRequirement { + type: 'anyOf' + requirements: readonly EnvRequirement[] +} + +export type EnvRequirement = EnvFieldRequirement | AllOfRequirement | AnyOfRequirement + +export type EnvProviderActivation = + | { mode: 'any-present'; keys: readonly string[] } + | { mode: 'enabled'; key: string } + +export interface EnvProviderValidationIssue { + kind: 'missing' | 'invalid' + fields: readonly string[] + message: string +} + +export interface EnvProviderDefinition { + id: TId + label: string + activation: EnvProviderActivation + requires: EnvRequirement + pairedFields?: readonly (readonly [string, string])[] + optionalFields?: readonly EnvFieldRequirement[] + validate?: (values: EnvCapabilityValues) => readonly EnvProviderValidationIssue[] +} + +export interface FallbackCapabilityDefinition< + TId extends string = string, + TProvider extends EnvProviderDefinition = EnvProviderDefinition, +> { + strategy: 'fallback' + id: TId + label: string + providers: readonly TProvider[] +} + +export type EnvDefaultProviderDefinition = + | { id: string; kind: 'built-in'; label: string } + | { id: string; kind: 'provider' } + +export interface SelectedCapabilityDefinition< + TId extends string = string, + TProvider extends EnvProviderDefinition = EnvProviderDefinition, +> { + strategy: 'selected' + id: TId + label: string + selectorKey?: string + whenUnset: 'default' | 'first-ready' + defaultProvider: EnvDefaultProviderDefinition + providers: readonly TProvider[] +} + +export type CapabilityDefinition = FallbackCapabilityDefinition | SelectedCapabilityDefinition + +export type DeclaredProviderId = + TDefinition['providers'][number]['id'] + +export type ProviderId = + TDefinition extends SelectedCapabilityDefinition + ? DeclaredProviderId | TDefinition['defaultProvider']['id'] + : DeclaredProviderId + +export type FallbackFactories = { + [TId in DeclaredProviderId]: () => TProvider | null +} + +export type ProviderConfigurationState = 'absent' | 'partial' | 'ready' | 'invalid' + +export interface ProviderInspection { + id: TId + label: string + active: boolean + state: ProviderConfigurationState + missingFields: readonly string[] + invalidFields: readonly string[] + invalidDetails: readonly string[] +} + +export interface FallbackCapabilityInspection { + strategy: 'fallback' + configured: boolean + providerIds: readonly TId[] + providers: readonly ProviderInspection[] + error: EnvCapabilityConfigurationError | null +} + +export interface SelectedCapabilityInspection< + TProviderId extends string = string, + TDeclaredProviderId extends string = TProviderId, +> { + strategy: 'selected' + providerId: TProviderId | null + providers: readonly ProviderInspection[] + error: EnvCapabilityConfigurationError | null +} + +export type CapabilityInspection = + TDefinition extends SelectedCapabilityDefinition + ? SelectedCapabilityInspection, DeclaredProviderId> + : FallbackCapabilityInspection> + +export class EnvCapabilityConfigurationError extends Error { + constructor( + readonly capabilityId: string, + message: string + ) { + super(message) + this.name = 'EnvCapabilityConfigurationError' + } +} + +function readValue(values: EnvCapabilityValues, key: string): EnvCapabilityValue { + if (values instanceof Map) return values.get(key) + return (values as Readonly>)[key] +} + +function hasValue(values: EnvCapabilityValues, key: string): boolean { + const value = readValue(values, key) + if (value === undefined || value === null || value === false) return false + if (typeof value !== 'string') return true + const normalized = value.trim().toLowerCase() + return normalized !== '' && normalized !== 'placeholder' +} + +function isTruthyValue(values: EnvCapabilityValues, key: string): boolean { + const value = readValue(values, key) + if (value === true || value === 1) return true + if (typeof value !== 'string') return false + const normalized = value.toLowerCase() + return normalized === 'true' || normalized === '1' +} + +/** Returns whether an environment field contains a usable configuration value. */ +export function hasEnvCapabilityValue(values: EnvCapabilityValues, key: string): boolean { + return hasValue(values, key) +} + +/** Resolves the boolean semantics shared by capability selectors and status reporting. */ +export function isTruthyEnvCapabilityValue(values: EnvCapabilityValues, key: string): boolean { + return isTruthyValue(values, key) +} + +function unique(values: readonly string[]): string[] { + return [...new Set(values)] +} + +export function envField( + key: string, + options: Pick = {} +): EnvFieldRequirement { + return { type: 'field', key, ...options } +} + +export function allOf(...requirements: readonly EnvRequirement[]): AllOfRequirement { + return { type: 'allOf', requirements } +} + +export function anyOf(...requirements: readonly EnvRequirement[]): AnyOfRequirement { + return { type: 'anyOf', requirements } +} + +function requirementKeys(requirement: EnvRequirement): string[] { + return requirement.type === 'field' + ? [requirement.key] + : requirement.requirements.flatMap(requirementKeys) +} + +function activationKeys(activation: EnvProviderActivation): readonly string[] { + return activation.mode === 'enabled' ? [activation.key] : activation.keys +} + +function providerKeys(provider: EnvProviderDefinition): string[] { + return [ + ...activationKeys(provider.activation), + ...requirementKeys(provider.requires), + ...(provider.pairedFields ?? []).flat(), + ...(provider.optionalFields ?? []).map((field) => field.key), + ] +} + +/** Returns every environment field that can affect one provider at runtime. */ +export function getProviderFields(provider: EnvProviderDefinition): readonly string[] { + return unique(providerKeys(provider)) +} + +function providerIsActive(provider: EnvProviderDefinition, values: EnvCapabilityValues): boolean { + return provider.activation.mode === 'enabled' + ? isTruthyValue(values, provider.activation.key) + : provider.activation.keys.some((key) => hasValue(values, key)) +} + +function capabilityKeys(definition: CapabilityDefinition): string[] { + return [ + ...(definition.strategy === 'selected' && definition.selectorKey + ? [definition.selectorKey] + : []), + ...definition.providers.flatMap(providerKeys), + ] +} + +/** Returns every environment field that can affect a capability at runtime. */ +export function getCapabilityFields(definition: CapabilityDefinition): readonly string[] { + return unique(capabilityKeys(definition)) +} + +function assertRequirementDefinition( + capabilityId: string, + providerId: string, + requirement: EnvRequirement +): void { + if (requirement.type === 'field') { + if (!requirement.key) { + throw new Error(`Capability ${capabilityId} provider ${providerId} has an empty field key`) + } + return + } + if (requirement.requirements.length === 0) { + throw new Error( + `Capability ${capabilityId} provider ${providerId} has an empty ${requirement.type}` + ) + } + for (const child of requirement.requirements) { + assertRequirementDefinition(capabilityId, providerId, child) + } +} + +function assertCapabilityDefinition(definition: CapabilityDefinition): void { + if (definition.providers.length === 0) { + throw new Error(`Capability ${definition.id} must declare at least one provider`) + } + + const providerIds = definition.providers.map((provider) => provider.id) + if (new Set(providerIds).size !== providerIds.length) { + throw new Error(`Capability ${definition.id} has duplicate provider ids`) + } + + if (definition.strategy === 'selected') { + const defaultIsDeclared = providerIds.includes(definition.defaultProvider.id) + if (definition.defaultProvider.kind === 'provider' && !defaultIsDeclared) { + throw new Error( + `Capability ${definition.id} default provider ${definition.defaultProvider.id} is not declared` + ) + } + if (definition.defaultProvider.kind === 'built-in' && defaultIsDeclared) { + throw new Error( + `Capability ${definition.id} built-in default ${definition.defaultProvider.id} also appears in providers` + ) + } + } + + for (const provider of definition.providers) { + if (provider.activation.mode === 'any-present' && provider.activation.keys.length === 0) { + throw new Error(`Capability ${definition.id} provider ${provider.id} has no activation keys`) + } + assertRequirementDefinition(definition.id, provider.id, provider.requires) + } +} + +export function defineCapability( + definition: TDefinition +): TDefinition { + assertCapabilityDefinition(definition) + return definition +} + +/** Returns the canonical command for configuring a runtime capability. */ +export function getCapabilitySetupCommand(definition: CapabilityDefinition): string { + return `npx @sim/setup add ${definition.id}` +} + +interface RequirementInspection { + ready: boolean + missingFields: readonly string[] + invalidFields: readonly string[] + invalidDetails: readonly string[] +} + +function isValidEnvCapabilityFieldValue( + validation: EnvValueValidation, + value: EnvCapabilityValue +): boolean { + const serialized = String(value) + if (validation.kind === 'integer') { + const number = Number(serialized) + return ( + Number.isInteger(number) && + (validation.min === undefined || number >= validation.min) && + (validation.max === undefined || number <= validation.max) + ) + } + if (validation.kind === 'json-object') { + try { + const parsed: unknown = JSON.parse(serialized) + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) return false + return (validation.requiredStringFields ?? []).every( + (field) => + field in parsed && + typeof (parsed as Record)[field] === 'string' && + ((parsed as Record)[field] as string).length > 0 + ) + } catch { + return false + } + } + if (validation.kind === 'pattern') { + validation.pattern.lastIndex = 0 + return validation.pattern.test(serialized) + } + if (validation.kind === 'immutable-e2b-template-ref') { + return isImmutableE2BTemplateRef(serialized) + } + if (validation.kind === 'immutable-daytona-snapshot-ref') { + return isImmutableDaytonaSnapshotRef(serialized) + } + if (validation.kind === 'sandbox-release-generation') { + return isValidSandboxReleaseGeneration(serialized) + } + try { + const parsed = new URL(serialized) + return !validation.protocols || validation.protocols.includes(parsed.protocol) + } catch { + return false + } +} + +function inspectField( + requirement: EnvFieldRequirement, + values: EnvCapabilityValues +): RequirementInspection { + if (!hasValue(values, requirement.key)) { + return { + ready: false, + missingFields: [requirement.key], + invalidFields: [], + invalidDetails: [], + } + } + + const value = readValue(values, requirement.key) + const valid = requirement.validation + ? isValidEnvCapabilityFieldValue(requirement.validation, value) + : true + + return valid + ? { ready: true, missingFields: [], invalidFields: [], invalidDetails: [] } + : { + ready: false, + missingFields: [], + invalidFields: [requirement.key], + invalidDetails: [`${requirement.key} ${requirement.validation?.message ?? 'is invalid'}`], + } +} + +function inspectRequirement( + requirement: EnvRequirement, + values: EnvCapabilityValues +): RequirementInspection { + if (requirement.type === 'field') return inspectField(requirement, values) + + const inspections = requirement.requirements.map((child) => inspectRequirement(child, values)) + if (requirement.type === 'anyOf') { + const ready = inspections.find((inspection) => inspection.ready) + if (ready) return ready + return inspections.reduce((best, candidate) => { + const bestIssueCount = best.missingFields.length + best.invalidFields.length + const candidateIssueCount = candidate.missingFields.length + candidate.invalidFields.length + return candidateIssueCount < bestIssueCount ? candidate : best + }) + } + + return { + ready: inspections.every((inspection) => inspection.ready), + missingFields: unique(inspections.flatMap((inspection) => inspection.missingFields)), + invalidFields: unique(inspections.flatMap((inspection) => inspection.invalidFields)), + invalidDetails: unique(inspections.flatMap((inspection) => inspection.invalidDetails)), + } +} + +function inspectProviderRequirements( + provider: TProvider, + values: EnvCapabilityValues, + active = true +): ProviderInspection { + const inspection = inspectRequirement(provider.requires, values) + const invalidOptionalFields = (provider.optionalFields ?? []).flatMap((field) => { + if (!hasValue(values, field.key)) return [] + return inspectField(field, values).invalidFields + }) + const invalidPairs = (provider.pairedFields ?? []).flatMap(([left, right]) => + hasValue(values, left) === hasValue(values, right) ? [] : [left, right] + ) + const customIssues = provider.validate?.(values) ?? [] + const missingFields = unique([ + ...inspection.missingFields, + ...customIssues + .filter((customIssue) => customIssue.kind === 'missing') + .flatMap((customIssue) => customIssue.fields), + ]) + const invalidFields = unique([ + ...inspection.invalidFields, + ...invalidOptionalFields, + ...invalidPairs, + ...customIssues + .filter((customIssue) => customIssue.kind === 'invalid') + .flatMap((customIssue) => customIssue.fields), + ]) + const invalidDetails = unique([ + ...inspection.invalidDetails, + ...(provider.optionalFields ?? []).flatMap((field) => { + if (!hasValue(values, field.key)) return [] + return inspectField(field, values).invalidDetails + }), + ...(provider.pairedFields ?? []).flatMap(([left, right]) => + hasValue(values, left) === hasValue(values, right) + ? [] + : [`${left} and ${right} must be set together`] + ), + ...customIssues.map((customIssue) => customIssue.message), + ]) + return { + id: provider.id, + label: provider.label, + active, + state: + invalidFields.length > 0 + ? 'invalid' + : missingFields.length > 0 || !inspection.ready + ? 'partial' + : 'ready', + missingFields, + invalidFields, + invalidDetails, + } +} + +function inspectRequiredProvider( + provider: TProvider, + values: EnvCapabilityValues +): ProviderInspection { + const active = providerIsActive(provider, values) + const inspection = inspectProviderRequirements(provider, values, active) + if (active || provider.activation.mode !== 'enabled') return inspection + + return { + ...inspection, + state: 'invalid', + invalidFields: unique([...inspection.invalidFields, provider.activation.key]), + invalidDetails: unique([ + ...inspection.invalidDetails, + `${provider.activation.key} must be enabled`, + ]), + } +} + +export function inspectProvider( + provider: TProvider, + values: EnvCapabilityValues +): ProviderInspection { + const active = providerIsActive(provider, values) + return active + ? inspectProviderRequirements(provider, values, true) + : { + id: provider.id, + label: provider.label, + active: false, + state: 'absent', + missingFields: [], + invalidFields: [], + invalidDetails: [], + } +} + +function providerProblems(inspection: ProviderInspection): string { + return [ + inspection.missingFields.length > 0 ? `missing ${inspection.missingFields.join(', ')}` : null, + inspection.invalidDetails.length > 0 + ? inspection.invalidDetails.join(', ') + : inspection.invalidFields.length > 0 + ? `invalid ${inspection.invalidFields.join(', ')}` + : null, + ] + .filter(Boolean) + .join('; ') +} + +export function getCapabilityConfigurationError( + definition: CapabilityDefinition, + inspections: readonly ProviderInspection[] +): EnvCapabilityConfigurationError | null { + const broken = inspections.filter( + (inspection) => inspection.state === 'partial' || inspection.state === 'invalid' + ) + if (broken.length === 0) return null + + const details = broken.map((inspection) => `${inspection.label}: ${providerProblems(inspection)}`) + + return new EnvCapabilityConfigurationError( + definition.id, + `${definition.label} is partially or incorrectly configured (${details.join(' | ')}). Run ${getCapabilitySetupCommand(definition)}.` + ) +} + +function replaceProviderInspection( + providers: readonly ProviderInspection[], + replacement: ProviderInspection +): ProviderInspection[] { + return providers.map((provider) => (provider.id === replacement.id ? replacement : provider)) +} + +function inspectSelectedCapability( + definition: TDefinition, + values: EnvCapabilityValues +): SelectedCapabilityInspection, DeclaredProviderId> { + const rawSelector = definition.selectorKey ? readValue(values, definition.selectorKey) : undefined + const selector = + definition.selectorKey && hasValue(values, definition.selectorKey) + ? String(rawSelector).trim().toLowerCase() + : null + let providers = definition.providers.map((provider) => inspectProvider(provider, values)) + const known = new Set([ + definition.defaultProvider.id, + ...definition.providers.map((provider) => provider.id), + ]) + + if (selector && !known.has(selector)) { + const error = new EnvCapabilityConfigurationError( + definition.id, + `Unknown ${definition.selectorKey} "${rawSelector}". Expected one of: ${[...known].join(', ')}` + ) + return { strategy: 'selected', providerId: null, providers, error } + } + + if (selector) { + const selectedDefinition = definition.providers.find((provider) => provider.id === selector) + if (!selectedDefinition) { + return { + strategy: 'selected', + providerId: definition.defaultProvider.id as ProviderId, + providers, + error: null, + } + } + const active = providerIsActive(selectedDefinition, values) + const selected = + !active && selectedDefinition.activation.mode === 'enabled' + ? inspectProvider(selectedDefinition, values) + : inspectProviderRequirements(selectedDefinition, values, active) + providers = replaceProviderInspection(providers, selected) + const error = + selected.state === 'ready' || + (selected.state === 'absent' && selectedDefinition.activation.mode === 'enabled') + ? null + : new EnvCapabilityConfigurationError( + definition.id, + `${definition.label} selects ${selector}, but that provider is not configured (${providerProblems(selected)}). Run ${getCapabilitySetupCommand(definition)}.` + ) + return { + strategy: 'selected', + providerId: selector as ProviderId, + providers, + error, + } + } + + if (definition.whenUnset === 'default') { + const defaultDefinition = definition.providers.find( + (provider) => provider.id === definition.defaultProvider.id + ) + if (!defaultDefinition) { + return { + strategy: 'selected', + providerId: definition.defaultProvider.id as ProviderId, + providers, + error: null, + } + } + const selected = providers.find((provider) => provider.id === definition.defaultProvider.id) + return { + strategy: 'selected', + providerId: definition.defaultProvider.id as ProviderId, + providers, + error: + !selected || selected.state === 'ready' || selected.state === 'absent' + ? null + : new EnvCapabilityConfigurationError( + definition.id, + `${definition.label} selects ${definition.defaultProvider.id}, but that provider is not configured (${providerProblems(selected)}). Run ${getCapabilitySetupCommand(definition)}.` + ), + } + } + + const candidates = providers.filter((provider) => provider.id !== definition.defaultProvider.id) + for (const candidate of candidates) { + if (candidate.state === 'ready') { + return { + strategy: 'selected', + providerId: candidate.id as ProviderId, + providers, + error: null, + } + } + if ( + candidate.state === 'invalid' && + candidate.missingFields.length === 0 && + candidate.invalidFields.length > 0 + ) { + return { + strategy: 'selected', + providerId: candidate.id as ProviderId, + providers, + error: new EnvCapabilityConfigurationError( + definition.id, + `${candidate.label} is incorrectly configured (${providerProblems(candidate)}). Run ${getCapabilitySetupCommand(definition)}.` + ), + } + } + } + + const error = getCapabilityConfigurationError(definition, candidates) + const broken = candidates.find( + (provider) => provider.state === 'partial' || provider.state === 'invalid' + ) + + return { + strategy: 'selected', + providerId: (broken?.id ?? definition.defaultProvider.id) as ProviderId, + providers, + error, + } +} + +function inspectFallbackCapability( + definition: TDefinition, + values: EnvCapabilityValues +): FallbackCapabilityInspection> { + const providers = definition.providers.map((provider) => inspectProvider(provider, values)) + const providerIds = providers + .filter((provider) => provider.state === 'ready') + .map((provider) => provider.id) as DeclaredProviderId[] + const configurationError = getCapabilityConfigurationError(definition, providers) + + return { + strategy: 'fallback', + configured: providerIds.length > 0, + providerIds, + providers, + error: providerIds.length === 0 ? configurationError : null, + } +} + +export function inspectCapability( + definition: TDefinition, + values: EnvCapabilityValues +): SelectedCapabilityInspection, DeclaredProviderId> +export function inspectCapability( + definition: TDefinition, + values: EnvCapabilityValues +): FallbackCapabilityInspection> +export function inspectCapability( + definition: CapabilityDefinition, + values: EnvCapabilityValues +): SelectedCapabilityInspection | FallbackCapabilityInspection +export function inspectCapability( + definition: CapabilityDefinition, + values: EnvCapabilityValues +): SelectedCapabilityInspection | FallbackCapabilityInspection { + return definition.strategy === 'selected' + ? inspectSelectedCapability(definition, values) + : inspectFallbackCapability(definition, values) +} + +export function requireCapability( + definition: TDefinition, + values: EnvCapabilityValues +): Omit< + SelectedCapabilityInspection, DeclaredProviderId>, + 'error' | 'providerId' +> & { + providerId: ProviderId +} +export function requireCapability( + definition: TDefinition, + values: EnvCapabilityValues +): Omit>, 'error'> +export function requireCapability( + definition: CapabilityDefinition, + values: EnvCapabilityValues +): + | (Omit & { + providerId: string + }) + | Omit { + const inspection = + definition.strategy === 'selected' + ? inspectSelectedCapability(definition, values) + : inspectFallbackCapability(definition, values) + if (inspection.error) throw inspection.error + if (inspection.strategy === 'selected') { + const providerId = inspection.providerId + if (providerId === null) { + throw new EnvCapabilityConfigurationError( + definition.id, + `${definition.label} has no selected provider. Run ${getCapabilitySetupCommand(definition)}.` + ) + } + const selected = inspection.providers.find((provider) => provider.id === providerId) + if (selected && selected.state !== 'ready') { + const selectedDefinition = definition.providers.find((provider) => provider.id === providerId) + const strictInspection = + selected.state === 'absent' && selectedDefinition + ? inspectRequiredProvider(selectedDefinition, values) + : selected + throw new EnvCapabilityConfigurationError( + definition.id, + `${definition.label} selects ${providerId}, but that provider is not configured (${providerProblems(strictInspection)}). Run ${getCapabilitySetupCommand(definition)}.` + ) + } + return { + strategy: 'selected', + providerId, + providers: inspection.providers, + } + } + if (!inspection.configured) { + throw new EnvCapabilityConfigurationError( + definition.id, + `${definition.label} is not configured. Run ${getCapabilitySetupCommand(definition)}.` + ) + } + const { error: _, ...resolution } = inspection + return resolution +} + +function findFieldRequirement( + requirement: EnvRequirement, + key: string +): EnvFieldRequirement | null { + if (requirement.type === 'field') return requirement.key === key ? requirement : null + for (const child of requirement.requirements) { + const field = findFieldRequirement(child, key) + if (field) return field + } + return null +} + +/** Validates a candidate environment value with the runtime field rule. */ +export function validateCapabilityFieldInput( + definition: CapabilityDefinition, + key: string, + value: string +): string | undefined { + if (!value) return 'required' + for (const provider of definition.providers) { + const field = + findFieldRequirement(provider.requires, key) ?? + provider.optionalFields?.find((candidate) => candidate.key === key) + if (!field) continue + if (!field.validation || isValidEnvCapabilityFieldValue(field.validation, value)) { + return undefined + } + return field.validation.message + } + throw new Error(`${definition.label} has no validation definition for ${key}`) +} + +export interface WireFallbackOptions { + definition: TDefinition + values: EnvCapabilityValues + factories: FallbackFactories + shouldFallback?: (error: unknown, providerId: DeclaredProviderId) => boolean + onFailure?: (providerId: DeclaredProviderId, error: unknown) => void +} + +export function wireFallback({ + definition, + values, + factories, + shouldFallback, + onFailure, +}: WireFallbackOptions) { + const resolution = inspectCapability(definition, values) + if (resolution.error) throw resolution.error + const providers = resolution.providerIds.map((providerId) => { + const provider = factories[providerId]() + if (!provider) { + throw new EnvCapabilityConfigurationError( + definition.id, + `${definition.label} provider ${providerId} resolved as ready but its factory returned null` + ) + } + return { id: providerId, provider } + }) + + return { + configured: resolution.configured, + providerIds: resolution.providerIds, + providers: providers.map(({ provider }) => provider), + async execute( + operation: ( + provider: TProvider, + providerId: DeclaredProviderId + ) => Promise + ): Promise { + if (resolution.providerIds.length === 0) { + throw new EnvCapabilityConfigurationError( + definition.id, + `${definition.label} is not configured. Run ${getCapabilitySetupCommand(definition)}.` + ) + } + + const failures: unknown[] = [] + for (const { id: providerId, provider } of providers) { + try { + return await operation(provider, providerId) + } catch (error) { + if (shouldFallback && !shouldFallback(error, providerId)) throw error + failures.push(error) + onFailure?.(providerId, error) + } + } + + throw new AggregateError( + failures, + `All ${definition.label} providers failed: ${resolution.providerIds.join(', ')}` + ) + }, + } +} + +export const EMAIL_CAPABILITY = defineCapability({ + strategy: 'fallback', + id: 'email', + label: 'Email', + providers: [ + { + id: 'resend', + label: 'Resend', + activation: { mode: 'any-present', keys: ['RESEND_API_KEY'] }, + requires: envField('RESEND_API_KEY'), + }, + { + id: 'ses', + label: 'Amazon SES', + activation: { mode: 'any-present', keys: ['AWS_SES_REGION'] }, + requires: envField('AWS_SES_REGION'), + }, + { + id: 'smtp', + label: 'SMTP', + activation: { + mode: 'any-present', + keys: ['SMTP_HOST', 'SMTP_PORT', 'SMTP_USER', 'SMTP_PASS'], + }, + requires: allOf( + envField('SMTP_HOST'), + envField('SMTP_PORT', { + validation: { + kind: 'integer', + min: 1, + max: 65535, + message: 'must be a valid port between 1 and 65535', + }, + }) + ), + optionalFields: [envField('SMTP_USER'), envField('SMTP_PASS')], + }, + { + id: 'azure', + label: 'Azure Communication Services', + activation: { + mode: 'any-present', + keys: ['AZURE_ACS_CONNECTION_STRING'], + }, + requires: envField('AZURE_ACS_CONNECTION_STRING'), + }, + { + id: 'gmail', + label: 'Gmail', + activation: { + mode: 'any-present', + keys: ['GMAIL_CREDENTIALS_JSON', 'GMAIL_SENDER'], + }, + requires: allOf( + envField('GMAIL_CREDENTIALS_JSON', { + validation: { + kind: 'json-object', + requiredStringFields: ['client_email', 'private_key'], + message: 'must be service account JSON with client_email and private_key', + }, + }), + envField('GMAIL_SENDER') + ), + }, + ], +} as const) + +export const STORAGE_CAPABILITY = defineCapability({ + strategy: 'selected', + id: 'storage', + label: 'File storage', + selectorKey: 'STORAGE_PROVIDER', + whenUnset: 'first-ready', + defaultProvider: { id: 'local', kind: 'built-in', label: 'Local disk' }, + providers: [ + { + id: 'azure', + label: 'Azure Blob Storage', + activation: { + mode: 'any-present', + keys: [ + 'AZURE_CONNECTION_STRING', + 'AZURE_ACCOUNT_NAME', + 'AZURE_ACCOUNT_KEY', + 'AZURE_STORAGE_CONTAINER_NAME', + ], + }, + requires: allOf( + envField('AZURE_STORAGE_CONTAINER_NAME'), + anyOf( + envField('AZURE_CONNECTION_STRING'), + allOf(envField('AZURE_ACCOUNT_NAME'), envField('AZURE_ACCOUNT_KEY')) + ) + ), + }, + { + id: 's3', + label: 'S3', + activation: { + mode: 'any-present', + keys: [ + 'S3_BUCKET_NAME', + 'S3_KB_BUCKET_NAME', + 'S3_EXECUTION_FILES_BUCKET_NAME', + 'S3_CHAT_BUCKET_NAME', + 'S3_COPILOT_BUCKET_NAME', + 'S3_PROFILE_PICTURES_BUCKET_NAME', + 'S3_OG_IMAGES_BUCKET_NAME', + 'S3_WORKSPACE_LOGOS_BUCKET_NAME', + 'S3_ENDPOINT', + ], + }, + requires: allOf(envField('AWS_REGION'), envField('S3_BUCKET_NAME')), + pairedFields: [['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']], + optionalFields: [ + envField('S3_ENDPOINT', { + validation: { + kind: 'url', + protocols: ['http:', 'https:'], + message: 'must be a valid http:// or https:// URL', + }, + }), + envField('S3_FORCE_PATH_STYLE'), + ], + }, + { + id: 'gcs', + label: 'Google Cloud Storage', + activation: { mode: 'any-present', keys: ['GCS_BUCKET_NAME'] }, + requires: envField('GCS_BUCKET_NAME'), + optionalFields: [ + envField('GCS_CREDENTIALS_JSON', { + validation: { + kind: 'json-object', + requiredStringFields: ['client_email', 'private_key'], + message: 'must be service account JSON with client_email and private_key', + }, + }), + envField('GCS_PROJECT_ID'), + ], + }, + ], +} as const) + +export const SANDBOX_CAPABILITY = defineCapability({ + strategy: 'selected', + id: 'sandbox', + label: 'Remote sandbox', + selectorKey: 'SANDBOX_PROVIDER', + whenUnset: 'default', + defaultProvider: { id: 'e2b', kind: 'provider' }, + providers: [ + { + id: 'e2b', + label: 'E2B', + activation: { mode: 'enabled', key: 'E2B_ENABLED' }, + requires: allOf( + envField('E2B_API_KEY'), + envField('E2B_FUNCTION_TEMPLATE_ID', { + validation: { + kind: 'immutable-e2b-template-ref', + message: IMMUTABLE_E2B_TEMPLATE_REF_ERROR, + }, + }), + envField('E2B_FUNCTION_TEMPLATE_GENERATION', { + validation: { + kind: 'sandbox-release-generation', + message: SANDBOX_RELEASE_GENERATION_ERROR, + }, + }) + ), + optionalFields: [ + envField('NEXT_PUBLIC_E2B_ENABLED'), + envField('NEXT_PUBLIC_SANDBOXES_ENABLED'), + ], + }, + { + id: 'daytona', + label: 'Daytona', + activation: { + mode: 'any-present', + keys: ['DAYTONA_API_KEY', 'DAYTONA_FUNCTION_SNAPSHOT_ID'], + }, + requires: allOf( + envField('DAYTONA_API_KEY'), + envField('DAYTONA_FUNCTION_SNAPSHOT_ID', { + validation: { + kind: 'immutable-daytona-snapshot-ref', + message: IMMUTABLE_DAYTONA_SNAPSHOT_REF_ERROR, + }, + }) + ), + optionalFields: [ + envField('NEXT_PUBLIC_E2B_ENABLED'), + envField('NEXT_PUBLIC_SANDBOXES_ENABLED'), + ], + }, + ], +} as const) + +export const ASYNC_JOBS_CAPABILITY = defineCapability({ + strategy: 'selected', + id: 'jobs', + label: 'Async jobs', + whenUnset: 'first-ready', + defaultProvider: { + id: 'database', + kind: 'built-in', + label: 'Database queue', + }, + providers: [ + { + id: 'trigger-dev', + label: 'Trigger.dev', + activation: { mode: 'enabled', key: 'TRIGGER_DEV_ENABLED' }, + requires: allOf(envField('TRIGGER_PROJECT_ID'), envField('TRIGGER_SECRET_KEY')), + }, + ], +} as const) + +/** Validates the one provider dependency that cannot be expressed as a field-shape rule. */ +function validateRedisProvider(values: EnvCapabilityValues): readonly EnvProviderValidationIssue[] { + if (!hasValue(values, 'REDIS_URL')) return [] + let redisUrl: URL + try { + redisUrl = new URL(String(readValue(values, 'REDIS_URL'))) + } catch { + return [] + } + if ( + redisUrl.protocol === 'rediss:' && + /^\d+\.\d+\.\d+\.\d+$/.test(redisUrl.hostname) && + !hasValue(values, 'REDIS_TLS_SERVERNAME') + ) { + return [ + { + kind: 'missing', + fields: ['REDIS_TLS_SERVERNAME'], + message: 'REDIS_TLS_SERVERNAME is required for rediss:// IP addresses', + }, + ] + } + return [] +} + +export const CACHE_CAPABILITY = defineCapability({ + strategy: 'selected', + id: 'cache', + label: 'Cache', + whenUnset: 'first-ready', + defaultProvider: { id: 'database', kind: 'built-in', label: 'Postgres' }, + providers: [ + { + id: 'redis', + label: 'Redis', + activation: { mode: 'any-present', keys: ['REDIS_URL'] }, + requires: envField('REDIS_URL', { + validation: { + kind: 'url', + protocols: ['redis:', 'rediss:'], + message: 'must be a valid redis:// or rediss:// URL', + }, + }), + optionalFields: [envField('REDIS_TLS_SERVERNAME')], + validate: validateRedisProvider, + }, + ], +} as const) + +export const OCR_CAPABILITY = defineCapability({ + strategy: 'selected', + id: 'knowledge', + label: 'PDF OCR', + selectorKey: 'OCR_PROVIDER', + whenUnset: 'first-ready', + defaultProvider: { id: 'local', kind: 'built-in', label: 'Local parser' }, + providers: [ + { + id: 'azure-mistral', + label: 'Azure Mistral OCR', + activation: { + mode: 'any-present', + keys: ['OCR_AZURE_API_KEY', 'OCR_AZURE_ENDPOINT', 'OCR_AZURE_MODEL_NAME'], + }, + requires: allOf( + envField('OCR_AZURE_API_KEY'), + envField('OCR_AZURE_ENDPOINT', { + validation: { + kind: 'url', + protocols: ['http:', 'https:'], + message: 'must be a valid HTTP(S) URL', + }, + }), + envField('OCR_AZURE_MODEL_NAME') + ), + }, + { + id: 'mistral', + label: 'Mistral OCR', + activation: { mode: 'any-present', keys: ['MISTRAL_API_KEY'] }, + requires: envField('MISTRAL_API_KEY'), + }, + ], +} as const) + +export const KNOWLEDGE_EMBEDDINGS_CAPABILITY = defineCapability({ + strategy: 'fallback', + id: 'knowledge-embeddings', + label: 'Knowledge embeddings', + providers: [ + { + id: 'azure-openai', + label: 'Azure OpenAI', + activation: { + mode: 'any-present', + keys: ['AZURE_OPENAI_API_KEY', 'AZURE_OPENAI_ENDPOINT', 'AZURE_OPENAI_API_VERSION'], + }, + requires: allOf( + envField('AZURE_OPENAI_API_KEY'), + envField('AZURE_OPENAI_ENDPOINT', { + validation: { + kind: 'url', + protocols: ['http:', 'https:'], + message: 'must be a valid HTTP(S) URL', + }, + }), + envField('AZURE_OPENAI_API_VERSION') + ), + optionalFields: [envField('KB_OPENAI_MODEL_NAME')], + }, + { + id: 'openai', + label: 'OpenAI', + activation: { + mode: 'any-present', + keys: ['OPENAI_API_KEY', 'OPENAI_API_KEY_1', 'OPENAI_API_KEY_2', 'OPENAI_API_KEY_3'], + }, + requires: anyOf( + envField('OPENAI_API_KEY'), + envField('OPENAI_API_KEY_1'), + envField('OPENAI_API_KEY_2'), + envField('OPENAI_API_KEY_3') + ), + }, + { + id: 'openrouter', + label: 'OpenRouter', + activation: { mode: 'any-present', keys: ['OPENROUTER_API_KEY'] }, + requires: envField('OPENROUTER_API_KEY'), + }, + ], +} as const) + +export const OAUTH_CLIENT_CAPABILITIES = { + google: ['GOOGLE_CLIENT_ID', 'GOOGLE_CLIENT_SECRET'], + x: ['X_CLIENT_ID', 'X_CLIENT_SECRET'], + tiktok: ['TIKTOK_CLIENT_ID', 'TIKTOK_CLIENT_SECRET'], + confluence: ['CONFLUENCE_CLIENT_ID', 'CONFLUENCE_CLIENT_SECRET'], + jira: ['JIRA_CLIENT_ID', 'JIRA_CLIENT_SECRET'], + calcom: ['CALCOM_CLIENT_ID'], + airtable: ['AIRTABLE_CLIENT_ID', 'AIRTABLE_CLIENT_SECRET'], + notion: ['NOTION_CLIENT_ID', 'NOTION_CLIENT_SECRET'], + microsoft: ['MICROSOFT_CLIENT_ID', 'MICROSOFT_CLIENT_SECRET'], + clickup: ['CLICKUP_CLIENT_ID', 'CLICKUP_CLIENT_SECRET'], + linear: ['LINEAR_CLIENT_ID', 'LINEAR_CLIENT_SECRET'], + attio: ['ATTIO_CLIENT_ID', 'ATTIO_CLIENT_SECRET'], + box: ['BOX_CLIENT_ID', 'BOX_CLIENT_SECRET'], + docusign: ['DOCUSIGN_CLIENT_ID', 'DOCUSIGN_CLIENT_SECRET'], + dropbox: ['DROPBOX_CLIENT_ID', 'DROPBOX_CLIENT_SECRET'], + slack: ['SLACK_CLIENT_ID', 'SLACK_CLIENT_SECRET'], + reddit: ['REDDIT_CLIENT_ID', 'REDDIT_CLIENT_SECRET'], + wealthbox: ['WEALTHBOX_CLIENT_ID', 'WEALTHBOX_CLIENT_SECRET'], + webflow: ['WEBFLOW_CLIENT_ID', 'WEBFLOW_CLIENT_SECRET'], + asana: ['ASANA_CLIENT_ID', 'ASANA_CLIENT_SECRET'], + pipedrive: ['PIPEDRIVE_CLIENT_ID', 'PIPEDRIVE_CLIENT_SECRET'], + hubspot: ['HUBSPOT_CLIENT_ID', 'HUBSPOT_CLIENT_SECRET'], + linkedin: ['LINKEDIN_CLIENT_ID', 'LINKEDIN_CLIENT_SECRET'], + instagram: ['INSTAGRAM_CLIENT_ID', 'INSTAGRAM_CLIENT_SECRET'], + salesforce: ['SALESFORCE_CLIENT_ID', 'SALESFORCE_CLIENT_SECRET'], + shopify: ['SHOPIFY_CLIENT_ID', 'SHOPIFY_CLIENT_SECRET'], + zoom: ['ZOOM_CLIENT_ID', 'ZOOM_CLIENT_SECRET'], + wordpress: ['WORDPRESS_CLIENT_ID', 'WORDPRESS_CLIENT_SECRET'], + spotify: ['SPOTIFY_CLIENT_ID', 'SPOTIFY_CLIENT_SECRET'], + monday: ['MONDAY_CLIENT_ID', 'MONDAY_CLIENT_SECRET'], + trello: ['TRELLO_API_KEY'], + 'zoho-desk': ['ZOHO_CLIENT_ID', 'ZOHO_CLIENT_SECRET'], +} as const + +/** Single registry consumed by runtime status and environment-source detection. */ +export const ENV_CAPABILITIES = [ + EMAIL_CAPABILITY, + STORAGE_CAPABILITY, + SANDBOX_CAPABILITY, + ASYNC_JOBS_CAPABILITY, + CACHE_CAPABILITY, + OCR_CAPABILITY, + KNOWLEDGE_EMBEDDINGS_CAPABILITY, +] as const + +export const LLM_KEY_POOLS = { + openai: { + keys: ['OPENAI_API_KEY_1', 'OPENAI_API_KEY_2', 'OPENAI_API_KEY_3'], + fallbackKey: 'OPENAI_API_KEY', + }, + anthropic: { + keys: ['ANTHROPIC_API_KEY_1', 'ANTHROPIC_API_KEY_2', 'ANTHROPIC_API_KEY_3'], + }, + gemini: { + keys: ['GEMINI_API_KEY_1', 'GEMINI_API_KEY_2', 'GEMINI_API_KEY_3'], + fallbackKey: 'GEMINI_API_KEY', + }, + cohere: { + keys: ['COHERE_API_KEY_1', 'COHERE_API_KEY_2', 'COHERE_API_KEY_3'], + fallbackKey: 'COHERE_API_KEY', + }, + zai: { keys: ['ZAI_API_KEY_1', 'ZAI_API_KEY_2', 'ZAI_API_KEY_3'] }, + xai: { keys: ['XAI_API_KEY_1', 'XAI_API_KEY_2', 'XAI_API_KEY_3'] }, + kimi: { keys: ['KIMI_API_KEY_1', 'KIMI_API_KEY_2', 'KIMI_API_KEY_3'] }, + fireworks: { + keys: ['FIREWORKS_API_KEY_1', 'FIREWORKS_API_KEY_2', 'FIREWORKS_API_KEY_3'], + fallbackKey: 'FIREWORKS_API_KEY', + }, +} as const + +/** + * Environment keys whose process-level values can change setup status or make a + * setup write ineffective. The setup CLI uses this exact runtime-owned list to + * avoid claiming it manages a development configuration shadowed by the shell. + */ +export const DEPLOYMENT_CONFIGURATION_KEYS: readonly string[] = [ + ...new Set([ + ...CORE_CONFIGURATION_KEYS, + ...ENV_CAPABILITIES.flatMap(capabilityKeys), + 'EMAIL_VERIFICATION_ENABLED', + 'NEXT_PUBLIC_E2B_ENABLED', + 'NEXT_PUBLIC_SANDBOXES_ENABLED', + ...Object.values(LLM_KEY_POOLS).flatMap((pool) => [ + ...pool.keys, + ...('fallbackKey' in pool ? [pool.fallbackKey] : []), + ]), + ...Object.values(OAUTH_CLIENT_CAPABILITIES).flat(), + ]), +] + +export type OAuthClientCapabilityId = keyof typeof OAUTH_CLIENT_CAPABILITIES +export type OAuthClientCapabilityField = + (typeof OAUTH_CLIENT_CAPABILITIES)[TCapabilityId][number] + +export interface ConfiguredOAuthClient { + state: 'ready' + missingFields: readonly [] + setupCommand: string + values: Readonly> +} + +const GOOGLE_OAUTH_SERVICES = new Set([ + 'gmail', + 'google-email', + 'google-drive', + 'google-docs', + 'google-sheets', + 'google-calendar', + 'google-contacts', + 'google-ads', + 'google-bigquery', + 'google-tasks', + 'google-vault', + 'google-forms', + 'google-groups', + 'google-meet', + 'vertex-ai', +]) + +const MICROSOFT_OAUTH_SERVICES = new Set([ + 'microsoft', + 'outlook', + 'onedrive', + 'sharepoint', + 'microsoft-ad', + 'microsoft-dataverse', + 'microsoft-excel', + 'microsoft-teams', + 'microsoft-planner', +]) + +export function resolveOAuthClientCapabilityId(serviceId: string): OAuthClientCapabilityId | null { + const normalized = serviceId.toLowerCase().replace(/_/g, '-') + if (GOOGLE_OAUTH_SERVICES.has(normalized)) return 'google' + if (MICROSOFT_OAUTH_SERVICES.has(normalized)) return 'microsoft' + if (normalized === 'zoho') return 'zoho-desk' + // One consumer key serves both Salesforce login hosts, so the sandbox provider + // is configured by the same env pair — without this alias it is silently dropped. + if (normalized === 'salesforce-sandbox') return 'salesforce' + return normalized in OAUTH_CLIENT_CAPABILITIES ? (normalized as OAuthClientCapabilityId) : null +} + +export function getOAuthClientCapabilityFields(serviceId: string): readonly string[] | null { + const providerId = resolveOAuthClientCapabilityId(serviceId) + return providerId ? OAUTH_CLIENT_CAPABILITIES[providerId] : null +} + +export interface OAuthClientCapabilityInspection { + state: ProviderConfigurationState + missingFields: readonly string[] + setupCommand: string +} + +function readOAuthClientFieldValue(values: EnvCapabilityValues, key: string): string | null { + const value = readValue(values, key) + if (typeof value !== 'string' || !hasValue(values, key)) return null + return value +} + +export function inspectOAuthClientCapability( + providerId: string, + values: EnvCapabilityValues +): OAuthClientCapabilityInspection { + const capabilityId = resolveOAuthClientCapabilityId(providerId) + const fields = capabilityId ? OAUTH_CLIENT_CAPABILITIES[capabilityId] : null + if (!fields) { + return { + state: 'absent', + missingFields: [], + setupCommand: `npx @sim/setup add integration ${providerId}`, + } + } + + const present = fields.filter((key) => readOAuthClientFieldValue(values, key) !== null) + return { + state: present.length === 0 ? 'absent' : present.length === fields.length ? 'ready' : 'partial', + missingFields: fields.filter((key) => readOAuthClientFieldValue(values, key) === null), + setupCommand: `npx @sim/setup add integration ${providerId}`, + } +} + +export function requireOAuthClientCapability( + providerId: TCapabilityId, + values: EnvCapabilityValues +): ConfiguredOAuthClient> +export function requireOAuthClientCapability( + providerId: string, + values: EnvCapabilityValues +): ConfiguredOAuthClient +export function requireOAuthClientCapability( + providerId: string, + values: EnvCapabilityValues +): ConfiguredOAuthClient { + const inspection = inspectOAuthClientCapability(providerId, values) + if (inspection.state !== 'ready') { + const detail = + inspection.state === 'partial' || inspection.state === 'invalid' + ? ` is partially configured — missing ${inspection.missingFields.join(', ')}` + : ' is not configured' + throw new EnvCapabilityConfigurationError( + 'oauth', + `OAuth client ${providerId}${detail}. Run ${inspection.setupCommand}.` + ) + } + + const fields = getOAuthClientCapabilityFields(providerId) + if (!fields) { + throw new EnvCapabilityConfigurationError( + 'oauth', + `OAuth client ${providerId} has no capability definition. Run ${inspection.setupCommand}.` + ) + } + + const configuredValues: Record = {} + for (const field of fields) { + const value = readOAuthClientFieldValue(values, field) + if (value === null) { + throw new EnvCapabilityConfigurationError( + 'oauth', + `OAuth client ${providerId} has an invalid ${field}. Run ${inspection.setupCommand}.` + ) + } + configuredValues[field] = value + } + + return { + ...inspection, + state: 'ready', + missingFields: [], + values: configuredValues, + } +} diff --git a/packages/deployment-config/src/integration-availability.ts b/packages/deployment-config/src/integration-availability.ts new file mode 100644 index 00000000000..43391bcf897 --- /dev/null +++ b/packages/deployment-config/src/integration-availability.ts @@ -0,0 +1,142 @@ +import type { EnvCapabilityValues } from './env-capabilities' +import { inspectOAuthClientCapability, resolveOAuthClientCapabilityId } from './env-capabilities' +import integrationsJson from './integrations.json' +import { getServiceAccountMetadata } from './service-account-metadata' + +export type IntegrationAvailabilityState = 'ready' | 'limited' | 'unavailable' | 'misconfigured' + +export interface IntegrationAvailability { + type: string + slug: string + name: string + state: IntegrationAvailabilityState + oauthAvailable: boolean + serviceAccountAvailable: boolean + missingFields: readonly string[] + setupCommand?: string +} + +interface DeploymentIntegration { + type: string + slug: string + name: string + authType: 'oauth' | 'api-key' | 'none' + oauthServiceId?: string +} + +const integrations = integrationsJson.integrations as readonly DeploymentIntegration[] +const deploymentGatedIntegrationTypes = new Set( + integrations + .filter((integration) => integration.authType === 'oauth') + .map((integration) => integration.type.toLowerCase()) +) +const integrationTypesByOAuthServiceId = new Map() +const previewServiceAccountProvidersByIntegrationType = new Map() +for (const integration of integrations) { + if (integration.authType !== 'oauth' || !integration.oauthServiceId) continue + const serviceId = integration.oauthServiceId.toLowerCase() + const current = integrationTypesByOAuthServiceId.get(serviceId) ?? [] + const integrationType = integration.type.toLowerCase() + integrationTypesByOAuthServiceId.set(serviceId, [...current, integrationType]) + + const serviceAccount = getServiceAccountMetadata(serviceId) + if (serviceAccount?.deploymentRequirement !== 'preview-gated') continue + previewServiceAccountProvidersByIntegrationType.set(integrationType, serviceAccount.providerId) +} + +export function isDeploymentGatedIntegrationType(blockType: string): boolean { + return deploymentGatedIntegrationTypes.has(blockType.toLowerCase()) +} + +/** Returns the generated integration block types authenticated by one OAuth service entry. */ +export function getIntegrationTypesForOAuthServiceId(serviceId: string): readonly string[] { + return integrationTypesByOAuthServiceId.get(serviceId.toLowerCase()) ?? [] +} + +/** Applies an integration allowlist to an OAuth service without loading executable registries. */ +export function isOAuthServiceAllowedByIntegrationTypes( + serviceId: string, + allowedIntegrationTypes: ReadonlySet | null +): boolean { + if (allowedIntegrationTypes === null) return true + const integrationTypes = getIntegrationTypesForOAuthServiceId(serviceId) + return ( + integrationTypes.length === 0 || + integrationTypes.some((blockType) => allowedIntegrationTypes.has(blockType)) + ) +} + +/** Returns the preview-gated service-account provider for an integration type. */ +export function getPreviewServiceAccountProviderId(integrationType: string): string | undefined { + return previewServiceAccountProvidersByIntegrationType.get(integrationType.toLowerCase()) +} + +function resolveOAuthIntegrationAvailability( + integration: DeploymentIntegration, + values: EnvCapabilityValues +): IntegrationAvailability { + const { oauthServiceId } = integration + if (!oauthServiceId) { + throw new Error(`OAuth integration ${integration.slug} is missing oauthServiceId`) + } + + const capabilityId = resolveOAuthClientCapabilityId(oauthServiceId) + const serviceAccount = getServiceAccountMetadata(oauthServiceId) + + if (!capabilityId) { + throw new Error( + `OAuth integration ${integration.slug} has no OAuth client capability definition` + ) + } + + const oauth = inspectOAuthClientCapability(capabilityId, values) + const setupCommand = `npx @sim/setup add integration ${capabilityId}` + const serviceAccountAvailable = Boolean( + serviceAccount && + serviceAccount.deploymentRequirement !== 'preview-gated' && + (serviceAccount.deploymentRequirement !== 'oauth-client' || oauth.state === 'ready') + ) + const state: IntegrationAvailabilityState = + oauth.state === 'ready' + ? 'ready' + : serviceAccountAvailable + ? 'limited' + : oauth.state === 'partial' || oauth.state === 'invalid' + ? 'misconfigured' + : 'unavailable' + + return { + type: integration.type, + slug: integration.slug, + name: integration.name, + state, + oauthAvailable: oauth.state === 'ready', + serviceAccountAvailable, + missingFields: oauth.missingFields, + setupCommand, + } +} + +/** + * Resolves deployment availability for every integration in the generated + * catalog using only caller-supplied environment values and pure metadata. + */ +export function resolveIntegrationAvailability( + values: EnvCapabilityValues +): readonly IntegrationAvailability[] { + return integrations.map((integration) => { + if (integration.authType === 'oauth') { + return resolveOAuthIntegrationAvailability(integration, values) + } + + return { + type: integration.type, + slug: integration.slug, + name: integration.name, + state: 'ready', + oauthAvailable: false, + serviceAccountAvailable: false, + missingFields: [], + } + }) +} diff --git a/apps/sim/lib/integrations/integrations.json b/packages/deployment-config/src/integrations.json similarity index 100% rename from apps/sim/lib/integrations/integrations.json rename to packages/deployment-config/src/integrations.json diff --git a/packages/deployment-config/src/service-account-metadata.ts b/packages/deployment-config/src/service-account-metadata.ts new file mode 100644 index 00000000000..859aff2475c --- /dev/null +++ b/packages/deployment-config/src/service-account-metadata.ts @@ -0,0 +1,48 @@ +import { SERVICE_ACCOUNT_PROVIDER_BY_OAUTH_SERVICE_ID } from './service-account-providers.generated' + +type DeploymentRequirement = 'preview-gated' | 'oauth-client' +type ServiceAccountOAuthServiceId = keyof typeof SERVICE_ACCOUNT_PROVIDER_BY_OAUTH_SERVICE_ID + +export interface ServiceAccountMetadata { + providerId: string + deploymentRequirement?: DeploymentRequirement +} + +/** Handwritten deployment policy layered over generated OAuth registry facts. */ +const DEPLOYMENT_REQUIREMENT_BY_OAUTH_SERVICE_ID = { + slack: 'preview-gated', + trello: 'oauth-client', +} as const satisfies Partial> + +function getDeploymentRequirement(oauthServiceId: string): DeploymentRequirement | undefined { + if (!Object.hasOwn(DEPLOYMENT_REQUIREMENT_BY_OAUTH_SERVICE_ID, oauthServiceId)) return undefined + return DEPLOYMENT_REQUIREMENT_BY_OAUTH_SERVICE_ID[ + oauthServiceId as keyof typeof DEPLOYMENT_REQUIREMENT_BY_OAUTH_SERVICE_ID + ] +} + +function buildServiceAccountMetadata(): Readonly> { + const metadata: Record = {} + for (const [oauthServiceId, providerId] of Object.entries( + SERVICE_ACCOUNT_PROVIDER_BY_OAUTH_SERVICE_ID + )) { + const deploymentRequirement = getDeploymentRequirement(oauthServiceId) + metadata[oauthServiceId] = { + providerId, + ...(deploymentRequirement ? { deploymentRequirement } : {}), + } + } + return metadata +} + +/** Lightweight deployment metadata safe to consume outside the application graph. */ +export const SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID = buildServiceAccountMetadata() + +export function getServiceAccountMetadata( + oauthServiceId: string +): ServiceAccountMetadata | undefined { + if (!Object.hasOwn(SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID, oauthServiceId)) { + return undefined + } + return SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID[oauthServiceId] +} diff --git a/packages/deployment-config/src/service-account-providers.generated.ts b/packages/deployment-config/src/service-account-providers.generated.ts new file mode 100644 index 00000000000..400db266738 --- /dev/null +++ b/packages/deployment-config/src/service-account-providers.generated.ts @@ -0,0 +1,39 @@ +/** + * Generated by `bun run deployment-config:generate` from the canonical OAuth + * registry and integration catalog. Do not edit this file directly. + */ +export const SERVICE_ACCOUNT_PROVIDER_BY_OAUTH_SERVICE_ID = { + airtable: 'airtable-service-account', + asana: 'asana-service-account', + attio: 'attio-service-account', + box: 'box-service-account', + calcom: 'calcom-service-account', + clickup: 'clickup-service-account', + confluence: 'atlassian-service-account', + gmail: 'google-service-account', + 'google-bigquery': 'google-service-account', + 'google-calendar': 'google-service-account', + 'google-contacts': 'google-service-account', + 'google-docs': 'google-service-account', + 'google-drive': 'google-service-account', + 'google-forms': 'google-service-account', + 'google-groups': 'google-service-account', + 'google-meet': 'google-service-account', + 'google-sheets': 'google-service-account', + 'google-tasks': 'google-service-account', + 'google-vault': 'google-service-account', + hubspot: 'hubspot-service-account', + jira: 'atlassian-service-account', + linear: 'linear-service-account', + monday: 'monday-service-account', + notion: 'notion-service-account', + pipedrive: 'pipedrive-service-account', + salesforce: 'salesforce-service-account', + shopify: 'shopify-service-account', + slack: 'slack-custom-bot', + trello: 'trello-service-account', + wealthbox: 'wealthbox-service-account', + webflow: 'webflow-service-account', + 'zoho-desk': 'zoho-desk-service-account', + zoom: 'zoom-service-account', +} as const diff --git a/packages/deployment-config/tsconfig.json b/packages/deployment-config/tsconfig.json new file mode 100644 index 00000000000..62cb360345a --- /dev/null +++ b/packages/deployment-config/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "@sim/tsconfig/base.json", + "include": ["src/**/*"], + "exclude": ["node_modules"] +} diff --git a/packages/sim-setup/LICENSE b/packages/sim-setup/LICENSE new file mode 100644 index 00000000000..f4e76aaaac1 --- /dev/null +++ b/packages/sim-setup/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 Sim Studio, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/sim-setup/README.md b/packages/sim-setup/README.md new file mode 100644 index 00000000000..5208cb148b5 --- /dev/null +++ b/packages/sim-setup/README.md @@ -0,0 +1,14 @@ +# @sim/setup + +Set up and manage a self-hosted Sim installation. + +```bash +npx @sim/setup +``` + +Outside a Sim source checkout, the command creates a Docker Compose installation using published +images. Inside a Sim source checkout, it exposes the complete development and deployment wizard. + +By default, a standalone installation is written to `./sim`. Use `--dir ` to choose a +different directory. The `sim` npm package remains the Sim API CLI; this package intentionally +publishes only the `sim-setup` binary. diff --git a/packages/sim-setup/THIRD_PARTY_LICENSES b/packages/sim-setup/THIRD_PARTY_LICENSES new file mode 100644 index 00000000000..c6742214e29 --- /dev/null +++ b/packages/sim-setup/THIRD_PARTY_LICENSES @@ -0,0 +1,55 @@ +The @sim/setup bundle includes the following third-party software. + +@clack/core and @clack/prompts +Copyright (c) 2025-Present Bombshell contributors + +@next/env +Copyright (c) 2016-present Vercel, Inc. + +chalk +Copyright (c) Sindre Sorhus (https://sindresorhus.com) + +The dependencies above are licensed under the MIT License: + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +postgres +Copyright (c) Rasmus Porsager (https://www.porsager.com) + +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or distribute this +software, either in source code form or as a compiled binary, for any purpose, +commercial or non-commercial, and by any means. + +In jurisdictions that recognize copyright laws, the author or authors of this +software dedicate any and all copyright interest in the software to the public +domain. We make this dedication for the benefit of the public at large and to +the detriment of our heirs and successors. We intend this dedication to be an +overt act of relinquishment in perpetuity of all present and future rights to +this software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to . diff --git a/packages/sim-setup/package.json b/packages/sim-setup/package.json new file mode 100644 index 00000000000..9c45c689962 --- /dev/null +++ b/packages/sim-setup/package.json @@ -0,0 +1,63 @@ +{ + "name": "@sim/setup", + "version": "1.0.0", + "description": "Set up and manage a self-hosted Sim installation", + "type": "module", + "bin": { + "sim-setup": "dist/index.js" + }, + "scripts": { + "prebuild": "bun run clean", + "build": "bun build src/index.ts --target=node --format=esm --packages=bundle --reject-unresolved --outfile=dist/index.js && bun run build:assets", + "build:assets": "bun run src/build-assets.ts", + "clean": "bun -e \"import { rmSync } from 'node:fs'; rmSync('dist', { recursive: true, force: true })\"", + "type-check": "tsc --noEmit", + "lint": "biome check --write --unsafe .", + "lint:check": "biome check .", + "format": "biome format --write .", + "format:check": "biome format .", + "test": "vitest run", + "prepublishOnly": "bun run build" + }, + "files": [ + "dist", + "THIRD_PARTY_LICENSES" + ], + "keywords": [ + "sim", + "self-hosted", + "docker", + "compose", + "setup" + ], + "author": "Sim", + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/simstudioai/sim.git", + "directory": "packages/sim-setup" + }, + "homepage": "https://github.com/simstudioai/sim/tree/main/packages/sim-setup#readme", + "bugs": { + "url": "https://github.com/simstudioai/sim/issues" + }, + "publishConfig": { + "access": "public" + }, + "engines": { + "node": ">=20" + }, + "devDependencies": { + "@clack/prompts": "1.7.0", + "@next/env": "16.2.12", + "@sim/deployment-config": "workspace:*", + "@sim/security": "workspace:*", + "@sim/tsconfig": "workspace:*", + "@sim/utils": "workspace:*", + "@types/node": "24.2.1", + "chalk": "5.6.2", + "postgres": "^3.4.5", + "typescript": "^7.0.2", + "vitest": "^4.1.0" + } +} diff --git a/scripts/setup/banner.ts b/packages/sim-setup/src/banner.ts similarity index 95% rename from scripts/setup/banner.ts rename to packages/sim-setup/src/banner.ts index cbce65e80ff..ff2ecf316a1 100644 --- a/scripts/setup/banner.ts +++ b/packages/sim-setup/src/banner.ts @@ -1,7 +1,7 @@ import { sleep } from '@sim/utils/helpers' import chalk from 'chalk' -import { restoreTerminal } from './terminal.ts' -import { isRich, theme } from './theme.ts' +import { restoreTerminal } from './terminal' +import { isRich, theme } from './theme' const WORDMARK = [' ▀ ', '▄▀▀▀ █ █▀█▀█', '▀▀▀▄ █ █ █ █', '▄▄▄▀ █ █ █ █'] as const const TAGLINE = 'the AI workspace' diff --git a/packages/sim-setup/src/build-assets.ts b/packages/sim-setup/src/build-assets.ts new file mode 100644 index 00000000000..50722e78125 --- /dev/null +++ b/packages/sim-setup/src/build-assets.ts @@ -0,0 +1,18 @@ +import { chmodSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') +const repositoryRoot = path.resolve(packageRoot, '../..') +const dist = path.join(packageRoot, 'dist') + +mkdirSync(dist, { recursive: true }) +const compose = readFileSync(path.join(repositoryRoot, 'docker-compose.prod.yml'), 'utf8') +if (/^\s+(?:build|context):/m.test(compose)) { + throw new Error('docker-compose.prod.yml must not contain local build dependencies') +} +if (!compose.includes('ghcr.io/simstudioai/simstudio')) { + throw new Error('docker-compose.prod.yml is missing the published Sim image') +} +writeFileSync(path.join(dist, 'docker-compose.prod.yml'), compose) +chmodSync(path.join(dist, 'index.js'), 0o755) diff --git a/scripts/setup/capability-config.test.ts b/packages/sim-setup/src/capability-config.test.ts similarity index 92% rename from scripts/setup/capability-config.test.ts rename to packages/sim-setup/src/capability-config.test.ts index de03288280e..eaf2d4920b4 100644 --- a/scripts/setup/capability-config.test.ts +++ b/packages/sim-setup/src/capability-config.test.ts @@ -1,10 +1,10 @@ -import { describe, expect, it } from 'bun:test' import { defineCapability, ENV_CAPABILITIES, envField, OAUTH_CLIENT_CAPABILITIES, -} from '../../apps/sim/lib/core/config/env-capabilities.ts' +} from '@sim/deployment-config/env-capabilities' +import { describe, expect, it } from 'vitest' import { CAPABILITY_SETUPS, defineCapabilitySetup, @@ -12,8 +12,8 @@ import { getOAuthClientSetupFields, KNOWLEDGE_EMBEDDINGS_SETUP, STORAGE_SETUP, -} from './capability-config.ts' -import { getCapabilitySetupOptions } from './capability-setup.ts' +} from './capability-config' +import { getCapabilitySetupOptions } from './capability-setup' describe('capability setup configuration', () => { it('maps every runtime capability and provider exactly once', () => { diff --git a/scripts/setup/capability-config.ts b/packages/sim-setup/src/capability-config.ts similarity index 99% rename from scripts/setup/capability-config.ts rename to packages/sim-setup/src/capability-config.ts index 486d15d6be7..b484f36306e 100644 --- a/scripts/setup/capability-config.ts +++ b/packages/sim-setup/src/capability-config.ts @@ -20,7 +20,7 @@ import { OCR_CAPABILITY, SANDBOX_CAPABILITY, STORAGE_CAPABILITY, -} from '../../apps/sim/lib/core/config/env-capabilities.ts' +} from '@sim/deployment-config/env-capabilities' export type SetupHint = | string @@ -863,7 +863,7 @@ export function getCapabilitySetup(id: string): CapabilitySetupDefinition | null } export function getSetupCommand(id: string): string { - return `bun run setup ${id}` + return `npx @sim/setup add ${id}` } type OAuthClientSetupFields = { diff --git a/scripts/setup/capability-setup.test.ts b/packages/sim-setup/src/capability-setup.test.ts similarity index 86% rename from scripts/setup/capability-setup.test.ts rename to packages/sim-setup/src/capability-setup.test.ts index 4b4f8c857a8..cf94c5183bf 100644 --- a/scripts/setup/capability-setup.test.ts +++ b/packages/sim-setup/src/capability-setup.test.ts @@ -1,6 +1,6 @@ -import { describe, expect, it } from 'bun:test' -import type { SetupFieldPrompt } from './capability-config.ts' -import { formatCapabilitySetupFieldMessage, markCurrentlyUsed } from './capability-setup.ts' +import { describe, expect, it } from 'vitest' +import type { SetupFieldPrompt } from './capability-config' +import { formatCapabilitySetupFieldMessage, markCurrentlyUsed } from './capability-setup' describe('capability setup presentation', () => { it('marks the effective option as currently used without replacing its hint', () => { diff --git a/scripts/setup/capability-setup.ts b/packages/sim-setup/src/capability-setup.ts similarity index 99% rename from scripts/setup/capability-setup.ts rename to packages/sim-setup/src/capability-setup.ts index 413f9d7f914..98119261ea5 100644 --- a/scripts/setup/capability-setup.ts +++ b/packages/sim-setup/src/capability-setup.ts @@ -12,15 +12,15 @@ import { inspectCapability, isTruthyEnvCapabilityValue, validateCapabilityFieldInput, -} from '../../apps/sim/lib/core/config/env-capabilities.ts' +} from '@sim/deployment-config/env-capabilities' import { type CapabilitySetupDefinition, matchesSetupCondition, type SetupCondition, type SetupHint, type SetupPrompt, -} from './capability-config.ts' -import * as p from './prompter.ts' +} from './capability-config' +import * as p from './prompter' export interface CapabilitySetupContext { containerized: boolean diff --git a/scripts/setup/capability-status.test.ts b/packages/sim-setup/src/capability-status.test.ts similarity index 97% rename from scripts/setup/capability-status.test.ts rename to packages/sim-setup/src/capability-status.test.ts index ca712316c63..f7966a7cac3 100644 --- a/scripts/setup/capability-status.test.ts +++ b/packages/sim-setup/src/capability-status.test.ts @@ -1,6 +1,6 @@ -import { describe, expect, it } from 'bun:test' -import { OAUTH_CLIENT_CAPABILITIES } from '../../apps/sim/lib/core/config/env-capabilities.ts' -import { buildEnvCapabilityStatus } from './capability-status.ts' +import { OAUTH_CLIENT_CAPABILITIES } from '@sim/deployment-config/env-capabilities' +import { describe, expect, it } from 'vitest' +import { buildEnvCapabilityStatus } from './capability-status' const DAYTONA_FUNCTION_SNAPSHOT_ID = '00000000-0000-4000-8000-000000000002' @@ -38,7 +38,7 @@ describe('env capability status', () => { expect(status.features.sandbox).toEqual({ id: 'sandbox', label: 'Function sandboxes', - setupCommand: 'bun run setup sandbox', + setupCommand: 'npx @sim/setup add sandbox', state: 'default', providerId: 'disabled', }) diff --git a/scripts/setup/capability-status.ts b/packages/sim-setup/src/capability-status.ts similarity index 99% rename from scripts/setup/capability-status.ts rename to packages/sim-setup/src/capability-status.ts index f2603c6ab61..89ad6d3ed37 100644 --- a/scripts/setup/capability-status.ts +++ b/packages/sim-setup/src/capability-status.ts @@ -24,8 +24,8 @@ import { type ProviderInspection, SANDBOX_CAPABILITY, STORAGE_CAPABILITY, -} from '../../apps/sim/lib/core/config/env-capabilities.ts' -import { SETUP_FEATURES, type SetupFeatureId } from './capability-config.ts' +} from '@sim/deployment-config/env-capabilities' +import { SETUP_FEATURES, type SetupFeatureId } from './capability-config' export type SetupStatusFeatureId = Exclude export type CapabilityStatusState = 'default' | 'configured' | 'missing' | 'partial' | 'invalid' @@ -38,7 +38,7 @@ export interface CapabilityStatusIssue { interface FeatureStatusBase { id: TId label: string - setupCommand: `bun run setup ${TId}` + setupCommand: `npx @sim/setup add ${TId}` state: CapabilityStatusState issue?: CapabilityStatusIssue } @@ -143,7 +143,7 @@ function featureMetadata(id: TId) { return { id, label: definition.label, - setupCommand: `bun run setup ${id}` as const, + setupCommand: `npx @sim/setup add ${id}` as const, } } diff --git a/scripts/setup/checks.ts b/packages/sim-setup/src/checks.ts similarity index 97% rename from scripts/setup/checks.ts rename to packages/sim-setup/src/checks.ts index 1183f85ad8d..a047f30a0cc 100644 --- a/scripts/setup/checks.ts +++ b/packages/sim-setup/src/checks.ts @@ -11,9 +11,9 @@ import { requireCapability, SANDBOX_CAPABILITY, STORAGE_CAPABILITY, -} from '../../apps/sim/lib/core/config/env-capabilities.ts' -import { getSetupCommand } from './capability-config.ts' -import { portOpen } from './detect.ts' +} from '@sim/deployment-config/env-capabilities' +import { getSetupCommand } from './capability-config' +import { portOpen } from './detect' import { type EnvFile, type EnvTarget, @@ -26,9 +26,9 @@ import { SHARED_KEYS, secretRequirement, writeEnvValues, -} from './env-files.ts' -import { httpHealth, pgProbe, redisPing } from './probes.ts' -import { FLAG_TWINS, LOGIN_PROVIDERS } from './twins.ts' +} from './env-files' +import { httpHealth, pgProbe, redisPing } from './probes' +import { FLAG_TWINS, LOGIN_PROVIDERS } from './twins' export type CheckGroup = 'files' | 'schema' | 'consistency' | 'coherence' | 'live' export type CheckStatus = 'pass' | 'warn' | 'fail' | 'skip' @@ -113,7 +113,7 @@ function checkFiles(ctx: CheckContext): Finding[] { group: 'files', status: 'fail', message: 'no env files found', - fix: 'run: bun run setup', + fix: 'run: npx @sim/setup', }, ] } @@ -135,7 +135,7 @@ function checkFiles(ctx: CheckContext): Finding[] { message: `${rel(file)} is missing`, fix: canSeed ? `run doctor --fix to seed it from apps/${target === 'db' ? '../packages/db' : target}/.env.example + apps/sim/.env` - : 'run: bun run setup', + : 'run: npx @sim/setup', autofix: canSeed ? () => { const keys = target === 'db' ? ['DATABASE_URL'] : [...SHARED_KEYS] @@ -559,7 +559,7 @@ async function checkDatabase(sim: EnvFile): Promise { group: 'live', status: 'fail', message: `database unreachable: ${probe.error}`, - fix: 'start Postgres (bun run setup can manage a pgvector container) or fix DATABASE_URL', + fix: 'start Postgres (npx @sim/setup can manage a pgvector container) or fix DATABASE_URL', }) } else { findings.push({ @@ -577,7 +577,7 @@ async function checkDatabase(sim: EnvFile): Promise { } const { applied, journal } = probe.migrations ?? { applied: null, - journal: 0, + journal: null, } if (applied === null) { findings.push({ @@ -586,7 +586,7 @@ async function checkDatabase(sim: EnvFile): Promise { message: 'migrations have never run on this database', fix: 'cd packages/db && bun run db:migrate', }) - } else if (applied < journal) { + } else if (journal !== null && applied < journal) { findings.push({ group: 'live', status: 'warn', @@ -597,7 +597,10 @@ async function checkDatabase(sim: EnvFile): Promise { findings.push({ group: 'live', status: 'pass', - message: `migrations up to date (${applied})`, + message: + journal === null + ? `database has ${applied} migrations applied` + : `migrations up to date (${applied})`, }) } } diff --git a/scripts/setup/cli-auth.ts b/packages/sim-setup/src/cli-auth.ts similarity index 98% rename from scripts/setup/cli-auth.ts rename to packages/sim-setup/src/cli-auth.ts index c9f3dfec9ee..9ec7700a230 100644 --- a/scripts/setup/cli-auth.ts +++ b/packages/sim-setup/src/cli-auth.ts @@ -4,8 +4,8 @@ import { generateSecureToken } from '@sim/security/tokens' import { sleep } from '@sim/utils/helpers' import { generateShortId } from '@sim/utils/id' import { parseRetryAfter } from '@sim/utils/retry' -import * as p from './prompter.ts' -import { link, theme } from './theme.ts' +import * as p from './prompter' +import { link, theme } from './theme' // Generous enough for a first-time user to create an account, wait for the email // OTP, land back on /cli/auth, and approve — a few minutes is routine. The diff --git a/packages/sim-setup/src/compose-asset.test.ts b/packages/sim-setup/src/compose-asset.test.ts new file mode 100644 index 00000000000..8251ddd1dc1 --- /dev/null +++ b/packages/sim-setup/src/compose-asset.test.ts @@ -0,0 +1,73 @@ +import { createHash } from 'node:crypto' +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { ensureProductionComposeFile } from './compose-asset' + +const roots: string[] = [] + +function tempRoot(): string { + const root = mkdtempSync(path.join(tmpdir(), 'sim-setup-compose-')) + roots.push(root) + return root +} + +function standaloneContext(root: string) { + return { kind: 'standalone', root, existing: false } as const +} + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }) +}) + +describe('ensureProductionComposeFile', () => { + it('materializes the packaged production Compose file and managed state', () => { + const root = tempRoot() + + const composeFile = ensureProductionComposeFile(standaloneContext(root)) + + expect(readFileSync(composeFile, 'utf8')).toContain('ghcr.io/simstudioai/simstudio') + expect(JSON.parse(readFileSync(path.join(root, '.sim-setup.json'), 'utf8'))).toMatchObject({ + schemaVersion: 1, + composeSha256: expect.stringMatching(/^[0-9a-f]{64}$/), + }) + }) + + it('updates a previously managed Compose file', () => { + const root = tempRoot() + const composeFile = ensureProductionComposeFile(standaloneContext(root)) + const previous = `${readFileSync(composeFile, 'utf8')}\nservices: {}\n` + writeFileSync(composeFile, previous) + writeFileSync( + path.join(root, '.sim-setup.json'), + JSON.stringify({ + schemaVersion: 1, + composeSha256: createHash('sha256').update(previous).digest('hex'), + }) + ) + + ensureProductionComposeFile({ kind: 'standalone', root, existing: true }) + + expect(readFileSync(composeFile, 'utf8')).not.toBe(previous) + }) + + it('fails instead of overwriting local Compose changes', () => { + const root = tempRoot() + const composeFile = ensureProductionComposeFile(standaloneContext(root)) + writeFileSync(composeFile, `${readFileSync(composeFile, 'utf8')}\n# local change\n`) + + expect(() => ensureProductionComposeFile({ kind: 'standalone', root, existing: true })).toThrow( + 'has local changes' + ) + }) + + it('fails on an unrelated Compose file', () => { + const root = tempRoot() + writeFileSync(path.join(root, 'docker-compose.prod.yml'), 'services: {}\n') + + expect(() => ensureProductionComposeFile(standaloneContext(root))).toThrow( + 'is not a recognized Sim Compose file' + ) + }) +}) diff --git a/packages/sim-setup/src/compose-asset.ts b/packages/sim-setup/src/compose-asset.ts new file mode 100644 index 00000000000..5d99a5f1744 --- /dev/null +++ b/packages/sim-setup/src/compose-asset.ts @@ -0,0 +1,84 @@ +import { createHash } from 'node:crypto' +import { copyFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import { SETUP_CONTEXT, type SetupContext } from './context' + +const COMPOSE_FILE = 'docker-compose.prod.yml' +const SIM_COMPOSE_MARKER = 'ghcr.io/simstudioai/simstudio' +const MANAGED_STATE_FILE = '.sim-setup.json' + +interface ManagedState { + schemaVersion: 1 + composeSha256: string +} + +function sha256(contents: string): string { + return createHash('sha256').update(contents).digest('hex') +} + +function readManagedState(root: string): ManagedState | null { + const file = path.join(root, MANAGED_STATE_FILE) + if (!existsSync(file)) return null + const parsed: unknown = JSON.parse(readFileSync(file, 'utf8')) + if ( + typeof parsed !== 'object' || + parsed === null || + !('schemaVersion' in parsed) || + parsed.schemaVersion !== 1 || + !('composeSha256' in parsed) || + typeof parsed.composeSha256 !== 'string' + ) { + throw new Error(`${file} is not a valid @sim/setup managed-state file`) + } + return { schemaVersion: 1, composeSha256: parsed.composeSha256 } +} + +function writeManagedState(root: string, contents: string): void { + const state: ManagedState = { schemaVersion: 1, composeSha256: sha256(contents) } + writeFileSync(path.join(root, MANAGED_STATE_FILE), `${JSON.stringify(state, null, 2)}\n`) +} + +function packagedComposeFile(): string { + const moduleDirectory = path.dirname(fileURLToPath(import.meta.url)) + return path.basename(moduleDirectory) === 'src' + ? path.resolve(moduleDirectory, '../../..', COMPOSE_FILE) + : path.join(moduleDirectory, COMPOSE_FILE) +} + +/** Materializes the published Compose asset without overwriting user-customized files. */ +export function ensureProductionComposeFile(context: SetupContext = SETUP_CONTEXT): string { + const destination = path.join(context.root, COMPOSE_FILE) + if (context.kind === 'source') { + if (!existsSync(destination)) { + throw new Error(`Sim source checkout is missing ${COMPOSE_FILE}`) + } + return destination + } + + const bundled = packagedComposeFile() + if (!existsSync(bundled)) { + throw new Error(`The @sim/setup package is missing its bundled ${COMPOSE_FILE}`) + } + mkdirSync(context.root, { recursive: true }) + const packaged = readFileSync(bundled, 'utf8') + if (existsSync(destination)) { + const current = readFileSync(destination, 'utf8') + if (current === packaged) { + writeManagedState(context.root, packaged) + return destination + } + if (!current.includes(SIM_COMPOSE_MARKER)) { + throw new Error(`${destination} exists but is not a recognized Sim Compose file`) + } + const managed = readManagedState(context.root) + if (!managed || managed.composeSha256 !== sha256(current)) { + throw new Error( + `${destination} has local changes; preserve or remove your customizations before updating it.` + ) + } + } + copyFileSync(bundled, destination) + writeManagedState(context.root, packaged) + return destination +} diff --git a/scripts/setup/configuration-sources.test.ts b/packages/sim-setup/src/configuration-sources.test.ts similarity index 99% rename from scripts/setup/configuration-sources.test.ts rename to packages/sim-setup/src/configuration-sources.test.ts index a30f47d55e6..5c2e4ce3421 100644 --- a/scripts/setup/configuration-sources.test.ts +++ b/packages/sim-setup/src/configuration-sources.test.ts @@ -1,14 +1,14 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import path from 'node:path' -import { afterEach, describe, expect, it } from 'bun:test' +import { afterEach, describe, expect, it } from 'vitest' import { type ConfigurationCommandRunner, type ConfigurationSourceDiscoveryOptions, discoverConfigurationSources as discoverConfigurationSourcesFromEnvironment, parseComposeFileEnvironment, resolveKubernetesContainerEnvironment, -} from './configuration-sources.ts' +} from './configuration-sources' const temporaryDirectories: string[] = [] diff --git a/scripts/setup/configuration-sources.ts b/packages/sim-setup/src/configuration-sources.ts similarity index 99% rename from scripts/setup/configuration-sources.ts rename to packages/sim-setup/src/configuration-sources.ts index 296ab2f4217..34fd8047074 100644 --- a/scripts/setup/configuration-sources.ts +++ b/packages/sim-setup/src/configuration-sources.ts @@ -5,8 +5,8 @@ import { loadEnvConfig } from '@next/env' import { CORE_CONFIGURATION_KEYS, DEPLOYMENT_CONFIGURATION_KEYS, -} from '../../apps/sim/lib/core/config/env-capabilities.ts' -import { isPlaceholder, parseEnv, ROOT, SHARED_KEYS } from './env-files.ts' +} from '@sim/deployment-config/env-capabilities' +import { isPlaceholder, parseEnv, ROOT, SHARED_KEYS } from './env-files' export type ConfigurationSourceKind = 'dev' | 'compose' | 'helm' diff --git a/packages/sim-setup/src/context.test.ts b/packages/sim-setup/src/context.test.ts new file mode 100644 index 00000000000..b5bf0e1262e --- /dev/null +++ b/packages/sim-setup/src/context.test.ts @@ -0,0 +1,96 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { resolveSetupContext } from './context' + +const roots: string[] = [] + +function tempRoot(): string { + const root = mkdtempSync(path.join(tmpdir(), 'sim-setup-context-')) + roots.push(root) + return root +} + +function writePackage(root: string, relativePath: string, name: string): void { + const file = path.join(root, relativePath) + mkdirSync(path.dirname(file), { recursive: true }) + writeFileSync(file, JSON.stringify({ name })) +} + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }) +}) + +describe('resolveSetupContext', () => { + it('finds a Sim checkout from a nested directory', () => { + const root = tempRoot() + writePackage(root, 'package.json', 'simstudio') + writePackage(root, 'apps/sim/package.json', '@sim/app') + writePackage(root, 'apps/realtime/package.json', '@sim/realtime') + writePackage(root, 'packages/db/package.json', '@sim/db') + const nested = path.join(root, 'apps/sim/lib') + mkdirSync(nested, { recursive: true }) + + expect(resolveSetupContext(nested, [])).toEqual({ kind: 'source', root }) + }) + + it('finds an existing standalone installation', () => { + const root = tempRoot() + writeFileSync( + path.join(root, 'docker-compose.prod.yml'), + 'image: ghcr.io/simstudioai/simstudio:latest\n' + ) + + expect(resolveSetupContext(root, [])).toEqual({ kind: 'standalone', root, existing: true }) + }) + + it('creates a dedicated child directory outside an installation', () => { + const root = tempRoot() + + expect(resolveSetupContext(root, [])).toEqual({ + kind: 'standalone', + root: path.join(root, 'sim'), + existing: false, + }) + }) + + it('fails on a partial Sim checkout', () => { + const root = tempRoot() + writePackage(root, 'package.json', 'simstudio') + writePackage(root, 'apps/sim/package.json', '@sim/app') + + expect(() => resolveSetupContext(root, [])).toThrow('Incomplete Sim source checkout') + }) + + it('fails on malformed package metadata instead of ignoring it', () => { + const root = tempRoot() + writeFileSync(path.join(root, 'package.json'), '{') + + expect(() => resolveSetupContext(root, [])).toThrow(SyntaxError) + }) + + it('honors an explicit installation directory', () => { + const root = tempRoot() + + expect(resolveSetupContext(root, ['--dir', 'custom'])).toEqual({ + kind: 'standalone', + root: path.join(root, 'custom'), + existing: false, + }) + }) + + it('does not replace an explicit standalone directory with an ancestor checkout', () => { + const root = tempRoot() + writePackage(root, 'package.json', 'simstudio') + writePackage(root, 'apps/sim/package.json', '@sim/app') + writePackage(root, 'apps/realtime/package.json', '@sim/realtime') + writePackage(root, 'packages/db/package.json', '@sim/db') + + expect(resolveSetupContext(root, ['--dir', 'deployment'])).toEqual({ + kind: 'standalone', + root: path.join(root, 'deployment'), + existing: false, + }) + }) +}) diff --git a/packages/sim-setup/src/context.ts b/packages/sim-setup/src/context.ts new file mode 100644 index 00000000000..2f178636926 --- /dev/null +++ b/packages/sim-setup/src/context.ts @@ -0,0 +1,112 @@ +import { existsSync, readFileSync } from 'node:fs' +import path from 'node:path' + +const SOURCE_PACKAGE_MARKERS = [ + ['apps/sim/package.json', '@sim/app'], + ['apps/realtime/package.json', '@sim/realtime'], + ['packages/db/package.json', '@sim/db'], +] as const +const SIM_COMPOSE_MARKER = 'ghcr.io/simstudioai/simstudio' +const COMPOSE_FILE = 'docker-compose.prod.yml' + +export type SetupContext = + | { kind: 'source'; root: string } + | { kind: 'standalone'; root: string; existing: boolean } + +function readPackageName(file: string): string | null { + if (!existsSync(file)) return null + const parsed: unknown = JSON.parse(readFileSync(file, 'utf8')) + if (typeof parsed !== 'object' || parsed === null || !('name' in parsed)) return null + return typeof parsed.name === 'string' ? parsed.name : null +} + +function parentDirectories(start: string): string[] { + const directories: string[] = [] + let current = path.resolve(start) + while (true) { + directories.push(current) + const parent = path.dirname(current) + if (parent === current) return directories + current = parent + } +} + +function inspectSourceRoot(candidate: string): 'valid' | 'partial' | 'absent' { + const rootName = readPackageName(path.join(candidate, 'package.json')) + const markerNames = SOURCE_PACKAGE_MARKERS.map(([file]) => + readPackageName(path.join(candidate, file)) + ) + const looksLikeSource = rootName === 'simstudio' || markerNames.some((name) => name !== null) + if (!looksLikeSource) return 'absent' + if ( + rootName === 'simstudio' && + SOURCE_PACKAGE_MARKERS.every(([, expected], index) => markerNames[index] === expected) + ) { + return 'valid' + } + return 'partial' +} + +function isStandaloneInstall(candidate: string): boolean { + const composeFile = path.join(candidate, COMPOSE_FILE) + if (!existsSync(composeFile)) return false + return readFileSync(composeFile, 'utf8').includes(SIM_COMPOSE_MARKER) +} + +function directoryOverride(args: readonly string[]): string | null { + const equalsArg = args.find((arg) => arg.startsWith('--dir=')) + if (equalsArg) { + const value = equalsArg.slice('--dir='.length) + if (!value) throw new Error('--dir requires a directory path') + return value + } + const index = args.indexOf('--dir') + if (index === -1) return null + const value = args[index + 1] + if (!value || value.startsWith('-')) throw new Error('--dir requires a directory path') + return value +} + +/** Resolves the filesystem context before setup reads or writes any installation state. */ +export function resolveSetupContext( + start: string = process.cwd(), + args: readonly string[] = process.argv.slice(2) +): SetupContext { + const override = directoryOverride(args) + const searchStart = path.resolve(start, override ?? '.') + + if (override) { + const source = inspectSourceRoot(searchStart) + if (source === 'valid') return { kind: 'source', root: searchStart } + if (source === 'partial') { + throw new Error( + `Incomplete Sim source checkout at ${searchStart}; expected package.json plus apps/sim, apps/realtime, and packages/db package manifests.` + ) + } + return { + kind: 'standalone', + root: searchStart, + existing: isStandaloneInstall(searchStart), + } + } + + for (const candidate of parentDirectories(searchStart)) { + const source = inspectSourceRoot(candidate) + if (source === 'valid') return { kind: 'source', root: candidate } + if (source === 'partial') { + throw new Error( + `Incomplete Sim source checkout at ${candidate}; expected package.json plus apps/sim, apps/realtime, and packages/db package manifests.` + ) + } + } + + for (const candidate of parentDirectories(searchStart)) { + if (isStandaloneInstall(candidate)) { + return { kind: 'standalone', root: candidate, existing: true } + } + } + + return { kind: 'standalone', root: path.join(path.resolve(start), 'sim'), existing: false } +} + +export const SETUP_CONTEXT = resolveSetupContext() diff --git a/scripts/setup/db.ts b/packages/sim-setup/src/db.ts similarity index 97% rename from scripts/setup/db.ts rename to packages/sim-setup/src/db.ts index 5794618a9c5..2eadc6b6bb3 100644 --- a/scripts/setup/db.ts +++ b/packages/sim-setup/src/db.ts @@ -1,11 +1,11 @@ import { spawnSync } from 'node:child_process' -import { DB_CONTAINER, type Detection } from './detect.ts' -import { ensureDocker } from './docker.ts' -import { generateSecret } from './env-files.ts' -import { SetupError } from './errors.ts' -import { pgProbe, waitFor } from './probes.ts' -import * as p from './prompter.ts' -import { glyph, theme } from './theme.ts' +import { DB_CONTAINER, type Detection } from './detect' +import { ensureDocker } from './docker' +import { generateSecret } from './env-files' +import { SetupError } from './errors' +import { pgProbe, waitFor } from './probes' +import * as p from './prompter' +import { glyph, theme } from './theme' const DEFAULT_DSN = 'postgresql://postgres:postgres@localhost:5432/simstudio' diff --git a/scripts/setup/detect.ts b/packages/sim-setup/src/detect.ts similarity index 95% rename from scripts/setup/detect.ts rename to packages/sim-setup/src/detect.ts index aebfabf0d9d..6df723367bb 100644 --- a/scripts/setup/detect.ts +++ b/packages/sim-setup/src/detect.ts @@ -1,7 +1,8 @@ import { spawnSync } from 'node:child_process' import net from 'node:net' import os from 'node:os' -import { ROOT, readEnvFile } from './env-files.ts' +import { ROOT, readEnvFile } from './env-files' +import { executableExists } from './executables' export const MANAGED_LABEL = 'managed-by=sim-setup' export const DB_CONTAINER = 'sim-postgres' @@ -146,9 +147,9 @@ export async function runDetection(): Promise { shellLlmKeys: SHELL_LLM_KEYS.filter((key) => process.env[key]), ollamaReachable: ollamaPortOpen ? await ollamaReachable() : false, binaries: { - kubectl: Bun.which('kubectl') !== null, - helm: Bun.which('helm') !== null, - kind: Bun.which('kind') !== null, + kubectl: executableExists('kubectl'), + helm: executableExists('helm'), + kind: executableExists('kind'), }, kubeContext: commandOutput('kubectl', ['config', 'current-context']), specs: detectSpecs(dockerRunning), diff --git a/scripts/setup/docker.ts b/packages/sim-setup/src/docker.ts similarity index 95% rename from scripts/setup/docker.ts rename to packages/sim-setup/src/docker.ts index f116ff8b102..1e3d95dd0ec 100644 --- a/scripts/setup/docker.ts +++ b/packages/sim-setup/src/docker.ts @@ -2,10 +2,11 @@ import { spawnSync } from 'node:child_process' import { existsSync } from 'node:fs' import { homedir } from 'node:os' import { join } from 'node:path' -import { SetupError } from './errors.ts' -import { waitFor } from './probes.ts' -import * as p from './prompter.ts' -import { glyph, theme } from './theme.ts' +import { SetupError } from './errors' +import { executableExists } from './executables' +import { waitFor } from './probes' +import * as p from './prompter' +import { glyph, theme } from './theme' const INSTALL_HINTS = [ 'install Docker Desktop: https://docker.com/products/docker-desktop', @@ -35,9 +36,8 @@ function daemonUp(): boolean { return spawnSync('docker', ['info'], { stdio: 'ignore' }).status === 0 } -/** Uses `Bun.which` rather than `which`, which is not a standard Windows command. */ function installed(): boolean { - return Bun.which('docker') !== null + return executableExists('docker') } /** diff --git a/scripts/setup/doctor.ts b/packages/sim-setup/src/doctor.ts similarity index 97% rename from scripts/setup/doctor.ts rename to packages/sim-setup/src/doctor.ts index 9db5be1c087..94e628716b9 100644 --- a/scripts/setup/doctor.ts +++ b/packages/sim-setup/src/doctor.ts @@ -1,5 +1,5 @@ -import { type CheckGroup, type Finding, loadCheckContext, runChecks } from './checks.ts' -import { glyph, theme } from './theme.ts' +import { type CheckGroup, type Finding, loadCheckContext, runChecks } from './checks' +import { glyph, theme } from './theme' const GROUP_TITLES: Record = { files: 'Env files', diff --git a/scripts/setup/env-files.test.ts b/packages/sim-setup/src/env-files.test.ts similarity index 96% rename from scripts/setup/env-files.test.ts rename to packages/sim-setup/src/env-files.test.ts index c4359deeb6c..46888820a4c 100644 --- a/scripts/setup/env-files.test.ts +++ b/packages/sim-setup/src/env-files.test.ts @@ -1,11 +1,11 @@ -import { describe, expect, it } from 'bun:test' +import { describe, expect, it } from 'vitest' import { isPlaceholder, isUsableSecret, parseEnv, reconcileEnvContent, upsertEnv, -} from './env-files.ts' +} from './env-files' describe('placeholder detection', () => { it('recognizes underscore and hyphen template prefixes', () => { diff --git a/scripts/setup/env-files.ts b/packages/sim-setup/src/env-files.ts similarity index 94% rename from scripts/setup/env-files.ts rename to packages/sim-setup/src/env-files.ts index 726c98b63ae..3dbdd4b058e 100644 --- a/scripts/setup/env-files.ts +++ b/packages/sim-setup/src/env-files.ts @@ -1,9 +1,9 @@ -import { existsSync, readFileSync, renameSync, writeFileSync } from 'node:fs' +import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs' import path from 'node:path' -import { fileURLToPath } from 'node:url' import { generateRandomHex } from '@sim/utils/random' +import { SETUP_CONTEXT } from './context' -export const ROOT = path.resolve(fileURLToPath(new URL('.', import.meta.url)), '../..') +export const ROOT = SETUP_CONTEXT.root export type EnvTarget = 'sim' | 'realtime' | 'db' | 'root' @@ -147,6 +147,7 @@ export function reconcileEnvValues( const example = EXAMPLE_PATHS[target] content = example && existsSync(example) ? readFileSync(example, 'utf8') : '' } + mkdirSync(path.dirname(filePath), { recursive: true }) writeFileSync(filePath, reconcileEnvContent(content, remove, values)) } @@ -156,7 +157,11 @@ export function writeEnvValues(target: EnvTarget, values: Record } export function archiveEnvFile(target: EnvTarget): string | null { - const filePath = ENV_PATHS[target] + return archiveFile(ENV_PATHS[target]) +} + +/** Archives an existing file next to itself and returns the backup path. */ +export function archiveFile(filePath: string): string | null { if (!existsSync(filePath)) return null const backup = `${filePath}.bak-${new Date().toISOString().replace(/[:.]/g, '-')}` renameSync(filePath, backup) diff --git a/scripts/setup/errors.ts b/packages/sim-setup/src/errors.ts similarity index 100% rename from scripts/setup/errors.ts rename to packages/sim-setup/src/errors.ts diff --git a/packages/sim-setup/src/executables.ts b/packages/sim-setup/src/executables.ts new file mode 100644 index 00000000000..cc0cff91f02 --- /dev/null +++ b/packages/sim-setup/src/executables.ts @@ -0,0 +1,8 @@ +import { spawnSync } from 'node:child_process' + +/** Checks executable resolution without depending on a shell or Bun runtime. */ +export function executableExists(command: string): boolean { + const result = spawnSync(command, ['--version'], { stdio: 'ignore' }) + if (!result.error) return true + return (result.error as NodeJS.ErrnoException).code !== 'ENOENT' +} diff --git a/scripts/setup/feature-setup.test.ts b/packages/sim-setup/src/feature-setup.test.ts similarity index 95% rename from scripts/setup/feature-setup.test.ts rename to packages/sim-setup/src/feature-setup.test.ts index c6ab04f2503..7774c71dceb 100644 --- a/scripts/setup/feature-setup.test.ts +++ b/packages/sim-setup/src/feature-setup.test.ts @@ -1,12 +1,12 @@ -import { describe, expect, it } from 'bun:test' import { SANDBOX_CAPABILITY, validateCapabilityFieldInput, -} from '../../apps/sim/lib/core/config/env-capabilities.ts' -import { SANDBOX_SETUP } from './capability-config.ts' -import { buildCapabilitySetupTransition } from './capability-setup.ts' -import type { ConfigurationSource } from './configuration-sources.ts' -import { reconcileLlmSetup, resolveFeatureSetupDestination } from './feature-setup.ts' +} from '@sim/deployment-config/env-capabilities' +import { describe, expect, it } from 'vitest' +import { SANDBOX_SETUP } from './capability-config' +import { buildCapabilitySetupTransition } from './capability-setup' +import type { ConfigurationSource } from './configuration-sources' +import { reconcileLlmSetup, resolveFeatureSetupDestination } from './feature-setup' const E2B_FUNCTION_TEMPLATE_ID = 'sim-function:00000000-0000-4000-8000-000000000001' const DAYTONA_FUNCTION_SNAPSHOT_ID = '00000000-0000-4000-8000-000000000002' diff --git a/scripts/setup/feature-setup.ts b/packages/sim-setup/src/feature-setup.ts similarity index 88% rename from scripts/setup/feature-setup.ts rename to packages/sim-setup/src/feature-setup.ts index b973e2272b0..3f39caa6492 100644 --- a/scripts/setup/feature-setup.ts +++ b/packages/sim-setup/src/feature-setup.ts @@ -2,18 +2,18 @@ import { LLM_KEY_POOLS, OAUTH_CLIENT_CAPABILITIES, resolveOAuthClientCapabilityId, -} from '../../apps/sim/lib/core/config/env-capabilities.ts' +} from '@sim/deployment-config/env-capabilities' import { getCapabilitySetup, getOAuthClientSetupFields, SETUP_FEATURES, type SetupFeatureId, -} from './capability-config.ts' -import { promptCapabilitySetup } from './capability-setup.ts' -import { type ConfigurationSource, discoverConfigurationSources } from './configuration-sources.ts' -import { type EnvTarget, reconcileEnvValues } from './env-files.ts' -import * as p from './prompter.ts' -import { theme } from './theme.ts' +} from './capability-config' +import { promptCapabilitySetup } from './capability-setup' +import { type ConfigurationSource, discoverConfigurationSources } from './configuration-sources' +import { type EnvTarget, reconcileEnvValues } from './env-files' +import * as p from './prompter' +import { theme } from './theme' function isSetupFeatureId(value: string): value is SetupFeatureId { return SETUP_FEATURES.some((feature) => feature.id === value) @@ -24,7 +24,7 @@ async function setupIntegration( vars: Map ): Promise> { if (!requestedId) { - throw new Error('Missing integration id. Example: bun run setup integration slack') + throw new Error('Missing integration id. Example: npx @sim/setup add integration slack') } const providerId = resolveOAuthClientCapabilityId(requestedId) if (!providerId) { @@ -131,30 +131,30 @@ export function resolveFeatureSetupDestination( sources: readonly ConfigurationSource[] ): FeatureSetupDestination { if (sources.length === 0) { - throw new Error('No Sim configuration was detected. Run bun run setup first.') + throw new Error('No Sim configuration was detected. Run npx @sim/setup first.') } const managed = sources.filter((source) => source.managedByCurrentCheckout) if (managed.length === 0) { throw new Error( - 'No effective configuration is safely writable by this checkout. Process overrides, higher-precedence development env files, external Compose projects, and Helm releases must be updated at their source. Run bun run setup status for the detected sources.' + 'No effective configuration is safely writable by this checkout. Process overrides, higher-precedence development env files, external Compose projects, and Helm releases must be updated at their source. Run npx @sim/setup config for the detected sources.' ) } if (managed.length > 1) { throw new Error( - `More than one effective configuration is writable by this checkout (${managed.map((source) => source.label).join(', ')}). Run bun run setup status and remove the ambiguity before configuring a feature.` + `More than one effective configuration is writable by this checkout (${managed.map((source) => source.label).join(', ')}). Run npx @sim/setup config and remove the ambiguity before configuring a feature.` ) } const source = managed[0] if (!source.values) { throw new Error( - `${source.label} is managed by this checkout, but its effective environment could not be resolved. Run bun run setup status and fix the reported source error first.` + `${source.label} is managed by this checkout, but its effective environment could not be resolved. Run npx @sim/setup config and fix the reported source error first.` ) } if (source.kind === 'helm') { throw new Error( - 'Helm configuration cannot be updated by bun run setup. Update the release Secret or values and upgrade the release.' + 'Helm configuration cannot be updated by npx @sim/setup add. Update the release Secret or values and upgrade the release.' ) } diff --git a/packages/sim-setup/src/index.ts b/packages/sim-setup/src/index.ts new file mode 100644 index 00000000000..c21ddf40ee3 --- /dev/null +++ b/packages/sim-setup/src/index.ts @@ -0,0 +1,185 @@ +#!/usr/bin/env node +import { readFileSync } from 'node:fs' +import { getErrorMessage } from '@sim/utils/errors' +import { SetupError } from './errors' +import { exitWith, restoreTerminal } from './terminal' +import { theme } from './theme' + +type WizardMode = 'compose' | 'dev' | 'k8s' + +const LIFECYCLE_COMMANDS = [ + 'start', + 'stop', + 'restart', + 'update', + 'status', + 'logs', + 'down', + 'reset', +] as const +type LifecycleCommand = (typeof LIFECYCLE_COMMANDS)[number] +const SETUP_FEATURES = + 'email | storage | sandbox | jobs | cache | knowledge | knowledge-embeddings | llm | integration ' + +function isLifecycleCommand(value: string | undefined): value is LifecycleCommand { + return Boolean(value && (LIFECYCLE_COMMANDS as readonly string[]).includes(value)) +} + +const USAGE = `Usage: + npx @sim/setup run the setup wizard + npx @sim/setup [--quick] [--dir ] create a Compose installation + npx @sim/setup config show configured capabilities and integrations + npx @sim/setup add configure ${SETUP_FEATURES} + npx @sim/setup doctor [--fix] [--json] check your setup + npx @sim/setup start | stop | restart bring your install up / down / cycle + npx @sim/setup update pull and apply Compose images + npx @sim/setup status show what's installed and healthy + npx @sim/setup logs follow logs + npx @sim/setup down remove containers (data kept) + npx @sim/setup reset archive .env + wipe managed data + +Inside a Sim source checkout, the existing commands remain available: + bun run setup run the setup wizard + bun run setup status show configured capabilities and integrations + bun run setup configure ${SETUP_FEATURES} + bun run sim setup [--quick] [--mode compose|dev|k8s] + bun run sim config show configured capabilities and integrations + bun run sim add configure one feature + bun run sim doctor [--fix] [--json] check your setup + bun run sim start | stop | restart bring your install up / down / cycle + bun run sim update pull/rebuild and apply Compose images + bun run sim status what's installed and healthy + bun run sim logs follow logs + bun run sim down remove containers (data kept) + bun run sim reset archive .env + wipe managed data` + +function readPackageVersion(): string { + const metadata: unknown = JSON.parse( + readFileSync(new URL('../package.json', import.meta.url), 'utf8') + ) + if ( + typeof metadata !== 'object' || + metadata === null || + !('version' in metadata) || + typeof metadata.version !== 'string' + ) { + throw new Error('@sim/setup package metadata is missing a valid version') + } + return metadata.version +} + +function withoutDirectoryOption(args: readonly string[]): string[] { + const filtered: string[] = [] + for (let index = 0; index < args.length; index += 1) { + const arg = args[index] + if (arg.startsWith('--dir=')) continue + if (arg === '--dir') { + index += 1 + continue + } + filtered.push(arg) + } + return filtered +} + +function parseMode(value: string | undefined): WizardMode { + if (value === 'compose' || value === 'dev' || value === 'k8s') return value + throw new Error(`invalid --mode "${value}" — expected compose, dev, or k8s`) +} + +async function main(): Promise { + const rawArgs = process.argv.slice(2) + if (rawArgs.includes('--version') || rawArgs.includes('-V')) { + console.log(readPackageVersion()) + return + } + const args = withoutDirectoryOption(rawArgs) + if (args.includes('--help') || args.includes('-h')) { + console.log(USAGE) + return + } + process.on('SIGINT', () => exitWith(130)) + + const command = args[0] + + if (command === 'config') { + const { runSetupStatus } = await import('./setup-status') + process.exitCode = await runSetupStatus() + return + } + + if (command === 'add') { + const feature = args[1] + if (!feature || feature.startsWith('-')) { + throw new Error(`Missing feature. Expected: ${SETUP_FEATURES}`) + } + const { runFeatureSetup } = await import('./feature-setup') + await runFeatureSetup(feature, args.slice(2)) + return + } + + if (command === 'doctor') { + const { runDoctor } = await import('./doctor') + process.exitCode = await runDoctor({ + fix: args.includes('--fix'), + json: args.includes('--json'), + }) + return + } + + if (isLifecycleCommand(command)) { + const { runLifecycle } = await import('./lifecycle') + await runLifecycle(command) + return + } + + if (!command || command === 'setup' || command.startsWith('-')) { + const setupArgs = command === 'setup' ? args.slice(1) : args + const feature = setupArgs[0]?.startsWith('-') ? undefined : setupArgs[0] + if (feature === 'status') { + const { runSetupStatus } = await import('./setup-status') + process.exitCode = await runSetupStatus() + return + } + if (feature) { + const featureIndex = setupArgs.indexOf(feature) + const { runFeatureSetup } = await import('./feature-setup') + await runFeatureSetup(feature, setupArgs.slice(featureIndex + 1)) + return + } + const modeIdx = setupArgs.indexOf('--mode') + const { runWizard } = await import('./wizard') + await runWizard({ + quick: setupArgs.includes('--quick'), + mode: modeIdx === -1 ? undefined : parseMode(setupArgs[modeIdx + 1]), + }) + return + } + + console.error(`Unknown command: ${command}\n`) + console.log(USAGE) + process.exitCode = 1 +} + +function renderFailure(error: unknown): void { + const hints = error instanceof SetupError ? error.hints : [] + console.error() + console.error(`${theme.error('✗ Setup failed')}\n`) + console.error(` ${getErrorMessage(error).split('\n').join('\n ')}`) + if (hints.length > 0) { + console.error(`\n ${theme.heading('Try:')}`) + for (const hint of hints) { + console.error(` ${theme.muted('•')} ${hint}`) + } + } + console.error( + `\n ${theme.muted('Your progress is saved — re-run')} ${theme.command('npx @sim/setup')} ${theme.muted('to pick up where you left off.')}` + ) +} + +main() + .catch((error) => { + renderFailure(error) + process.exitCode = 1 + }) + .finally(restoreTerminal) diff --git a/scripts/setup/lifecycle.test.ts b/packages/sim-setup/src/lifecycle.test.ts similarity index 90% rename from scripts/setup/lifecycle.test.ts rename to packages/sim-setup/src/lifecycle.test.ts index 76e7a942b74..a0dbfbbf41c 100644 --- a/scripts/setup/lifecycle.test.ts +++ b/packages/sim-setup/src/lifecycle.test.ts @@ -1,5 +1,5 @@ -import { describe, expect, it } from 'bun:test' -import { getComposeUpdateMode, isLifecycleCommand } from './lifecycle.ts' +import { describe, expect, it } from 'vitest' +import { getComposeUpdateMode, isLifecycleCommand } from './lifecycle' describe('setup lifecycle', () => { it('recognizes update as a lifecycle command', () => { diff --git a/scripts/setup/lifecycle.ts b/packages/sim-setup/src/lifecycle.ts similarity index 91% rename from scripts/setup/lifecycle.ts rename to packages/sim-setup/src/lifecycle.ts index e93c43d9b57..2135d6acb80 100644 --- a/scripts/setup/lifecycle.ts +++ b/packages/sim-setup/src/lifecycle.ts @@ -1,14 +1,16 @@ import { spawnSync } from 'node:child_process' import { readFileSync } from 'node:fs' import path from 'node:path' -import { DB_CONTAINER, type Detection, REDIS_CONTAINER, runDetection } from './detect.ts' -import { archiveEnvFile, ROOT } from './env-files.ts' -import { SetupError } from './errors.ts' -import { forwardCommands, isLocalKubeContext } from './modes/k8s.ts' -import { httpHealth } from './probes.ts' -import * as p from './prompter.ts' -import { glyph, theme } from './theme.ts' -import { APP_SIGNUP_URL, APP_URL } from './urls.ts' +import { ensureProductionComposeFile } from './compose-asset' +import { SETUP_CONTEXT } from './context' +import { DB_CONTAINER, type Detection, REDIS_CONTAINER, runDetection } from './detect' +import { archiveEnvFile, archiveFile, ROOT } from './env-files' +import { SetupError } from './errors' +import { forwardCommands, isLocalKubeContext } from './modes/k8s' +import { httpHealth } from './probes' +import * as p from './prompter' +import { glyph, theme } from './theme' +import { APP_SIGNUP_URL, APP_URL } from './urls' const REALTIME_HEALTH = 'http://localhost:3002/health' const POSTGRES_VOLUME = 'sim-postgres-data' @@ -101,7 +103,7 @@ const SIM_COMPOSE_MARKERS = ['ghcr.io/simstudioai/simstudio', 'docker/app.Docker /** * `docker-compose.prod.yml` is a common filename, so the name alone cannot say a - * project is ours — and `sim reset` runs `compose down -v`, which would destroy + * project is ours — and `reset` runs `compose down -v`, which would destroy * an unrelated stack's volumes. Read the file Docker recorded for the project and * require a Sim marker inside it. The old ROOT-scoped `-f` probe was implicitly * safe because it could only ever see the local project; discovering projects @@ -259,7 +261,11 @@ function start(install: Install): void { dockerRun(composeArgs(install, 'up', '-d'), 'docker compose up failed', install.dir) spin.stop('Containers up') p.note( - [`open ${APP_SIGNUP_URL}`, 'follow logs: sim logs', 'stop: sim stop'].join('\n'), + [ + `open ${APP_SIGNUP_URL}`, + 'follow logs: npx @sim/setup logs', + 'stop: npx @sim/setup stop', + ].join('\n'), 'Running' ) return @@ -269,7 +275,10 @@ function start(install: Install): void { for (const name of names) dockerRun(['start', name], `docker start ${name} failed`) if (names.length) p.log.step(`Started ${names.join(', ')}`) p.note( - ['start the dev server: bun run dev:full', 'stop DB/Redis: sim stop'].join('\n'), + [ + 'start the dev server: bun run dev:full', + 'stop DB/Redis: npx @sim/setup stop', + ].join('\n'), 'Ready' ) return @@ -283,7 +292,10 @@ function stop(install: Install): void { spin.start('Stopping containers…') dockerRun(composeArgs(install, 'stop'), 'docker compose stop failed', install.dir) spin.stop('Containers stopped (data kept)') - p.note(['start again: sim start', 'remove: sim down'].join('\n'), 'Stopped') + p.note( + ['start again: npx @sim/setup start', 'remove: npx @sim/setup down'].join('\n'), + 'Stopped' + ) return } if (install.kind === 'dev') { @@ -301,7 +313,7 @@ function stop(install: Install): void { [ `scale down: kubectl --context ${c} -n ${K8S_NAMESPACE} scale deploy --all --replicas=0`, `scale up: kubectl --context ${c} -n ${K8S_NAMESPACE} scale deploy --all --replicas=1`, - 'tear down: sim down', + 'tear down: npx @sim/setup down', ].join('\n'), 'Kubernetes' ) @@ -338,12 +350,12 @@ export function getComposeUpdateMode(file: string): ComposeUpdateMode { function update(install: Install): void { if (install.kind === 'dev') { - throw new SetupError('sim update is only available for Docker Compose installs.', [ + throw new SetupError('update is only available for Docker Compose installs.', [ 'update the source checkout with git, run bun install, then restart bun run dev:full', ]) } if (install.kind === 'k8s') { - throw new SetupError('sim update does not upgrade Kubernetes releases.', [ + throw new SetupError('update does not upgrade Kubernetes releases.', [ 'upgrade the release with helm after reviewing the chart and release notes', ]) } @@ -351,6 +363,12 @@ function update(install: Install): void { const mode = getComposeUpdateMode(install.file) const spin = p.spinner() if (mode === 'pull') { + if ( + SETUP_CONTEXT.kind === 'standalone' && + path.resolve(SETUP_CONTEXT.root) === path.resolve(install.dir) + ) { + install.file = ensureProductionComposeFile(SETUP_CONTEXT) + } spin.start('Pulling configured Sim images…') dockerRun(composeArgs(install, 'pull'), 'docker compose pull failed', install.dir) } else { @@ -363,8 +381,8 @@ function update(install: Install): void { p.note( [ `version: ${theme.command(`SIM_VERSION in ${path.join(install.dir, '.env')}`)} (latest when unset)`, - `check: ${theme.command('bun run sim status')}`, - `logs: ${theme.command('bun run sim logs')}`, + `check: ${theme.command('npx @sim/setup status')}`, + `logs: ${theme.command('npx @sim/setup logs')}`, ].join('\n'), 'Update complete' ) @@ -441,9 +459,14 @@ async function reset(install: Install | null): Promise { p.log.info('Reset cancelled.') return } - for (const target of ['sim', 'realtime', 'db', 'root'] as const) { - const backup = archiveEnvFile(target) + if (install?.kind === 'compose') { + const backup = archiveFile(path.join(install.dir, '.env')) if (backup) p.log.step(`Archived ${backup}`) + } else { + for (const target of ['sim', 'realtime', 'db', 'root'] as const) { + const backup = archiveEnvFile(target) + if (backup) p.log.step(`Archived ${backup}`) + } } if (install?.kind === 'compose') { dockerRun(composeArgs(install, 'down', '-v'), 'docker compose down -v failed', install.dir) @@ -472,7 +495,7 @@ async function reset(install: Install | null): Promise { } p.log.step(`Uninstalled ${K8S_RELEASE}`) } - p.note(`start fresh with ${theme.command('sim setup')}`, 'Reset complete') + p.note(`start fresh with ${theme.command('npx @sim/setup')}`, 'Reset complete') } async function status(): Promise { @@ -493,7 +516,7 @@ async function status(): Promise { if (installs.length === 0) { console.log( docker - ? ` ${glyph.warn} No Sim install detected — run ${theme.command('sim setup')}.` + ? ` ${glyph.warn} No Sim install detected — run ${theme.command('npx @sim/setup')}.` : ` ${glyph.warn} No install detected, but that may just be Docker being down.` ) return @@ -523,7 +546,7 @@ export async function runLifecycle(command: LifecycleCommand): Promise { const install = await resolveInstall(installs) if (!install) { - p.log.warn(`No Sim install detected. Run ${theme.command('sim setup')} first.`) + p.log.warn(`No Sim install detected. Run ${theme.command('npx @sim/setup')} first.`) return } switch (command) { diff --git a/scripts/setup/modes/compose.ts b/packages/sim-setup/src/modes/compose.ts similarity index 81% rename from scripts/setup/modes/compose.ts rename to packages/sim-setup/src/modes/compose.ts index 39fb7d1cd02..98176e438e7 100644 --- a/scripts/setup/modes/compose.ts +++ b/packages/sim-setup/src/modes/compose.ts @@ -1,13 +1,16 @@ import { spawnSync } from 'node:child_process' -import { EMAIL_SETUP, STORAGE_SETUP } from '../capability-config.ts' -import { promptCapabilitySetup, stageCapabilitySetupTransition } from '../capability-setup.ts' -import type { Detection } from '../detect.ts' -import { ensureDocker } from '../docker.ts' -import { ROOT, readEnvFile, reconcileEnvValues } from '../env-files.ts' -import { SetupError } from '../errors.ts' -import { ensurePortsFree } from '../ports.ts' -import { httpHealth, waitFor } from '../probes.ts' -import * as p from '../prompter.ts' +import path from 'node:path' +import { EMAIL_SETUP, STORAGE_SETUP } from '../capability-config' +import { promptCapabilitySetup, stageCapabilitySetupTransition } from '../capability-setup' +import { ensureProductionComposeFile } from '../compose-asset' +import { SETUP_CONTEXT } from '../context' +import type { Detection } from '../detect' +import { ensureDocker } from '../docker' +import { ROOT, readEnvFile, reconcileEnvValues } from '../env-files' +import { SetupError } from '../errors' +import { ensurePortsFree } from '../ports' +import { httpHealth, waitFor } from '../probes' +import * as p from '../prompter' import { chatFlagValues, collectSecrets, @@ -18,9 +21,9 @@ import { promptSecurity, promptSignInProviders, promptUnlocks, -} from '../steps.ts' -import { glyph, theme } from '../theme.ts' -import { APP_SIGNUP_URL, APP_URL } from '../urls.ts' +} from '../steps' +import { glyph, theme } from '../theme' +import { APP_SIGNUP_URL, APP_URL } from '../urls' const REQUIRED_PORTS = [3000, 3002] as const @@ -78,7 +81,7 @@ async function ensureComposePortsFree(composeFile: string): Promise { if (toCheck.length === 0) return if (await ensurePortsFree(toCheck)) return throw new SetupError(`ports ${toCheck.map((port) => `:${port}`).join('/')} are in use`, [ - `free the ports, then re-run: ${theme.command('bun run setup')}`, + `free the ports, then re-run: ${theme.command('npx @sim/setup')}`, `see what holds them: ${theme.command('lsof -nP -iTCP:3000 -sTCP:LISTEN')}`, `stop a container publishing them: ${theme.command('docker ps')}`, `compose file in play: ${composeFile}`, @@ -87,25 +90,27 @@ async function ensureComposePortsFree(composeFile: string): Promise { export async function runComposeMode(detection: Detection, quick: boolean): Promise { await ensureDocker(true) - const variant = quick - ? 'prod' - : await p.select({ - message: 'Which images?', - options: [ - { - value: 'prod', - label: 'Published images', - hint: 'pulls ghcr.io/simstudioai/* — fastest', - }, - { - value: 'local', - label: 'Build from source', - hint: 'builds docker/*.Dockerfile — for testing local changes', - }, - ], - initialValue: 'prod', - }) - const composeFile = variant === 'prod' ? 'docker-compose.prod.yml' : 'docker-compose.local.yml' + const variant = + SETUP_CONTEXT.kind === 'standalone' || quick + ? 'prod' + : await p.select({ + message: 'Which images?', + options: [ + { + value: 'prod', + label: 'Published images', + hint: 'pulls ghcr.io/simstudioai/* — fastest', + }, + { + value: 'local', + label: 'Build from source', + hint: 'builds docker/*.Dockerfile — for testing local changes', + }, + ], + initialValue: 'prod', + }) + const composeFile = + variant === 'prod' ? ensureProductionComposeFile() : path.join(ROOT, 'docker-compose.local.yml') const root = readEnvFile('root') const values = collectSecrets(root) @@ -150,6 +155,16 @@ export async function runComposeMode(detection: Detection, quick: boolean): Prom reconcileEnvValues('root', [...remove], values) p.log.step('Wrote .env (compose reads it for variable substitution)') + const validation = spawnSync('docker', ['compose', '-f', composeFile, 'config'], { + cwd: ROOT, + encoding: 'utf8', + }) + if (validation.status !== 0) { + throw new SetupError( + `docker compose config failed: ${validation.stderr.trim() || validation.stdout.trim()}` + ) + } + await ensureComposePortsFree(composeFile) p.log.step(`Running docker compose -f ${composeFile} up -d`) diff --git a/scripts/setup/modes/dev.ts b/packages/sim-setup/src/modes/dev.ts similarity index 93% rename from scripts/setup/modes/dev.ts rename to packages/sim-setup/src/modes/dev.ts index 86d44a88023..3fa36b7ae8e 100644 --- a/scripts/setup/modes/dev.ts +++ b/packages/sim-setup/src/modes/dev.ts @@ -1,15 +1,15 @@ import { spawnSync } from 'node:child_process' import path from 'node:path' import { truncate } from '@sim/utils/string' -import { EMAIL_SETUP, JOBS_SETUP, STORAGE_SETUP } from '../capability-config.ts' -import { promptCapabilitySetup, stageCapabilitySetupTransition } from '../capability-setup.ts' -import { resolveDatabase } from '../db.ts' -import type { Detection } from '../detect.ts' -import { ROOT, readEnvFile, reconcileEnvValues, writeEnvValues } from '../env-files.ts' -import { SetupError } from '../errors.ts' -import { pgProbe } from '../probes.ts' -import * as p from '../prompter.ts' -import { ensureRedis, resolveRedis } from '../redis.ts' +import { EMAIL_SETUP, JOBS_SETUP, STORAGE_SETUP } from '../capability-config' +import { promptCapabilitySetup, stageCapabilitySetupTransition } from '../capability-setup' +import { resolveDatabase } from '../db' +import type { Detection } from '../detect' +import { ROOT, readEnvFile, reconcileEnvValues, writeEnvValues } from '../env-files' +import { SetupError } from '../errors' +import { pgProbe } from '../probes' +import * as p from '../prompter' +import { ensureRedis, resolveRedis } from '../redis' import { chatFlagValues, collectSecrets, @@ -20,9 +20,9 @@ import { promptSecurity, promptSignInProviders, promptUnlocks, -} from '../steps.ts' -import { glyph, theme } from '../theme.ts' -import { APP_URL } from '../urls.ts' +} from '../steps' +import { glyph, theme } from '../theme' +import { APP_URL } from '../urls' /** * A migrate failure on a never-migrated database means setup failed — abort. diff --git a/scripts/setup/modes/k8s.ts b/packages/sim-setup/src/modes/k8s.ts similarity index 97% rename from scripts/setup/modes/k8s.ts rename to packages/sim-setup/src/modes/k8s.ts index b4261e37dad..b28fe852779 100644 --- a/scripts/setup/modes/k8s.ts +++ b/packages/sim-setup/src/modes/k8s.ts @@ -1,21 +1,21 @@ import { spawn, spawnSync } from 'node:child_process' import { getErrorMessage } from '@sim/utils/errors' -import { KNOWLEDGE_EMBEDDINGS_SETUP } from '../capability-config.ts' -import { getCapabilitySetupFields, stageCapabilitySetupTransition } from '../capability-setup.ts' -import type { Detection } from '../detect.ts' -import { ensureDocker } from '../docker.ts' -import { generateSecret, ROOT } from '../env-files.ts' -import { SetupError } from '../errors.ts' -import { waitFor } from '../probes.ts' -import * as p from '../prompter.ts' +import { KNOWLEDGE_EMBEDDINGS_SETUP } from '../capability-config' +import { getCapabilitySetupFields, stageCapabilitySetupTransition } from '../capability-setup' +import type { Detection } from '../detect' +import { ensureDocker } from '../docker' +import { generateSecret, ROOT } from '../env-files' +import { SetupError } from '../errors' +import { waitFor } from '../probes' +import * as p from '../prompter' import { chatFlagValues, mothershipOverride, promptCopilotKey, promptKnowledgeEmbeddings, -} from '../steps.ts' -import { glyph, theme } from '../theme.ts' -import { APP_SIGNUP_URL, APP_URL } from '../urls.ts' +} from '../steps' +import { glyph, theme } from '../theme' +import { APP_SIGNUP_URL, APP_URL } from '../urls' const RELEASE = 'sim-dev' const NAMESPACE = 'sim-dev' @@ -174,7 +174,7 @@ async function ensureLocalContext(detection: Detection): Promise { spin.stop(`${glyph.fail} kind cluster "sim" would not start`) throw new SetupError('the kind cluster "sim" exists but will not come up.', [ `inspect it: ${theme.command('docker ps -a --filter name=sim-control-plane')}`, - `recreate it: ${theme.command('kind delete cluster --name sim')}, then re-run ${theme.command('bun run setup')}`, + `recreate it: ${theme.command('kind delete cluster --name sim')}, then re-run ${theme.command('npx @sim/setup')}`, ]) } spin.stop('kind cluster "sim" started') diff --git a/scripts/setup/ports.ts b/packages/sim-setup/src/ports.ts similarity index 96% rename from scripts/setup/ports.ts rename to packages/sim-setup/src/ports.ts index 78b4659e1fb..3bc09708559 100644 --- a/scripts/setup/ports.ts +++ b/packages/sim-setup/src/ports.ts @@ -1,8 +1,8 @@ import { getErrorMessage } from '@sim/utils/errors' -import { type PortOwnerInfo, portOpen, portOwner } from './detect.ts' -import { waitFor } from './probes.ts' -import * as p from './prompter.ts' -import { theme } from './theme.ts' +import { type PortOwnerInfo, portOpen, portOwner } from './detect' +import { waitFor } from './probes' +import * as p from './prompter' +import { theme } from './theme' interface BusyPort { port: number diff --git a/scripts/setup/probes.ts b/packages/sim-setup/src/probes.ts similarity index 93% rename from scripts/setup/probes.ts rename to packages/sim-setup/src/probes.ts index 7c86e0063b4..1067fa0cac8 100644 --- a/scripts/setup/probes.ts +++ b/packages/sim-setup/src/probes.ts @@ -1,21 +1,22 @@ -import { readFileSync } from 'node:fs' +import { existsSync, readFileSync } from 'node:fs' import net from 'node:net' import path from 'node:path' import tls from 'node:tls' import { getErrorMessage } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' import postgres from 'postgres' -import { ROOT } from './env-files.ts' +import { ROOT } from './env-files' export interface PgProbeResult { ok: boolean error?: string pgvectorAvailable?: boolean - migrations?: { applied: number | null; journal: number } + migrations?: { applied: number | null; journal: number | null } } -function journalMigrationCount(): number { +function journalMigrationCount(): number | null { const journalPath = path.join(ROOT, 'packages/db/migrations/meta/_journal.json') + if (!existsSync(journalPath)) return null const journal = JSON.parse(readFileSync(journalPath, 'utf8')) as { entries: unknown[] } return journal.entries.length } diff --git a/scripts/setup/prompter.ts b/packages/sim-setup/src/prompter.ts similarity index 60% rename from scripts/setup/prompter.ts rename to packages/sim-setup/src/prompter.ts index 705d850ac24..d6d17b65495 100644 --- a/scripts/setup/prompter.ts +++ b/packages/sim-setup/src/prompter.ts @@ -1,6 +1,6 @@ import * as clack from '@clack/prompts' -import { exitWith } from './terminal.ts' -import { isRich, theme } from './theme.ts' +import { exitWith } from './terminal' +import { isRich, theme } from './theme' const SPINNER_FRAMES = ['◐', '◓', '◑', '◒'] @@ -18,6 +18,17 @@ export interface SelectOption { hint?: string } +function formatOptions(options: SelectOption[]) { + return options.map((option) => { + if (option.hint === undefined) return { value: option.value, label: option.label } + return { + value: option.value, + label: option.label, + hint: isRich() ? theme.muted(option.hint) : option.hint, + } + }) +} + export async function select(params: { message: string options: SelectOption[] @@ -26,11 +37,8 @@ export async function select(params: { return guardCancel( await clack.select({ message: isRich() ? theme.accent(params.message) : params.message, - options: params.options.map((o) => ({ - ...o, - hint: o.hint && isRich() ? theme.muted(o.hint) : o.hint, - })), - initialValue: params.initialValue, + options: formatOptions(params.options) as clack.Option[], + ...(params.initialValue === undefined ? {} : { initialValue: params.initialValue }), }) ) } @@ -43,8 +51,8 @@ export async function multiselect(params: { return guardCancel( await clack.multiselect({ message: isRich() ? theme.accent(params.message) : params.message, - options: params.options, - initialValues: params.initialValues, + options: formatOptions(params.options) as clack.Option[], + ...(params.initialValues === undefined ? {} : { initialValues: params.initialValues }), required: false, }) ) @@ -60,10 +68,12 @@ export async function text(params: { return guardCancel( await clack.text({ message: isRich() ? theme.accent(params.message) : params.message, - placeholder: params.placeholder, - initialValue: params.initialValue, - defaultValue: params.defaultValue, - validate: params.validate, + ...(params.placeholder === undefined ? {} : { placeholder: params.placeholder }), + ...(params.initialValue === undefined ? {} : { initialValue: params.initialValue }), + ...(params.defaultValue === undefined ? {} : { defaultValue: params.defaultValue }), + ...(params.validate === undefined + ? {} + : { validate: (value: string | undefined) => params.validate?.(value ?? '') }), }) ) } @@ -75,7 +85,9 @@ export async function password(params: { return guardCancel( await clack.password({ message: isRich() ? theme.accent(params.message) : params.message, - validate: params.validate, + ...(params.validate === undefined + ? {} + : { validate: (value: string | undefined) => params.validate?.(value ?? '') }), }) ) } @@ -87,7 +99,7 @@ export async function confirm(params: { return guardCancel( await clack.confirm({ message: isRich() ? theme.accent(params.message) : params.message, - initialValue: params.initialValue, + ...(params.initialValue === undefined ? {} : { initialValue: params.initialValue }), }) ) } diff --git a/scripts/setup/redis.ts b/packages/sim-setup/src/redis.ts similarity index 95% rename from scripts/setup/redis.ts rename to packages/sim-setup/src/redis.ts index f8257448ef6..329c31e799c 100644 --- a/scripts/setup/redis.ts +++ b/packages/sim-setup/src/redis.ts @@ -1,11 +1,11 @@ import { spawnSync } from 'node:child_process' -import { docker } from './db.ts' -import { type Detection, REDIS_CONTAINER } from './detect.ts' -import { ensureDocker } from './docker.ts' -import { SetupError } from './errors.ts' -import { redisPing, waitFor } from './probes.ts' -import * as p from './prompter.ts' -import { glyph, theme } from './theme.ts' +import { docker } from './db' +import { type Detection, REDIS_CONTAINER } from './detect' +import { ensureDocker } from './docker' +import { SetupError } from './errors' +import { redisPing, waitFor } from './probes' +import * as p from './prompter' +import { glyph, theme } from './theme' const LOCAL_URL = 'redis://localhost:6379' diff --git a/scripts/setup/setup-status.test.ts b/packages/sim-setup/src/setup-status.test.ts similarity index 97% rename from scripts/setup/setup-status.test.ts rename to packages/sim-setup/src/setup-status.test.ts index d69339eb3f5..c38a636041e 100644 --- a/scripts/setup/setup-status.test.ts +++ b/packages/sim-setup/src/setup-status.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from 'bun:test' +import { describe, expect, it } from 'vitest' import type { ConfigurationSource } from './configuration-sources' import { buildSetupStatusReport, renderSetupStatusReport } from './setup-status' @@ -76,7 +76,7 @@ describe('setup status', () => { expect(report.capabilityStatus?.features.email.state).toBe('configured') expect(output).toContain('Email delivery: Resend') expect(output).toContain('SMTP_PORT') - expect(output).toContain('configure: bun run setup email') + expect(output).toContain('configure: npx @sim/setup add email') expect(output).not.toContain('resend-super-secret') }) diff --git a/scripts/setup/setup-status.ts b/packages/sim-setup/src/setup-status.ts similarity index 95% rename from scripts/setup/setup-status.ts rename to packages/sim-setup/src/setup-status.ts index fb4bffc0a98..212ec8dec3a 100644 --- a/scripts/setup/setup-status.ts +++ b/packages/sim-setup/src/setup-status.ts @@ -1,21 +1,21 @@ import { type EnvCapabilityValues, hasEnvCapabilityValue, -} from '../../apps/sim/lib/core/config/env-capabilities.ts' +} from '@sim/deployment-config/env-capabilities' import { type IntegrationAvailability, resolveIntegrationAvailability, -} from '../../apps/sim/lib/integrations/availability.ts' -import { SETUP_FEATURES } from './capability-config.ts' +} from '@sim/deployment-config/integration-availability' +import { SETUP_FEATURES } from './capability-config' import { buildEnvCapabilityStatus, type EnvCapabilityFeatureStatuses, type SetupStatusFeatureId, -} from './capability-status.ts' -import { REQUIRED_APP_KEYS } from './checks.ts' -import { type ConfigurationSource, discoverConfigurationSources } from './configuration-sources.ts' -import { isPlaceholder, isUsableSecret } from './env-files.ts' -import { glyph, theme } from './theme.ts' +} from './capability-status' +import { REQUIRED_APP_KEYS } from './checks' +import { type ConfigurationSource, discoverConfigurationSources } from './configuration-sources' +import { isPlaceholder, isUsableSecret } from './env-files' +import { glyph, theme } from './theme' type FeatureStatus = EnvCapabilityFeatureStatuses[keyof EnvCapabilityFeatureStatuses] @@ -132,7 +132,7 @@ function featureGlyph(feature: FeatureStatus): string { } function withoutSetupCommand(message: string): string { - return message.replace(/\s+Run bun run setup[^.]*\.$/, '') + return message.replace(/\s+Run npx @sim\/setup setup[^.]*\.$/, '') } function setupHint(source: SetupStatusSource, command: string): string | null { @@ -330,7 +330,7 @@ export function renderSetupStatusReport(report: SetupStatusReport): string { const representedOAuthClients = new Set( deploymentIntegrations.flatMap((integration) => { if (!integration.setupCommand) return [] - return [integration.setupCommand.replace('bun run setup integration ', '')] + return [integration.setupCommand.replace('npx @sim/setup add integration ', '')] }) ) const additionalOAuthClients = Object.values(report.capabilityStatus.oauthClients.clients).filter( @@ -357,7 +357,7 @@ export async function runSetupStatus(): Promise { console.log(`\n${theme.heading('◆ Sim setup status')}\n`) if (sources.length === 0) { console.log(` ${glyph.fail} No local-dev, Docker Compose, or Helm configuration detected.`) - console.log(` ${theme.muted('run: bun run setup')}`) + console.log(` ${theme.muted('run: npx @sim/setup')}`) return 1 } diff --git a/scripts/setup/steps.test.ts b/packages/sim-setup/src/steps.test.ts similarity index 97% rename from scripts/setup/steps.test.ts rename to packages/sim-setup/src/steps.test.ts index d28295c5ccc..a54710b97b0 100644 --- a/scripts/setup/steps.test.ts +++ b/packages/sim-setup/src/steps.test.ts @@ -1,4 +1,3 @@ -import { describe, expect, it } from 'bun:test' import { EMAIL_CAPABILITY, inspectCapability, @@ -6,12 +5,13 @@ import { requireCapability, STORAGE_CAPABILITY, validateCapabilityFieldInput, -} from '../../apps/sim/lib/core/config/env-capabilities.ts' -import { EMAIL_SETUP, KNOWLEDGE_EMBEDDINGS_SETUP, STORAGE_SETUP } from './capability-config.ts' +} from '@sim/deployment-config/env-capabilities' +import { describe, expect, it } from 'vitest' +import { EMAIL_SETUP, KNOWLEDGE_EMBEDDINGS_SETUP, STORAGE_SETUP } from './capability-config' import { buildCapabilitySetupTransition, resolveCurrentCapabilitySetupOptionId, -} from './capability-setup.ts' +} from './capability-setup' function applyResult( initial: Record, diff --git a/scripts/setup/steps.ts b/packages/sim-setup/src/steps.ts similarity index 94% rename from scripts/setup/steps.ts rename to packages/sim-setup/src/steps.ts index 91f05b4181c..cd96a29fc49 100644 --- a/scripts/setup/steps.ts +++ b/packages/sim-setup/src/steps.ts @@ -1,11 +1,12 @@ -import { KNOWLEDGE_EMBEDDINGS_SETUP } from './capability-config.ts' +import { KNOWLEDGE_EMBEDDINGS_SETUP } from './capability-config' import { type CapabilitySetupContext, type EnvCapabilitySetupTransition, promptOptionalCapabilitySetup, -} from './capability-setup.ts' -import { browserKeyFlow } from './cli-auth.ts' -import type { Detection } from './detect.ts' +} from './capability-setup' +import { browserKeyFlow } from './cli-auth' +import { SETUP_CONTEXT } from './context' +import type { Detection } from './detect' import { type EnvFile, generateSecret, @@ -14,10 +15,10 @@ import { isUsableSecret, SECRET_KEYS, secretRequirement, -} from './env-files.ts' -import * as p from './prompter.ts' -import { link, theme } from './theme.ts' -import { FLAG_TWINS, LOGIN_PROVIDERS, SELF_HOST_UNLOCKS } from './twins.ts' +} from './env-files' +import * as p from './prompter' +import { link, theme } from './theme' +import { FLAG_TWINS, LOGIN_PROVIDERS, SELF_HOST_UNLOCKS } from './twins' /** Where the Chat key is minted when SIM_CLI_AUTH_ORIGIN is unset. */ const DEFAULT_CLI_AUTH_ORIGIN = 'https://www.sim.ai' @@ -71,7 +72,7 @@ export async function promptCopilotKey(existing?: string): Promise = {} if (detection.shellLlmKeys.length > 0) { const adopt = await p.multiselect({ - message: 'Found LLM API keys in your shell — copy into apps/sim/.env?', + message: `Found LLM API keys in your shell — copy into ${SETUP_CONTEXT.kind === 'source' ? 'apps/sim/.env' : '.env'}?`, options: detection.shellLlmKeys.map((key) => ({ value: key, label: key })), initialValues: detection.shellLlmKeys, }) diff --git a/scripts/setup/terminal.ts b/packages/sim-setup/src/terminal.ts similarity index 100% rename from scripts/setup/terminal.ts rename to packages/sim-setup/src/terminal.ts diff --git a/scripts/setup/theme.ts b/packages/sim-setup/src/theme.ts similarity index 100% rename from scripts/setup/theme.ts rename to packages/sim-setup/src/theme.ts diff --git a/scripts/setup/twins.ts b/packages/sim-setup/src/twins.ts similarity index 97% rename from scripts/setup/twins.ts rename to packages/sim-setup/src/twins.ts index 1f3374ca762..4ef7207cfd1 100644 --- a/scripts/setup/twins.ts +++ b/packages/sim-setup/src/twins.ts @@ -1,7 +1,4 @@ -import { - EMAIL_CAPABILITY, - inspectCapability, -} from '../../apps/sim/lib/core/config/env-capabilities.ts' +import { EMAIL_CAPABILITY, inspectCapability } from '@sim/deployment-config/env-capabilities' /** * Server/client feature-flag pairs that must be set together — server code diff --git a/scripts/setup/urls.ts b/packages/sim-setup/src/urls.ts similarity index 100% rename from scripts/setup/urls.ts rename to packages/sim-setup/src/urls.ts diff --git a/scripts/setup/wizard.ts b/packages/sim-setup/src/wizard.ts similarity index 86% rename from scripts/setup/wizard.ts rename to packages/sim-setup/src/wizard.ts index b9de2835fd0..b1412cff459 100644 --- a/scripts/setup/wizard.ts +++ b/packages/sim-setup/src/wizard.ts @@ -1,15 +1,16 @@ import { spawnSync } from 'node:child_process' -import { showBanner } from './banner.ts' -import { loadCheckContext, runChecks } from './checks.ts' -import { type Detection, runDetection } from './detect.ts' -import { archiveEnvFile, ROOT } from './env-files.ts' -import { runComposeMode } from './modes/compose.ts' -import { runDevMode } from './modes/dev.ts' -import { runK8sMode } from './modes/k8s.ts' -import { ensurePortsFree } from './ports.ts' -import * as p from './prompter.ts' -import { glyph, theme } from './theme.ts' -import { APP_SIGNUP_URL } from './urls.ts' +import { showBanner } from './banner' +import { loadCheckContext, runChecks } from './checks' +import { SETUP_CONTEXT } from './context' +import { type Detection, runDetection } from './detect' +import { archiveEnvFile, ROOT } from './env-files' +import { runComposeMode } from './modes/compose' +import { runDevMode } from './modes/dev' +import { runK8sMode } from './modes/k8s' +import { ensurePortsFree } from './ports' +import * as p from './prompter' +import { glyph, theme } from './theme' +import { APP_SIGNUP_URL } from './urls' export type WizardMode = 'compose' | 'dev' | 'k8s' @@ -52,6 +53,14 @@ const LOW_DOCKER_MEM_GB = 6 const LOW_DISK_GB = 15 async function selectMode(detection: Detection, flags: WizardFlags): Promise { + if (SETUP_CONTEXT.kind === 'standalone') { + if (flags.mode && flags.mode !== 'compose') { + throw new Error( + `${flags.mode} mode requires a Sim source checkout; run from a cloned Sim repository or choose --mode compose.` + ) + } + return 'compose' + } if (flags.mode) return flags.mode const { dockerMemGb } = detection.specs const vm = dockerMemGb !== null ? ` · VM ${dockerMemGb}GB` : '' @@ -127,7 +136,7 @@ export async function runWizard(flags: WizardFlags): Promise { ) if ((await handleExistingConfig(detection)) === 'doctor') { - const { runDoctor } = await import('./doctor.ts') + const { runDoctor } = await import('./doctor') process.exitCode = await runDoctor({ fix: false, json: false }) return } @@ -163,10 +172,9 @@ export async function runWizard(flags: WizardFlags): Promise { p.note( [ mode === 'k8s' ? `port-forward, then open ${APP_SIGNUP_URL}` : `open ${APP_SIGNUP_URL}`, - 'manage it: bun run sim start · stop · update · status · logs', - 'check your setup: bun run sim doctor', + 'manage it: npx @sim/setup start · stop · update · status · logs', + 'check your setup: npx @sim/setup doctor', mode === 'dev' && !startDevNow ? `start Sim: bun run ${devScript}` : null, - `prefer a bare "sim"? ${theme.command('bun link')} once (needs ~/.bun/bin on PATH)`, ] .filter(Boolean) .join('\n'), diff --git a/packages/sim-setup/tsconfig.json b/packages/sim-setup/tsconfig.json new file mode 100644 index 00000000000..98522576add --- /dev/null +++ b/packages/sim-setup/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "@sim/tsconfig/base.json", + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/sim-setup/vitest.config.ts b/packages/sim-setup/vitest.config.ts new file mode 100644 index 00000000000..2b1c323fe22 --- /dev/null +++ b/packages/sim-setup/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + environment: 'node', + }, +}) diff --git a/scripts/check-integration-catalog.ts b/scripts/check-integration-catalog.ts index 955f788e1dc..754df89cab7 100644 --- a/scripts/check-integration-catalog.ts +++ b/scripts/check-integration-catalog.ts @@ -8,7 +8,7 @@ import { stripVersionSuffix } from '@sim/utils/string' */ import { BLOCK_REGISTRY } from '../apps/sim/blocks/registry-maps' import { AuthMode, type BlockConfig } from '../apps/sim/blocks/types' -import integrationsJson from '../apps/sim/lib/integrations/integrations.json' +import integrationsJson from '../packages/deployment-config/src/integrations.json' import { DOCS_ORIGIN, DOCS_OUTPUT_PATH, defaultIntegrationDocsUrl } from './generate-docs' type CatalogAuthType = 'oauth' | 'api-key' | 'none' diff --git a/scripts/generate-deployment-config.ts b/scripts/generate-deployment-config.ts new file mode 100644 index 00000000000..7f1873026f8 --- /dev/null +++ b/scripts/generate-deployment-config.ts @@ -0,0 +1,96 @@ +#!/usr/bin/env bun +/** + * Generates deployment facts from the canonical OAuth registry. + * + * The setup package cannot import the application registry at runtime, so it + * consumes this checked-in projection instead. Deployment policy does not + * belong here; special availability rules remain handwritten in + * `packages/deployment-config/src/service-account-metadata.ts`. + * + * Usage: + * bun run scripts/generate-deployment-config.ts + * bun run scripts/generate-deployment-config.ts --check + */ +import { readFile, writeFile } from 'node:fs/promises' +import { dirname, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { getAllOAuthServices } from '../apps/sim/lib/oauth/utils' +import integrationsJson from '../packages/deployment-config/src/integrations.json' + +interface DeploymentIntegration { + authType: 'oauth' | 'api-key' | 'none' + oauthServiceId?: string +} + +const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url)) +const ROOT = resolve(SCRIPT_DIR, '..') +const OUTPUT_PATH = resolve( + ROOT, + 'packages/deployment-config/src/service-account-providers.generated.ts' +) +const CHECK_MODE = process.argv.includes('--check') + +function buildServiceAccountProviders(): ReadonlyMap { + const canonicalServices = new Map() + for (const service of getAllOAuthServices()) { + if (canonicalServices.has(service.serviceId)) { + throw new Error(`Duplicate canonical OAuth service id: ${service.serviceId}`) + } + canonicalServices.set(service.serviceId, service.serviceAccountProviderId) + } + + const catalogServiceIds = new Set() + for (const integration of integrationsJson.integrations as readonly DeploymentIntegration[]) { + if (integration.authType !== 'oauth') continue + if (!integration.oauthServiceId) { + throw new Error( + 'Generated integration catalog contains an OAuth entry without oauthServiceId' + ) + } + catalogServiceIds.add(integration.oauthServiceId) + } + + const providers = new Map() + for (const serviceId of [...catalogServiceIds].sort()) { + if (!canonicalServices.has(serviceId)) { + throw new Error(`Integration catalog references unknown OAuth service: ${serviceId}`) + } + const providerId = canonicalServices.get(serviceId) + if (providerId) providers.set(serviceId, providerId) + } + return providers +} + +function renderServiceAccountProviders(providers: ReadonlyMap): string { + const quote = (value: string) => `'${value.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'` + const entries = [...providers] + .map( + ([serviceId, providerId]) => + ` ${/^[A-Za-z_$][\w$]*$/.test(serviceId) ? serviceId : quote(serviceId)}: ${quote(providerId)},` + ) + .join('\n') + + return `/** + * Generated by \`bun run deployment-config:generate\` from the canonical OAuth + * registry and integration catalog. Do not edit this file directly. + */ +export const SERVICE_ACCOUNT_PROVIDER_BY_OAUTH_SERVICE_ID = { +${entries} +} as const +` +} + +const generated = renderServiceAccountProviders(buildServiceAccountProviders()) + +if (CHECK_MODE) { + const current = await readFile(OUTPUT_PATH, 'utf8').catch(() => '') + if (current !== generated) { + throw new Error( + 'Deployment config is stale. Run `bun run deployment-config:generate` and commit the result.' + ) + } + process.stdout.write('Deployment config is current.\n') +} else { + await writeFile(OUTPUT_PATH, generated) + process.stdout.write(`Generated ${OUTPUT_PATH}\n`) +} diff --git a/scripts/generate-docs.ts b/scripts/generate-docs.ts index ac1d9ca0c77..20f19cd3c1e 100755 --- a/scripts/generate-docs.ts +++ b/scripts/generate-docs.ts @@ -35,6 +35,7 @@ export function defaultIntegrationDocsUrl(blockType: string): string { const ICONS_PATH = path.join(rootDir, 'apps/sim/components/icons.tsx') const DOCS_ICONS_PATH = path.join(rootDir, 'apps/docs/components/icons.tsx') const INTEGRATIONS_DATA_PATH = path.join(rootDir, 'apps/sim/lib/integrations') +const INTEGRATIONS_CATALOG_PATH = path.join(rootDir, 'packages/deployment-config/src') const LANDING_INTEGRATIONS_DATA_PATH = path.join( rootDir, 'apps/sim/app/(landing)/integrations/data' @@ -1156,7 +1157,7 @@ async function writeIntegrationsJson(iconMapping: Record): Prom integrations.sort((a, b) => a.name.localeCompare(b.name)) - const jsonPath = path.join(INTEGRATIONS_DATA_PATH, 'integrations.json') + const jsonPath = path.join(INTEGRATIONS_CATALOG_PATH, 'integrations.json') // `JSON.stringify` always expands every array across multiple lines, but Biome's // JSON formatter inlines short arrays of primitive strings. Pre-collapse those // arrays here so the emitted file is already in Biome's canonical shape and diff --git a/scripts/run-audits.ts b/scripts/run-audits.ts index 05324688c19..d671a0908fa 100644 --- a/scripts/run-audits.ts +++ b/scripts/run-audits.ts @@ -26,6 +26,7 @@ const EXCLUDED: Record = { */ const EXTRA_AUDITS = [ 'tool-metadata:check', + 'deployment-config:check', 'integration-catalog:check', 'docs:check', 'agent-stream-docs:check', diff --git a/scripts/setup/index.ts b/scripts/setup/index.ts deleted file mode 100755 index d60d6bf8634..00000000000 --- a/scripts/setup/index.ts +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env bun -import { getErrorMessage } from '@sim/utils/errors' -import { runDoctor } from './doctor.ts' -import { SetupError } from './errors.ts' -import { runFeatureSetup, setupFeatureUsage } from './feature-setup.ts' -import { isLifecycleCommand, runLifecycle } from './lifecycle.ts' -import { runSetupStatus } from './setup-status.ts' -import { exitWith, restoreTerminal } from './terminal.ts' -import { theme } from './theme.ts' -import { runWizard, type WizardMode } from './wizard.ts' - -const USAGE = `Usage: - bun run setup run the setup wizard - bun run setup status show configured capabilities and integrations - bun run setup configure ${setupFeatureUsage()} - bun run sim setup [--quick] [--mode compose|dev|k8s] - bun run sim setup status show configured capabilities and integrations - bun run sim setup configure one feature - bun run sim doctor [--fix] [--json] check your setup - bun run sim start | stop | restart bring your install up / down / cycle - bun run sim update pull/rebuild and apply Compose images - bun run sim status what's installed and healthy - bun run sim logs follow logs - bun run sim down remove containers (data kept) - bun run sim reset archive .env + wipe managed data - -Prefer a bare "sim"? Run "bun link" once, then ensure ~/.bun/bin is on your -PATH (Homebrew's bun doesn't add it): export PATH="$HOME/.bun/bin:$PATH".` - -function parseMode(value: string | undefined): WizardMode { - if (value === 'compose' || value === 'dev' || value === 'k8s') return value - throw new Error(`invalid --mode "${value}" — expected compose, dev, or k8s`) -} - -async function main(): Promise { - const args = process.argv.slice(2) - if (args.includes('--help') || args.includes('-h')) { - console.log(USAGE) - return - } - process.on('SIGINT', () => exitWith(130)) - - const command = args[0] - - // Bare `sim` prints help; the wizard is `sim setup` (or `bun run setup`). - if (!command) { - console.log(USAGE) - return - } - - if (command === 'doctor') { - process.exitCode = await runDoctor({ - fix: args.includes('--fix'), - json: args.includes('--json'), - }) - return - } - - if (isLifecycleCommand(command)) { - await runLifecycle(command) - return - } - - if (command === 'setup') { - const setupArgs = args.slice(1) - const feature = setupArgs[0]?.startsWith('-') ? undefined : setupArgs[0] - if (feature === 'status') { - process.exitCode = await runSetupStatus() - return - } - if (feature) { - const featureIndex = setupArgs.indexOf(feature) - await runFeatureSetup(feature, setupArgs.slice(featureIndex + 1)) - return - } - const modeIdx = setupArgs.indexOf('--mode') - await runWizard({ - quick: setupArgs.includes('--quick'), - mode: modeIdx === -1 ? undefined : parseMode(setupArgs[modeIdx + 1]), - }) - return - } - - console.error(`Unknown command: ${command}\n`) - console.log(USAGE) - process.exitCode = 1 -} - -function renderFailure(error: unknown): void { - const hints = error instanceof SetupError ? error.hints : [] - console.error() - console.error(`${theme.error('✗ Setup failed')}\n`) - console.error(` ${getErrorMessage(error).split('\n').join('\n ')}`) - if (hints.length > 0) { - console.error(`\n ${theme.heading('Try:')}`) - for (const hint of hints) { - console.error(` ${theme.muted('•')} ${hint}`) - } - } - console.error( - `\n ${theme.muted('Your progress is saved — re-run')} ${theme.command('bun run setup')} ${theme.muted('to pick up where you left off.')}` - ) -} - -main() - .catch((error) => { - renderFailure(error) - process.exitCode = 1 - }) - .finally(restoreTerminal) diff --git a/scripts/setup/launcher.test.ts b/scripts/setup/launcher.test.ts deleted file mode 100644 index 204ba8e5d8b..00000000000 --- a/scripts/setup/launcher.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { describe, expect, it } from 'bun:test' -import { - isMissingDependencyError, - missingDependenciesMessage, - retrySetupCommand, -} from './launcher.ts' - -describe('setup launcher', () => { - it('recognizes missing packages without masking application import errors', () => { - expect( - isMissingDependencyError({ - code: 'ERR_MODULE_NOT_FOUND', - message: "Cannot find package '@clack/prompts' from '/repo/scripts/setup/prompter.ts'", - }) - ).toBe(true) - expect( - isMissingDependencyError({ - code: 'ERR_MODULE_NOT_FOUND', - message: "Cannot find module './missing-application-file.ts'", - }) - ).toBe(false) - expect(isMissingDependencyError(new Error('Invalid setup configuration'))).toBe(false) - }) - - it('prints the install command and the public retry command', () => { - expect(retrySetupCommand(['setup', 'status'])).toBe('bun run setup status') - expect(retrySetupCommand(['doctor'])).toBe('bun run sim doctor') - expect(missingDependenciesMessage('bun run setup status')).toContain('Run: bun install') - expect(missingDependenciesMessage('bun run setup status')).toContain( - 'Then retry: bun run setup status' - ) - }) -}) diff --git a/scripts/setup/launcher.ts b/scripts/setup/launcher.ts deleted file mode 100755 index 29e057809ff..00000000000 --- a/scripts/setup/launcher.ts +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bun - -interface ModuleResolutionError { - code?: unknown - message?: unknown -} - -function asModuleResolutionError(error: unknown): ModuleResolutionError | null { - return typeof error === 'object' && error !== null ? error : null -} - -/** Identifies dependency-resolution failures without masking setup/configuration errors. */ -export function isMissingDependencyError(error: unknown): boolean { - const candidate = asModuleResolutionError(error) - if (candidate?.code !== 'ERR_MODULE_NOT_FOUND' && candidate?.code !== 'MODULE_NOT_FOUND') { - return false - } - return ( - typeof candidate.message === 'string' && - /Cannot find (?:package|module) ['"][^./][^'"]*['"]/.test(candidate.message) - ) -} - -/** Reconstructs the public command instead of exposing the internal launcher path. */ -export function retrySetupCommand(args: readonly string[]): string { - if (args[0] === 'setup') return ['bun run setup', ...args.slice(1)].join(' ') - return ['bun run sim', ...args].join(' ') -} - -export function missingDependenciesMessage(retryCommand: string): string { - return [ - '', - '✗ Setup dependencies are missing or out of date.', - '', - ' Run: bun install', - ` Then retry: ${retryCommand}`, - ].join('\n') -} - -export async function launchSetupCli( - args: readonly string[] = process.argv.slice(2) -): Promise { - try { - await import('./index.ts') - } catch (error) { - if (!isMissingDependencyError(error)) throw error - console.error(missingDependenciesMessage(retrySetupCommand(args))) - process.exitCode = 1 - } -} - -if (import.meta.main) await launchSetupCli() From dc2dd656d11dc63c1880ed861570cbcc2ad4e8e3 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 19 Aug 2026 10:53:49 -0700 Subject: [PATCH 2/7] fix(setup): refresh discovered compose installs --- packages/sim-setup/src/context.ts | 30 ++++++++++++++---------- packages/sim-setup/src/lifecycle.test.ts | 29 ++++++++++++++++++++++- packages/sim-setup/src/lifecycle.ts | 15 ++++++------ 3 files changed, 54 insertions(+), 20 deletions(-) diff --git a/packages/sim-setup/src/context.ts b/packages/sim-setup/src/context.ts index 2f178636926..bd26182bdd7 100644 --- a/packages/sim-setup/src/context.ts +++ b/packages/sim-setup/src/context.ts @@ -67,6 +67,23 @@ function directoryOverride(args: readonly string[]): string | null { return value } +/** Classifies one exact filesystem root without walking through its ancestors. */ +export function resolveSetupContextAtRoot(root: string): SetupContext { + const resolvedRoot = path.resolve(root) + const source = inspectSourceRoot(resolvedRoot) + if (source === 'valid') return { kind: 'source', root: resolvedRoot } + if (source === 'partial') { + throw new Error( + `Incomplete Sim source checkout at ${resolvedRoot}; expected package.json plus apps/sim, apps/realtime, and packages/db package manifests.` + ) + } + return { + kind: 'standalone', + root: resolvedRoot, + existing: isStandaloneInstall(resolvedRoot), + } +} + /** Resolves the filesystem context before setup reads or writes any installation state. */ export function resolveSetupContext( start: string = process.cwd(), @@ -76,18 +93,7 @@ export function resolveSetupContext( const searchStart = path.resolve(start, override ?? '.') if (override) { - const source = inspectSourceRoot(searchStart) - if (source === 'valid') return { kind: 'source', root: searchStart } - if (source === 'partial') { - throw new Error( - `Incomplete Sim source checkout at ${searchStart}; expected package.json plus apps/sim, apps/realtime, and packages/db package manifests.` - ) - } - return { - kind: 'standalone', - root: searchStart, - existing: isStandaloneInstall(searchStart), - } + return resolveSetupContextAtRoot(searchStart) } for (const candidate of parentDirectories(searchStart)) { diff --git a/packages/sim-setup/src/lifecycle.test.ts b/packages/sim-setup/src/lifecycle.test.ts index a0dbfbbf41c..2cca78faeea 100644 --- a/packages/sim-setup/src/lifecycle.test.ts +++ b/packages/sim-setup/src/lifecycle.test.ts @@ -1,5 +1,10 @@ +import { createHash } from 'node:crypto' +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import path from 'node:path' import { describe, expect, it } from 'vitest' -import { getComposeUpdateMode, isLifecycleCommand } from './lifecycle' +import { ensureProductionComposeFile } from './compose-asset' +import { getComposeUpdateMode, isLifecycleCommand, refreshComposeFileForUpdate } from './lifecycle' describe('setup lifecycle', () => { it('recognizes update as a lifecycle command', () => { @@ -11,4 +16,26 @@ describe('setup lifecycle', () => { expect(getComposeUpdateMode('/repo/docker-compose.local.yml')).toBe('build') expect(() => getComposeUpdateMode('/repo/compose.yml')).toThrow(/Unsupported Sim Compose file/) }) + + it('refreshes a discovered standalone install outside the current setup context', () => { + const root = mkdtempSync(path.join(tmpdir(), 'sim-setup-lifecycle-')) + try { + const composeFile = ensureProductionComposeFile({ kind: 'standalone', root, existing: false }) + const packaged = readFileSync(composeFile, 'utf8') + const previous = `${packaged}\nservices: {}\n` + writeFileSync(composeFile, previous) + writeFileSync( + path.join(root, '.sim-setup.json'), + JSON.stringify({ + schemaVersion: 1, + composeSha256: createHash('sha256').update(previous).digest('hex'), + }) + ) + + expect(refreshComposeFileForUpdate(composeFile, root)).toBe(composeFile) + expect(readFileSync(composeFile, 'utf8')).toBe(packaged) + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) }) diff --git a/packages/sim-setup/src/lifecycle.ts b/packages/sim-setup/src/lifecycle.ts index 2135d6acb80..4a2a6500804 100644 --- a/packages/sim-setup/src/lifecycle.ts +++ b/packages/sim-setup/src/lifecycle.ts @@ -2,7 +2,7 @@ import { spawnSync } from 'node:child_process' import { readFileSync } from 'node:fs' import path from 'node:path' import { ensureProductionComposeFile } from './compose-asset' -import { SETUP_CONTEXT } from './context' +import { resolveSetupContextAtRoot } from './context' import { DB_CONTAINER, type Detection, REDIS_CONTAINER, runDetection } from './detect' import { archiveEnvFile, archiveFile, ROOT } from './env-files' import { SetupError } from './errors' @@ -348,6 +348,12 @@ export function getComposeUpdateMode(file: string): ComposeUpdateMode { throw new Error(`Unsupported Sim Compose file: ${file}`) } +/** Refreshes a published install's managed Compose file before applying an update. */ +export function refreshComposeFileForUpdate(file: string, dir: string): string { + if (getComposeUpdateMode(file) === 'build') return file + return ensureProductionComposeFile(resolveSetupContextAtRoot(dir)) +} + function update(install: Install): void { if (install.kind === 'dev') { throw new SetupError('update is only available for Docker Compose installs.', [ @@ -363,12 +369,7 @@ function update(install: Install): void { const mode = getComposeUpdateMode(install.file) const spin = p.spinner() if (mode === 'pull') { - if ( - SETUP_CONTEXT.kind === 'standalone' && - path.resolve(SETUP_CONTEXT.root) === path.resolve(install.dir) - ) { - install.file = ensureProductionComposeFile(SETUP_CONTEXT) - } + install.file = refreshComposeFileForUpdate(install.file, install.dir) spin.start('Pulling configured Sim images…') dockerRun(composeArgs(install, 'pull'), 'docker compose pull failed', install.dir) } else { From 99441fd57261fcd2b7e13150ba1627234deda5ec Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 19 Aug 2026 11:23:16 -0700 Subject: [PATCH 3/7] improvement(setup): unify repository command --- README.md | 2 +- package.json | 4 +-- packages/sim-setup/README.md | 3 ++- packages/sim-setup/src/index.ts | 46 +++++++++++---------------------- 4 files changed, 19 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 0e506fb3128..43bddedc719 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ Open [http://localhost:3000](http://localhost:3000) When it finishes, open [http://localhost:3000](http://localhost:3000). -Run the same command inside a cloned Sim repository to unlock the source-only local development and Kubernetes modes. The existing `bun run setup` and `bun run sim` commands remain available to contributors. +Inside a cloned Sim repository, run `bun run sim-setup` to unlock the source-only local development and Kubernetes modes. Reconfigure an optional capability without rerunning the full wizard: diff --git a/package.json b/package.json index 99e5f832bcf..587ef6854bc 100644 --- a/package.json +++ b/package.json @@ -84,9 +84,7 @@ "library:covers": "bun run scripts/generate-library-covers.tsx", "library:covers:check": "bun run scripts/generate-library-covers.tsx --check", "skills:sync": "bun run scripts/sync-skills.ts", - "setup": "bun run packages/sim-setup/src/index.ts setup", - "sim": "bun run packages/sim-setup/src/index.ts", - "doctor": "bun run packages/sim-setup/src/index.ts doctor", + "sim-setup": "bun run packages/sim-setup/src/index.ts", "agent-stream-docs:generate": "bun run scripts/sync-agent-stream-docs.ts", "agent-stream-docs:check": "bun run scripts/sync-agent-stream-docs.ts --check", "prepare": "bun husky", diff --git a/packages/sim-setup/README.md b/packages/sim-setup/README.md index 5208cb148b5..cfa077991bd 100644 --- a/packages/sim-setup/README.md +++ b/packages/sim-setup/README.md @@ -7,7 +7,8 @@ npx @sim/setup ``` Outside a Sim source checkout, the command creates a Docker Compose installation using published -images. Inside a Sim source checkout, it exposes the complete development and deployment wizard. +images. Inside a Sim source checkout, use `bun run sim-setup` to expose the complete development +and deployment wizard. By default, a standalone installation is written to `./sim`. Use `--dir ` to choose a different directory. The `sim` npm package remains the Sim API CLI; this package intentionally diff --git a/packages/sim-setup/src/index.ts b/packages/sim-setup/src/index.ts index c21ddf40ee3..050482a443a 100644 --- a/packages/sim-setup/src/index.ts +++ b/packages/sim-setup/src/index.ts @@ -38,20 +38,17 @@ const USAGE = `Usage: npx @sim/setup down remove containers (data kept) npx @sim/setup reset archive .env + wipe managed data -Inside a Sim source checkout, the existing commands remain available: - bun run setup run the setup wizard - bun run setup status show configured capabilities and integrations - bun run setup configure ${SETUP_FEATURES} - bun run sim setup [--quick] [--mode compose|dev|k8s] - bun run sim config show configured capabilities and integrations - bun run sim add configure one feature - bun run sim doctor [--fix] [--json] check your setup - bun run sim start | stop | restart bring your install up / down / cycle - bun run sim update pull/rebuild and apply Compose images - bun run sim status what's installed and healthy - bun run sim logs follow logs - bun run sim down remove containers (data kept) - bun run sim reset archive .env + wipe managed data` +Inside a Sim source checkout, use the repository command: + bun run sim-setup [--quick] [--mode compose|dev|k8s] + bun run sim-setup config show configured capabilities and integrations + bun run sim-setup add configure ${SETUP_FEATURES} + bun run sim-setup doctor [--fix] [--json] check your setup + bun run sim-setup start | stop | restart bring your install up / down / cycle + bun run sim-setup update pull/rebuild and apply Compose images + bun run sim-setup status what's installed and healthy + bun run sim-setup logs follow logs + bun run sim-setup down remove containers (data kept) + bun run sim-setup reset archive .env + wipe managed data` function readPackageVersion(): string { const metadata: unknown = JSON.parse( @@ -133,25 +130,12 @@ async function main(): Promise { return } - if (!command || command === 'setup' || command.startsWith('-')) { - const setupArgs = command === 'setup' ? args.slice(1) : args - const feature = setupArgs[0]?.startsWith('-') ? undefined : setupArgs[0] - if (feature === 'status') { - const { runSetupStatus } = await import('./setup-status') - process.exitCode = await runSetupStatus() - return - } - if (feature) { - const featureIndex = setupArgs.indexOf(feature) - const { runFeatureSetup } = await import('./feature-setup') - await runFeatureSetup(feature, setupArgs.slice(featureIndex + 1)) - return - } - const modeIdx = setupArgs.indexOf('--mode') + if (!command || command.startsWith('-')) { + const modeIdx = args.indexOf('--mode') const { runWizard } = await import('./wizard') await runWizard({ - quick: setupArgs.includes('--quick'), - mode: modeIdx === -1 ? undefined : parseMode(setupArgs[modeIdx + 1]), + quick: args.includes('--quick'), + mode: modeIdx === -1 ? undefined : parseMode(args[modeIdx + 1]), }) return } From 9a7088494260fba92398e72bc822e24c057d9dcc Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 19 Aug 2026 12:00:50 -0700 Subject: [PATCH 4/7] fix(setup): harden standalone package launch --- .github/workflows/publish-sim-setup.yml | 28 +-- packages/sim-setup/src/arguments.test.ts | 84 +++++++ packages/sim-setup/src/arguments.ts | 174 ++++++++++++++ packages/sim-setup/src/atomic-file.test.ts | 40 ++++ packages/sim-setup/src/atomic-file.ts | 32 +++ packages/sim-setup/src/cli-auth.test.ts | 132 +++++++++++ packages/sim-setup/src/cli-auth.ts | 257 +++++++++++++++------ packages/sim-setup/src/env-files.test.ts | 27 ++- packages/sim-setup/src/env-files.ts | 11 +- packages/sim-setup/src/index.ts | 105 ++------- packages/sim-setup/src/version.ts | 22 ++ 11 files changed, 743 insertions(+), 169 deletions(-) create mode 100644 packages/sim-setup/src/arguments.test.ts create mode 100644 packages/sim-setup/src/arguments.ts create mode 100644 packages/sim-setup/src/atomic-file.test.ts create mode 100644 packages/sim-setup/src/atomic-file.ts create mode 100644 packages/sim-setup/src/cli-auth.test.ts create mode 100644 packages/sim-setup/src/version.ts diff --git a/.github/workflows/publish-sim-setup.yml b/.github/workflows/publish-sim-setup.yml index 0d98c2cd5f4..bd1bfc3db40 100644 --- a/.github/workflows/publish-sim-setup.yml +++ b/.github/workflows/publish-sim-setup.yml @@ -137,21 +137,30 @@ jobs: cd "$SMOKE_DIR" node package/dist/index.js --version node package/dist/index.js --help + for command in config add doctor start stop restart update status logs down reset; do + node package/dist/index.js "$command" --help > /dev/null + done + node package/dist/index.js add integration --help > /dev/null + if node package/dist/index.js --quik > /dev/null 2>&1; then + echo 'Packed setup CLI accepted an unknown option.' >&2 + exit 1 + fi + if node package/dist/index.js start extra > /dev/null 2>&1; then + echo 'Packed setup CLI accepted an extra lifecycle operand.' >&2 + exit 1 + fi - - name: Check if version already exists - id: version_check + - name: Verify version is unpublished working-directory: packages/sim-setup env: VERSION: ${{ steps.release.outputs.version }} run: | if bun pm view "@sim/setup@$VERSION" version > /dev/null 2>&1; then - echo "exists=true" >> "$GITHUB_OUTPUT" - else - echo "exists=false" >> "$GITHUB_OUTPUT" + echo "@sim/setup@$VERSION is already published. Bump packages/sim-setup/package.json before releasing another build." >&2 + exit 1 fi - name: Publish to npm - if: steps.version_check.outputs.exists == 'false' working-directory: packages/sim-setup env: NPM_CONFIG_TOKEN: ${{ secrets.NPM_TOKEN }} @@ -159,14 +168,7 @@ jobs: run: bun publish --access public --tag "$NPM_TAG" --no-save - name: Summarize release - if: steps.version_check.outputs.exists == 'false' env: VERSION: ${{ steps.release.outputs.version }} NPM_TAG: ${{ steps.release.outputs.tag }} run: echo "Published @sim/setup@$VERSION with the '$NPM_TAG' tag." - - - name: Summarize skipped release - if: steps.version_check.outputs.exists == 'true' - env: - VERSION: ${{ steps.release.outputs.version }} - run: echo "Skipped @sim/setup@$VERSION because that version is already published." diff --git a/packages/sim-setup/src/arguments.test.ts b/packages/sim-setup/src/arguments.test.ts new file mode 100644 index 00000000000..e0630b62434 --- /dev/null +++ b/packages/sim-setup/src/arguments.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from 'vitest' +import { LIFECYCLE_COMMANDS, parseSetupArguments } from './arguments' + +describe('parseSetupArguments', () => { + it('parses wizard options in either supported value form', () => { + expect(parseSetupArguments(['--quick', '--mode', 'compose', '--dir', 'install'])).toEqual({ + kind: 'wizard', + quick: true, + mode: 'compose', + }) + expect(parseSetupArguments(['--mode=k8s', '--dir=install'])).toEqual({ + kind: 'wizard', + quick: false, + mode: 'k8s', + }) + }) + + it('rejects unknown and duplicate wizard options', () => { + expect(() => parseSetupArguments(['--quik'])).toThrow('Unknown setup option: --quik') + expect(() => parseSetupArguments(['--quick', '--quick'])).toThrow( + '--quick may only be provided once' + ) + expect(() => parseSetupArguments(['--mode', 'production'])).toThrow( + 'expected compose, dev, or k8s' + ) + }) + + it('requires one unambiguous directory override', () => { + expect(() => parseSetupArguments(['--dir'])).toThrow('--dir requires a directory path') + expect(() => parseSetupArguments(['--dir='])).toThrow('--dir requires a directory path') + expect(() => parseSetupArguments(['--dir=a', '--dir', 'b'])).toThrow( + '--dir may only be provided once' + ) + }) + + it('rejects extra lifecycle and configuration arguments', () => { + for (const command of [...LIFECYCLE_COMMANDS, 'config'] as const) { + expect(() => parseSetupArguments([command, 'extra'])).toThrow('does not accept: extra') + } + expect(() => parseSetupArguments(['doctor', '--json', '--unknown'])).toThrow( + 'Unknown doctor option: --unknown' + ) + }) + + it('validates add operands before loading configuration', () => { + expect(parseSetupArguments(['add', 'email'])).toEqual({ + kind: 'add', + feature: 'email', + args: [], + }) + expect(parseSetupArguments(['add', 'integration', 'slack'])).toEqual({ + kind: 'add', + feature: 'integration', + args: ['slack'], + }) + expect(() => parseSetupArguments(['add'])).toThrow('add requires a feature') + expect(() => parseSetupArguments(['add', 'integration'])).toThrow( + 'requires exactly one integration slug' + ) + expect(() => parseSetupArguments(['add', 'email', 'extra'])).toThrow( + 'add email does not accept: extra' + ) + }) + + it('allows help for every top-level command without executing it', () => { + expect(parseSetupArguments(['--help'])).toEqual({ kind: 'help' }) + expect(parseSetupArguments(['add', '--help'])).toEqual({ kind: 'help' }) + expect(parseSetupArguments(['add', 'integration', '--help'])).toEqual({ kind: 'help' }) + expect(parseSetupArguments(['doctor', '--help'])).toEqual({ kind: 'help' }) + for (const command of [...LIFECYCLE_COMMANDS, 'config'] as const) { + expect(parseSetupArguments([command, '--help'])).toEqual({ kind: 'help' }) + } + }) + + it('keeps version root-only and rejects conflicting global flags', () => { + expect(parseSetupArguments(['--version'])).toEqual({ kind: 'version' }) + expect(() => parseSetupArguments(['doctor', '--version'])).toThrow( + '--version does not accept: doctor' + ) + expect(() => parseSetupArguments(['--help', '--version'])).toThrow( + '--help and --version cannot be combined' + ) + }) +}) diff --git a/packages/sim-setup/src/arguments.ts b/packages/sim-setup/src/arguments.ts new file mode 100644 index 00000000000..65a9f2accd2 --- /dev/null +++ b/packages/sim-setup/src/arguments.ts @@ -0,0 +1,174 @@ +export type WizardMode = 'compose' | 'dev' | 'k8s' + +export const LIFECYCLE_COMMANDS = [ + 'start', + 'stop', + 'restart', + 'update', + 'status', + 'logs', + 'down', + 'reset', +] as const + +export type LifecycleCommand = (typeof LIFECYCLE_COMMANDS)[number] + +export type SetupInvocation = + | { kind: 'help' } + | { kind: 'version' } + | { kind: 'wizard'; quick: boolean; mode?: WizardMode } + | { kind: 'config' } + | { kind: 'add'; feature: string; args: string[] } + | { kind: 'doctor'; fix: boolean; json: boolean } + | { kind: 'lifecycle'; command: LifecycleCommand } + +export class SetupArgumentError extends Error { + constructor(message: string) { + super(message) + this.name = 'SetupArgumentError' + } +} + +function isLifecycleCommand(value: string | undefined): value is LifecycleCommand { + return Boolean(value && (LIFECYCLE_COMMANDS as readonly string[]).includes(value)) +} + +function fail(message: string): never { + throw new SetupArgumentError(message) +} + +function stripDirectoryOption(args: readonly string[]): string[] { + const filtered: string[] = [] + let found = false + + for (let index = 0; index < args.length; index += 1) { + const arg = args[index] + if (arg === '--dir' || arg.startsWith('--dir=')) { + if (found) fail('--dir may only be provided once') + found = true + + if (arg === '--dir') { + const value = args[index + 1] + if (!value || value.startsWith('-')) fail('--dir requires a directory path') + index += 1 + } else if (!arg.slice('--dir='.length)) { + fail('--dir requires a directory path') + } + continue + } + filtered.push(arg) + } + + return filtered +} + +function oneFlag(args: readonly string[], flag: string): boolean { + const count = args.filter((arg) => arg === flag).length + if (count > 1) fail(`${flag} may only be provided once`) + return count === 1 +} + +function parseMode(args: readonly string[]): { mode?: WizardMode; remaining: string[] } { + let mode: WizardMode | undefined + const remaining: string[] = [] + + for (let index = 0; index < args.length; index += 1) { + const arg = args[index] + if (arg !== '--mode' && !arg.startsWith('--mode=')) { + remaining.push(arg) + continue + } + if (mode) fail('--mode may only be provided once') + + const value = arg === '--mode' ? args[++index] : arg.slice('--mode='.length) + if (value !== 'compose' && value !== 'dev' && value !== 'k8s') { + fail(`invalid --mode "${value ?? ''}" — expected compose, dev, or k8s`) + } + mode = value + } + + return { mode, remaining } +} + +function expectNoArguments(command: string, args: readonly string[]): void { + if (args.length > 0) fail(`${command} does not accept: ${args.join(' ')}`) +} + +function parseCore( + args: readonly string[], + helpRequested: boolean +): Exclude { + const command = args[0] + + if (!command || command.startsWith('-')) { + const quick = oneFlag(args, '--quick') + const withoutQuick = args.filter((arg) => arg !== '--quick') + const { mode, remaining } = parseMode(withoutQuick) + if (remaining.length > 0) fail(`Unknown setup option: ${remaining[0]}`) + return { kind: 'wizard', quick, ...(mode ? { mode } : {}) } + } + + const commandArgs = args.slice(1) + if (command === 'config') { + expectNoArguments(command, commandArgs) + return { kind: 'config' } + } + + if (command === 'add') { + const feature = commandArgs[0] + if (!feature) { + if (helpRequested) return { kind: 'add', feature: '', args: [] } + fail('add requires a feature') + } + if (feature.startsWith('-')) fail(`Unknown add option: ${feature}`) + + const featureArgs = commandArgs.slice(1) + if (feature === 'integration') { + if (featureArgs.length === 0 && helpRequested) { + return { kind: 'add', feature, args: [] } + } + if (featureArgs.length !== 1 || featureArgs[0].startsWith('-')) { + fail('add integration requires exactly one integration slug') + } + } else if (featureArgs.length > 0) { + fail(`add ${feature} does not accept: ${featureArgs.join(' ')}`) + } + return { kind: 'add', feature, args: featureArgs } + } + + if (command === 'doctor') { + const fix = oneFlag(commandArgs, '--fix') + const json = oneFlag(commandArgs, '--json') + const remaining = commandArgs.filter((arg) => arg !== '--fix' && arg !== '--json') + if (remaining.length > 0) fail(`Unknown doctor option: ${remaining[0]}`) + return { kind: 'doctor', fix, json } + } + + if (isLifecycleCommand(command)) { + expectNoArguments(command, commandArgs) + return { kind: 'lifecycle', command } + } + + fail(`Unknown command: ${command}`) +} + +/** Parses and validates the complete public command surface before any setup work begins. */ +export function parseSetupArguments(rawArgs: readonly string[]): SetupInvocation { + const args = stripDirectoryOption(rawArgs) + const helpCount = args.filter((arg) => arg === '--help' || arg === '-h').length + if (helpCount > 1) fail('--help may only be provided once') + const versionCount = args.filter((arg) => arg === '--version' || arg === '-V').length + if (versionCount > 1) fail('--version may only be provided once') + if (helpCount > 0 && versionCount > 0) fail('--help and --version cannot be combined') + + if (versionCount === 1) { + const remaining = args.filter((arg) => arg !== '--version' && arg !== '-V') + expectNoArguments('--version', remaining) + return { kind: 'version' } + } + + const helpRequested = helpCount === 1 + const withoutHelp = args.filter((arg) => arg !== '--help' && arg !== '-h') + const invocation = parseCore(withoutHelp, helpRequested) + return helpRequested ? { kind: 'help' } : invocation +} diff --git a/packages/sim-setup/src/atomic-file.test.ts b/packages/sim-setup/src/atomic-file.test.ts new file mode 100644 index 00000000000..c6e28bd8545 --- /dev/null +++ b/packages/sim-setup/src/atomic-file.test.ts @@ -0,0 +1,40 @@ +import { + chmodSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { writeTextFileAtomic } from './atomic-file' + +const roots: string[] = [] + +function temporaryRoot(): string { + const root = mkdtempSync(path.join(tmpdir(), 'sim-setup-atomic-file-')) + roots.push(root) + return root +} + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }) +}) + +describe('writeTextFileAtomic', () => { + it('replaces the complete file with the requested permissions and no staged residue', () => { + const root = temporaryRoot() + const file = path.join(root, '.env') + writeFileSync(file, 'OLD=value\n') + chmodSync(file, 0o644) + + writeTextFileAtomic(file, 'SECRET=current\n', { mode: 0o600 }) + + expect(readFileSync(file, 'utf8')).toBe('SECRET=current\n') + expect(statSync(file).mode & 0o777).toBe(0o600) + expect(readdirSync(root)).toEqual(['.env']) + }) +}) diff --git a/packages/sim-setup/src/atomic-file.ts b/packages/sim-setup/src/atomic-file.ts new file mode 100644 index 00000000000..ed06736f435 --- /dev/null +++ b/packages/sim-setup/src/atomic-file.ts @@ -0,0 +1,32 @@ +import { existsSync, mkdirSync, renameSync, unlinkSync, writeFileSync } from 'node:fs' +import path from 'node:path' +import { generateShortId } from '@sim/utils/id' + +interface AtomicWriteOptions { + mode: number +} + +/** Writes a complete replacement beside its target, then publishes it with one rename. */ +export function writeTextFileAtomic( + filePath: string, + contents: string, + options: AtomicWriteOptions +): void { + const directory = path.dirname(filePath) + mkdirSync(directory, { recursive: true }) + const temporaryPath = path.join( + directory, + `.${path.basename(filePath)}.${process.pid}.${generateShortId(10)}.tmp` + ) + + try { + writeFileSync(temporaryPath, contents, { + encoding: 'utf8', + flag: 'wx', + mode: options.mode, + }) + renameSync(temporaryPath, filePath) + } finally { + if (existsSync(temporaryPath)) unlinkSync(temporaryPath) + } +} diff --git a/packages/sim-setup/src/cli-auth.test.ts b/packages/sim-setup/src/cli-auth.test.ts new file mode 100644 index 00000000000..36ae5172b22 --- /dev/null +++ b/packages/sim-setup/src/cli-auth.test.ts @@ -0,0 +1,132 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { buildApprovalUrl, normalizeAuthOrigin, pollOnce } from './cli-auth' +import { SETUP_USER_AGENT } from './version' + +const ORIGIN = 'https://www.sim.test/prefix' +const REQUEST = 'request' +const VERIFIER = 'verifier' + +function jsonResponse(status: number, body: unknown, headers?: Record): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json', ...headers }, + }) +} + +afterEach(() => { + vi.restoreAllMocks() +}) + +describe('normalizeAuthOrigin', () => { + it('keeps a path prefix and removes only trailing slashes', () => { + expect(normalizeAuthOrigin('https://www.sim.test/prefix///')).toBe(ORIGIN) + }) + + it('rejects malformed and ambiguous service roots', () => { + expect(() => normalizeAuthOrigin('not-a-url')).toThrow('absolute HTTP(S) URL') + expect(() => normalizeAuthOrigin('ftp://sim.test')).toThrow('expected HTTP or HTTPS') + expect(() => normalizeAuthOrigin('https://sim.test?target=other')).toThrow( + 'cannot contain credentials, a query, or a fragment' + ) + }) +}) + +describe('buildApprovalUrl', () => { + it('preserves the origin prefix and never includes the poll secret', () => { + const url = buildApprovalUrl(ORIGIN, REQUEST, 'challenge', 'ABCD-2345') + expect(url).toMatch(/^https:\/\/www\.sim\.test\/prefix\/cli\/auth\?/) + expect(url).toContain('challenge=challenge') + expect(url).not.toContain(VERIFIER) + }) +}) + +describe('pollOnce', () => { + it('bounds and identifies the request without following redirects', async () => { + const fetchMock = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValue(jsonResponse(200, { status: 'complete', key: { apiKey: 'key' } })) + + await expect(pollOnce(ORIGIN, REQUEST, VERIFIER)).resolves.toEqual({ + status: 'complete', + apiKey: 'key', + }) + expect(fetchMock.mock.calls[0][0]).toBe(`${ORIGIN}/api/cli/auth/poll`) + expect(fetchMock.mock.calls[0][1]).toMatchObject({ + method: 'POST', + redirect: 'manual', + headers: expect.objectContaining({ + accept: 'application/json', + 'user-agent': SETUP_USER_AGENT, + }), + signal: expect.any(AbortSignal), + }) + }) + + it('keeps pending, rate-limited, server, and transport failures retryable', async () => { + const fetchMock = vi.spyOn(globalThis, 'fetch') + fetchMock.mockResolvedValueOnce(jsonResponse(200, { status: 'pending' })) + await expect(pollOnce(ORIGIN, REQUEST, VERIFIER)).resolves.toEqual({ status: 'pending' }) + + fetchMock.mockResolvedValueOnce( + jsonResponse(429, { error: 'slow down' }, { 'retry-after': '4' }) + ) + await expect(pollOnce(ORIGIN, REQUEST, VERIFIER)).resolves.toEqual({ + status: 'pending', + retryAfterMs: 4000, + }) + + fetchMock.mockResolvedValueOnce(jsonResponse(503, { error: 'deploying' })) + await expect(pollOnce(ORIGIN, REQUEST, VERIFIER)).resolves.toEqual({ status: 'pending' }) + + fetchMock.mockResolvedValueOnce(jsonResponse(409, { error: 'name collision' })) + await expect(pollOnce(ORIGIN, REQUEST, VERIFIER)).resolves.toEqual({ status: 'pending' }) + + fetchMock.mockRejectedValueOnce(new TypeError('fetch failed')) + await expect(pollOnce(ORIGIN, REQUEST, VERIFIER)).resolves.toEqual({ status: 'pending' }) + + fetchMock.mockRejectedValueOnce(new DOMException('timed out', 'TimeoutError')) + await expect(pollOnce(ORIGIN, REQUEST, VERIFIER)).resolves.toEqual({ status: 'pending' }) + }) + + it('fails immediately on deliberate refusals and unexpected transport errors', async () => { + const fetchMock = vi.spyOn(globalThis, 'fetch') + fetchMock.mockResolvedValueOnce(jsonResponse(403, { error: { message: 'Forbidden' } })) + await expect(pollOnce(ORIGIN, REQUEST, VERIFIER)).rejects.toThrow('Forbidden') + + fetchMock.mockRejectedValueOnce(new Error('programming failure')) + await expect(pollOnce(ORIGIN, REQUEST, VERIFIER)).rejects.toThrow('programming failure') + }) + + it('refuses redirects instead of forwarding the poll secret', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response(null, { + status: 301, + headers: { location: 'https://other.sim.test/api/cli/auth/poll' }, + }) + ) + + await expect(pollOnce(ORIGIN, REQUEST, VERIFIER)).rejects.toThrow( + 'will not forward the poll secret across a redirect' + ) + }) + + it('rejects non-JSON and malformed successful responses', async () => { + const fetchMock = vi.spyOn(globalThis, 'fetch') + fetchMock.mockResolvedValueOnce(new Response('wrong host', { status: 200 })) + await expect(pollOnce(ORIGIN, REQUEST, VERIFIER)).rejects.toThrow('non-JSON response') + + fetchMock.mockResolvedValueOnce(jsonResponse(200, { status: 'complete' })) + await expect(pollOnce(ORIGIN, REQUEST, VERIFIER)).rejects.toThrow( + 'completed without a valid API key' + ) + }) + + it('rejects an invalid request timeout before calling fetch', async () => { + const fetchMock = vi.spyOn(globalThis, 'fetch') + + await expect(pollOnce(ORIGIN, REQUEST, VERIFIER, 0)).rejects.toThrow( + 'Poll timeout must be a positive whole number' + ) + expect(fetchMock).not.toHaveBeenCalled() + }) +}) diff --git a/packages/sim-setup/src/cli-auth.ts b/packages/sim-setup/src/cli-auth.ts index 9ec7700a230..a8f230c3335 100644 --- a/packages/sim-setup/src/cli-auth.ts +++ b/packages/sim-setup/src/cli-auth.ts @@ -3,25 +3,29 @@ import { sha256Base64Url } from '@sim/security/hash' import { generateSecureToken } from '@sim/security/tokens' import { sleep } from '@sim/utils/helpers' import { generateShortId } from '@sim/utils/id' +import { isRecordLike } from '@sim/utils/object' import { parseRetryAfter } from '@sim/utils/retry' +import { truncate } from '@sim/utils/string' import * as p from './prompter' import { link, theme } from './theme' +import { SETUP_USER_AGENT } from './version' -// Generous enough for a first-time user to create an account, wait for the email -// OTP, land back on /cli/auth, and approve — a few minutes is routine. The -// server-side approval record has its own short TTL, so a long client wait only -// costs cheap, rate-limited polls. const WAIT_MS = 900_000 const POLL_INTERVAL_MS = 2000 +const POLL_REQUEST_TIMEOUT_MS = 15_000 +const APPROVAL_PATH = '/cli/auth' +const POLL_PATH = '/api/cli/auth/poll' +const RETRYABLE_POLL_STATUSES = new Set([409, 429, 500, 502, 503, 504]) +const PAIRING_ALPHABET = 'ABCDEFGHJKLMNPQRSTUVWXYZ23456789' + +export type AuthPollResult = + | { status: 'pending'; retryAfterMs?: number } + | { status: 'complete'; apiKey: string } +/** Opens a safely encoded URL across platforms, including Windows' cmd-backed `start`. */ function openBrowser(url: string): void { if (process.env.SIM_SETUP_NO_BROWSER) return if (process.platform === 'win32') { - // `start` is a cmd builtin, not an executable — spawning it directly ENOENTs. - // cmd re-parses the command line and would treat `&` in the query string as a - // command separator, truncating the URL; quote it (the query is URL-encoded, so - // it never contains a `"`) and pass args verbatim so Node doesn't re-quote them. - // `""` is start's window-title placeholder, required before the URL. spawnSync('cmd', ['/c', 'start', '""', `"${url}"`], { stdio: 'ignore', windowsVerbatimArguments: true, @@ -32,95 +36,212 @@ function openBrowser(url: string): void { spawnSync(command, [url], { stdio: 'ignore' }) } -/** No O/0 or I/1 — this exists to be compared by eye against a browser tab. */ -const PAIRING_ALPHABET = 'ABCDEFGHJKLMNPQRSTUVWXYZ23456789' - -/** - * Short human-comparable code, shown in this terminal and on the approval page. - * - * The poll secret binds the *key* to this process, but nothing cryptographic - * tells the user whether the page they're approving belongs to their terminal - * or to a link someone sent them — an attacker supplies the request id and - * challenge. Comparing this code is the only thing that distinguishes the two. - */ +/** Short human-comparable code with no look-alike characters. */ function createPairingCode(): string { const chars = generateShortId(8, PAIRING_ALPHABET) return `${chars.slice(0, 4)}-${chars.slice(4)}` } -interface PollResponse { - status: 'pending' | 'complete' - key?: { apiKey?: string } +/** Validates the auth service root and preserves any path prefix it carries. */ +export function normalizeAuthOrigin(origin: string): string { + const value = origin.trim() + let parsed: URL + try { + parsed = new URL(value) + } catch { + throw new Error( + `Invalid Chat authorization origin "${origin}"; expected an absolute HTTP(S) URL.` + ) + } + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + throw new Error( + `Unsupported Chat authorization origin scheme "${parsed.protocol.replace(/:$/, '')}"; expected HTTP or HTTPS.` + ) + } + if (parsed.username || parsed.password || parsed.search || parsed.hash) { + throw new Error('Chat authorization origin cannot contain credentials, a query, or a fragment.') + } + const prefix = parsed.pathname.replace(/\/+$/, '') + return `${parsed.origin}${prefix}` +} + +function authUrl(origin: string, path: string): string { + return `${normalizeAuthOrigin(origin)}${path}` +} + +/** Builds the browser URL without ever including the redeemable poll secret. */ +export function buildApprovalUrl( + origin: string, + request: string, + challenge: string, + pairing: string +): string { + const query = new URLSearchParams({ request, challenge, pairing }) + return `${authUrl(origin, APPROVAL_PATH)}?${query}` +} + +function isRetryableTransportError(error: unknown): boolean { + if (error instanceof TypeError) return true + return ( + typeof DOMException !== 'undefined' && + error instanceof DOMException && + error.name === 'TimeoutError' + ) +} + +async function responseError(response: Response): Promise { + const fallback = `Chat authorization failed with HTTP ${response.status}` + const raw = await response.text() + if (!raw) return fallback + + let body: unknown + try { + body = JSON.parse(raw) + } catch { + return fallback + } + if (!isRecordLike(body)) return fallback + if (typeof body.error === 'string' && body.error) return truncate(body.error, 500) + if (isRecordLike(body.error) && typeof body.error.message === 'string' && body.error.message) { + return truncate(body.error.message, 500) + } + if (typeof body.message === 'string' && body.message) return truncate(body.message, 500) + return fallback +} + +function redirectError(origin: string, response: Response): Error { + const location = response.headers.get('location')?.trim() + if (!location) { + return new Error( + `Chat authorization origin ${normalizeAuthOrigin(origin)} returned HTTP ${response.status} without a redirect target.` + ) + } + + let target: URL + try { + target = new URL(location, `${normalizeAuthOrigin(origin)}/`) + } catch { + return new Error( + `Chat authorization origin ${normalizeAuthOrigin(origin)} returned HTTP ${response.status} with an invalid redirect target.` + ) + } + return new Error( + `Chat authorization origin redirected the key poll to ${target.href}. Set SIM_CLI_AUTH_ORIGIN to the final service URL; setup will not forward the poll secret across a redirect.` + ) +} + +function parsePollResponse(raw: string): AuthPollResult { + let body: unknown + try { + body = JSON.parse(raw) + } catch { + throw new Error('Chat authorization service returned a non-JSON response.') + } + if (!isRecordLike(body) || (body.status !== 'pending' && body.status !== 'complete')) { + throw new Error('Chat authorization service returned an invalid poll response.') + } + if (body.status === 'pending') return { status: 'pending' } + if (!isRecordLike(body.key) || typeof body.key.apiKey !== 'string' || !body.key.apiKey) { + throw new Error('Chat authorization completed without a valid API key.') + } + return { status: 'complete', apiKey: body.key.apiKey } +} + +/** Performs one bounded poll, retrying only known transient transport and service failures. */ +export async function pollOnce( + origin: string, + request: string, + verifier: string, + timeoutMs: number = POLL_REQUEST_TIMEOUT_MS +): Promise { + if (!Number.isInteger(timeoutMs) || timeoutMs <= 0) { + throw new Error( + `Poll timeout must be a positive whole number of milliseconds, got ${timeoutMs}.` + ) + } + let response: Response + try { + response = await fetch(authUrl(origin, POLL_PATH), { + method: 'POST', + headers: { + accept: 'application/json', + 'content-type': 'application/json', + 'user-agent': SETUP_USER_AGENT, + }, + body: JSON.stringify({ request, verifier }), + signal: AbortSignal.timeout(timeoutMs), + redirect: 'manual', + }) + } catch (error) { + if (isRetryableTransportError(error)) return { status: 'pending' } + throw error + } + + if (response.status >= 300 && response.status <= 399) throw redirectError(origin, response) + if (RETRYABLE_POLL_STATUSES.has(response.status)) { + const retryAfterMs = + response.status === 429 ? parseRetryAfter(response.headers.get('retry-after')) : null + return { + status: 'pending', + ...(retryAfterMs && retryAfterMs > 0 ? { retryAfterMs } : {}), + } + } + if (!response.ok) throw new Error(await responseError(response)) + return parsePollResponse(await response.text()) } /** - * Device-flow handoff: open the approval page and poll for the key over TLS. + * Opens the approval page and polls for the Chat key without a loopback listener. * - * No loopback listener — the terminal and browser need not share a machine, so - * this works over SSH and inside containers. The poll secret never leaves this - * process; only its digest reaches the server, so an observer of the request id - * cannot mint. Returns the key, or null on timeout / failed poll (re-run to - * retry). Ctrl-C exits setup via the SIGINT handler. + * The poll secret never enters the browser URL. Fatal protocol and authorization + * responses stop immediately; only known transient failures remain pending. */ export async function browserKeyFlow(origin: string): Promise { + const normalizedOrigin = normalizeAuthOrigin(origin) const request = generateSecureToken(32) const pollSecret = generateSecureToken(32) const challenge = sha256Base64Url(pollSecret) const pairingCode = createPairingCode() - - const query = new URLSearchParams({ request, challenge, pairing: pairingCode }) - const authUrl = `${origin}/cli/auth?${query}` + const approvalUrl = buildApprovalUrl(normalizedOrigin, request, challenge, pairingCode) p.note( `${theme.heading(pairingCode)}\n\n${theme.muted('The page should show this code. If it shows a different one,\nthe request is not from this terminal — close the tab.')}`, 'Confirm this code in your browser' ) p.log.info( - `Opening your browser — create your account (or sign in) and approve; the key comes back automatically.\n If it doesn't open: ${link(authUrl, authUrl)}` + `Opening your browser — create your account (or sign in) and approve; the key comes back automatically.\n If it doesn't open: ${link(approvalUrl, approvalUrl)}` ) - openBrowser(authUrl) + openBrowser(approvalUrl) const spin = p.spinner() spin.start('Waiting for approval in your browser') const deadline = Date.now() + WAIT_MS - while (Date.now() < deadline) { - await sleep(POLL_INTERVAL_MS) - // null means still pending or a transient error — either way, keep waiting. - const key = await pollOnce(origin, request, pollSecret) - if (key) { - spin.stop('Approved') - return key + let delayMs = POLL_INTERVAL_MS + try { + while (Date.now() < deadline) { + const beforePollMs = deadline - Date.now() + await sleep(Math.min(delayMs, beforePollMs)) + const remainingMs = deadline - Date.now() + if (remainingMs <= 0) break + + const result = await pollOnce( + normalizedOrigin, + request, + pollSecret, + Math.min(POLL_REQUEST_TIMEOUT_MS, remainingMs) + ) + if (result.status === 'complete') { + spin.stop('Approved') + return result.apiKey + } + delayMs = result.retryAfterMs ?? POLL_INTERVAL_MS } + } catch (error) { + spin.stop('Browser handoff failed') + throw error } spin.stop('Browser handoff timed out') return null } - -/** - * One poll. Returns the key when the approval completes, `null` while pending or - * on a transient error (the caller keeps waiting until the deadline). - */ -async function pollOnce(origin: string, request: string, verifier: string): Promise { - try { - const response = await fetch(`${origin}/api/cli/auth/poll`, { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ request, verifier }), - }) - // Behind a shared NAT the per-IP bucket can be hit — honor Retry-After and - // back off instead of hammering the endpoint every interval. - if (response.status === 429) { - const retryMs = parseRetryAfter(response.headers.get('retry-after')) - if (retryMs) await sleep(retryMs) - return null - } - if (!response.ok) return null - - const data = (await response.json()) as PollResponse - return data.status === 'complete' ? (data.key?.apiKey ?? null) : null - } catch { - return null - } -} diff --git a/packages/sim-setup/src/env-files.test.ts b/packages/sim-setup/src/env-files.test.ts index 46888820a4c..f4d64a87439 100644 --- a/packages/sim-setup/src/env-files.test.ts +++ b/packages/sim-setup/src/env-files.test.ts @@ -1,12 +1,22 @@ -import { describe, expect, it } from 'vitest' +import { chmodSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' import { isPlaceholder, isUsableSecret, parseEnv, reconcileEnvContent, upsertEnv, + writeEnvFile, } from './env-files' +const roots: string[] = [] + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }) +}) + describe('placeholder detection', () => { it('recognizes underscore and hyphen template prefixes', () => { expect(isPlaceholder('your_secret_key')).toBe(true) @@ -56,3 +66,18 @@ describe('reconcileEnvContent', () => { expect(parseEnv(content).get('SMTP_HOST')).toBe('old-host') }) }) + +describe('writeEnvFile', () => { + it('replaces existing contents with owner-only permissions', () => { + const root = mkdtempSync(path.join(tmpdir(), 'sim-setup-env-file-')) + roots.push(root) + const file = path.join(root, '.env') + writeFileSync(file, 'OLD=value\n') + chmodSync(file, 0o644) + + writeEnvFile(file, 'SECRET=current\n') + + expect(readFileSync(file, 'utf8')).toBe('SECRET=current\n') + expect(statSync(file).mode & 0o777).toBe(0o600) + }) +}) diff --git a/packages/sim-setup/src/env-files.ts b/packages/sim-setup/src/env-files.ts index 3dbdd4b058e..686f4e6bf6d 100644 --- a/packages/sim-setup/src/env-files.ts +++ b/packages/sim-setup/src/env-files.ts @@ -1,6 +1,7 @@ -import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs' +import { existsSync, readFileSync, renameSync } from 'node:fs' import path from 'node:path' import { generateRandomHex } from '@sim/utils/random' +import { writeTextFileAtomic } from './atomic-file' import { SETUP_CONTEXT } from './context' export const ROOT = SETUP_CONTEXT.root @@ -147,8 +148,12 @@ export function reconcileEnvValues( const example = EXAMPLE_PATHS[target] content = example && existsSync(example) ? readFileSync(example, 'utf8') : '' } - mkdirSync(path.dirname(filePath), { recursive: true }) - writeFileSync(filePath, reconcileEnvContent(content, remove, values)) + writeEnvFile(filePath, reconcileEnvContent(content, remove, values)) +} + +/** Atomically replaces a secret-bearing env file with owner-only permissions. */ +export function writeEnvFile(filePath: string, contents: string): void { + writeTextFileAtomic(filePath, contents, { mode: 0o600 }) } /** Writes values into an env file, seeding a missing file from its .env.example. */ diff --git a/packages/sim-setup/src/index.ts b/packages/sim-setup/src/index.ts index 050482a443a..9863f4a8692 100644 --- a/packages/sim-setup/src/index.ts +++ b/packages/sim-setup/src/index.ts @@ -1,30 +1,14 @@ #!/usr/bin/env node -import { readFileSync } from 'node:fs' import { getErrorMessage } from '@sim/utils/errors' +import { parseSetupArguments, SetupArgumentError } from './arguments' import { SetupError } from './errors' import { exitWith, restoreTerminal } from './terminal' import { theme } from './theme' +import { SETUP_VERSION } from './version' -type WizardMode = 'compose' | 'dev' | 'k8s' - -const LIFECYCLE_COMMANDS = [ - 'start', - 'stop', - 'restart', - 'update', - 'status', - 'logs', - 'down', - 'reset', -] as const -type LifecycleCommand = (typeof LIFECYCLE_COMMANDS)[number] const SETUP_FEATURES = 'email | storage | sandbox | jobs | cache | knowledge | knowledge-embeddings | llm | integration ' -function isLifecycleCommand(value: string | undefined): value is LifecycleCommand { - return Boolean(value && (LIFECYCLE_COMMANDS as readonly string[]).includes(value)) -} - const USAGE = `Usage: npx @sim/setup run the setup wizard npx @sim/setup [--quick] [--dir ] create a Compose installation @@ -50,99 +34,47 @@ Inside a Sim source checkout, use the repository command: bun run sim-setup down remove containers (data kept) bun run sim-setup reset archive .env + wipe managed data` -function readPackageVersion(): string { - const metadata: unknown = JSON.parse( - readFileSync(new URL('../package.json', import.meta.url), 'utf8') - ) - if ( - typeof metadata !== 'object' || - metadata === null || - !('version' in metadata) || - typeof metadata.version !== 'string' - ) { - throw new Error('@sim/setup package metadata is missing a valid version') - } - return metadata.version -} - -function withoutDirectoryOption(args: readonly string[]): string[] { - const filtered: string[] = [] - for (let index = 0; index < args.length; index += 1) { - const arg = args[index] - if (arg.startsWith('--dir=')) continue - if (arg === '--dir') { - index += 1 - continue - } - filtered.push(arg) - } - return filtered -} - -function parseMode(value: string | undefined): WizardMode { - if (value === 'compose' || value === 'dev' || value === 'k8s') return value - throw new Error(`invalid --mode "${value}" — expected compose, dev, or k8s`) -} - async function main(): Promise { - const rawArgs = process.argv.slice(2) - if (rawArgs.includes('--version') || rawArgs.includes('-V')) { - console.log(readPackageVersion()) + const invocation = parseSetupArguments(process.argv.slice(2)) + if (invocation.kind === 'version') { + console.log(SETUP_VERSION) return } - const args = withoutDirectoryOption(rawArgs) - if (args.includes('--help') || args.includes('-h')) { + if (invocation.kind === 'help') { console.log(USAGE) return } process.on('SIGINT', () => exitWith(130)) - const command = args[0] - - if (command === 'config') { + if (invocation.kind === 'config') { const { runSetupStatus } = await import('./setup-status') process.exitCode = await runSetupStatus() return } - if (command === 'add') { - const feature = args[1] - if (!feature || feature.startsWith('-')) { - throw new Error(`Missing feature. Expected: ${SETUP_FEATURES}`) - } + if (invocation.kind === 'add') { const { runFeatureSetup } = await import('./feature-setup') - await runFeatureSetup(feature, args.slice(2)) + await runFeatureSetup(invocation.feature, invocation.args) return } - if (command === 'doctor') { + if (invocation.kind === 'doctor') { const { runDoctor } = await import('./doctor') - process.exitCode = await runDoctor({ - fix: args.includes('--fix'), - json: args.includes('--json'), - }) + process.exitCode = await runDoctor({ fix: invocation.fix, json: invocation.json }) return } - if (isLifecycleCommand(command)) { + if (invocation.kind === 'lifecycle') { const { runLifecycle } = await import('./lifecycle') - await runLifecycle(command) + await runLifecycle(invocation.command) return } - if (!command || command.startsWith('-')) { - const modeIdx = args.indexOf('--mode') + if (invocation.kind === 'wizard') { const { runWizard } = await import('./wizard') - await runWizard({ - quick: args.includes('--quick'), - mode: modeIdx === -1 ? undefined : parseMode(args[modeIdx + 1]), - }) + await runWizard({ quick: invocation.quick, mode: invocation.mode }) return } - - console.error(`Unknown command: ${command}\n`) - console.log(USAGE) - process.exitCode = 1 } function renderFailure(error: unknown): void { @@ -163,7 +95,12 @@ function renderFailure(error: unknown): void { main() .catch((error) => { - renderFailure(error) + if (error instanceof SetupArgumentError) { + console.error(`Error: ${error.message}\n`) + console.error(USAGE) + } else { + renderFailure(error) + } process.exitCode = 1 }) .finally(restoreTerminal) diff --git a/packages/sim-setup/src/version.ts b/packages/sim-setup/src/version.ts new file mode 100644 index 00000000000..854f3782f9f --- /dev/null +++ b/packages/sim-setup/src/version.ts @@ -0,0 +1,22 @@ +import { readFileSync } from 'node:fs' + +function readPackageVersion(): string { + const metadata: unknown = JSON.parse( + readFileSync(new URL('../package.json', import.meta.url), 'utf8') + ) + if ( + typeof metadata !== 'object' || + metadata === null || + !('version' in metadata) || + typeof metadata.version !== 'string' + ) { + throw new Error('@sim/setup package metadata is missing a valid version') + } + return metadata.version +} + +/** The version in the manifest npm installs alongside the bundled executable. */ +export const SETUP_VERSION = readPackageVersion() + +/** Identifies setup traffic by package, runtime, platform, and architecture. */ +export const SETUP_USER_AGENT = `sim-setup/${SETUP_VERSION} node/${process.versions.node} (${process.platform}; ${process.arch})` From 173a9a8f2c29d221f57d3792d2c8346867137f65 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 19 Aug 2026 19:23:39 -0400 Subject: [PATCH 5/7] Update README.md --- packages/sim-setup/README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/sim-setup/README.md b/packages/sim-setup/README.md index cfa077991bd..41e19179fc7 100644 --- a/packages/sim-setup/README.md +++ b/packages/sim-setup/README.md @@ -9,7 +9,3 @@ npx @sim/setup Outside a Sim source checkout, the command creates a Docker Compose installation using published images. Inside a Sim source checkout, use `bun run sim-setup` to expose the complete development and deployment wizard. - -By default, a standalone installation is written to `./sim`. Use `--dir ` to choose a -different directory. The `sim` npm package remains the Sim API CLI; this package intentionally -publishes only the `sim-setup` binary. From 9984a7a49bf790f041afc5b3990b19a7e118ae90 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 19 Aug 2026 16:44:00 -0700 Subject: [PATCH 6/7] fix(setup): isolate standalone compose installs --- .github/workflows/publish-sim-setup.yml | 2 +- .../sim-setup/src/compose-project.test.ts | 27 ++++++ packages/sim-setup/src/compose-project.ts | 22 +++++ packages/sim-setup/src/context.ts | 2 +- packages/sim-setup/src/index.ts | 33 +++---- packages/sim-setup/src/lifecycle.test.ts | 36 +++++++- packages/sim-setup/src/lifecycle.ts | 52 +++++++++-- packages/sim-setup/src/modes/compose.ts | 89 ++++++++++++++++--- 8 files changed, 219 insertions(+), 44 deletions(-) create mode 100644 packages/sim-setup/src/compose-project.test.ts create mode 100644 packages/sim-setup/src/compose-project.ts diff --git a/.github/workflows/publish-sim-setup.yml b/.github/workflows/publish-sim-setup.yml index bd1bfc3db40..a64041d5cb9 100644 --- a/.github/workflows/publish-sim-setup.yml +++ b/.github/workflows/publish-sim-setup.yml @@ -16,7 +16,7 @@ permissions: concurrency: group: publish-sim-setup-${{ github.ref }} - cancel-in-progress: true + cancel-in-progress: false jobs: publish-npm: diff --git a/packages/sim-setup/src/compose-project.test.ts b/packages/sim-setup/src/compose-project.test.ts new file mode 100644 index 00000000000..df6487dbb45 --- /dev/null +++ b/packages/sim-setup/src/compose-project.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from 'vitest' +import { legacyComposeProjectName, standaloneComposeProjectName } from './compose-project' + +describe('standaloneComposeProjectName', () => { + it('is stable for one installation directory', () => { + expect(standaloneComposeProjectName('/srv/one/sim')).toBe( + standaloneComposeProjectName('/srv/one/sim') + ) + }) + + it('isolates installations that share the same directory basename', () => { + expect(standaloneComposeProjectName('/srv/one/sim')).not.toBe( + standaloneComposeProjectName('/srv/two/sim') + ) + }) + + it('produces a valid Compose project name without exposing the path', () => { + expect(standaloneComposeProjectName('/Users/example/Customer Project/sim')).toMatch( + /^sim-[0-9a-f]{12}$/ + ) + }) + + it('retains the directory-derived name for legacy installations', () => { + expect(legacyComposeProjectName('/srv/Sim.Demo')).toBe('simdemo') + expect(() => legacyComposeProjectName('/srv/---')).toThrow(/Cannot derive/) + }) +}) diff --git a/packages/sim-setup/src/compose-project.ts b/packages/sim-setup/src/compose-project.ts new file mode 100644 index 00000000000..2ec93d39adb --- /dev/null +++ b/packages/sim-setup/src/compose-project.ts @@ -0,0 +1,22 @@ +import { createHash } from 'node:crypto' +import path from 'node:path' + +/** Returns the stable Docker Compose project name for a new standalone installation. */ +export function standaloneComposeProjectName(root: string): string { + const digest = createHash('sha256').update(path.resolve(root)).digest('hex').slice(0, 12) + return `sim-${digest}` +} + +/** Reproduces Compose's directory-derived identity for installs created before names were stored. */ +export function legacyComposeProjectName(root: string): string { + const name = path + .basename(path.resolve(root)) + .toLowerCase() + .replace(/[^a-z0-9_-]/g, '') + if (!/^[a-z0-9]/.test(name)) { + throw new Error( + `Cannot derive the existing Compose project name from ${root}; set COMPOSE_PROJECT_NAME in its .env file.` + ) + } + return name +} diff --git a/packages/sim-setup/src/context.ts b/packages/sim-setup/src/context.ts index bd26182bdd7..b074d23216d 100644 --- a/packages/sim-setup/src/context.ts +++ b/packages/sim-setup/src/context.ts @@ -53,7 +53,7 @@ function isStandaloneInstall(candidate: string): boolean { return readFileSync(composeFile, 'utf8').includes(SIM_COMPOSE_MARKER) } -function directoryOverride(args: readonly string[]): string | null { +export function directoryOverride(args: readonly string[]): string | null { const equalsArg = args.find((arg) => arg.startsWith('--dir=')) if (equalsArg) { const value = equalsArg.slice('--dir='.length) diff --git a/packages/sim-setup/src/index.ts b/packages/sim-setup/src/index.ts index 9863f4a8692..656a382320a 100644 --- a/packages/sim-setup/src/index.ts +++ b/packages/sim-setup/src/index.ts @@ -10,29 +10,18 @@ const SETUP_FEATURES = 'email | storage | sandbox | jobs | cache | knowledge | knowledge-embeddings | llm | integration ' const USAGE = `Usage: - npx @sim/setup run the setup wizard - npx @sim/setup [--quick] [--dir ] create a Compose installation - npx @sim/setup config show configured capabilities and integrations - npx @sim/setup add configure ${SETUP_FEATURES} - npx @sim/setup doctor [--fix] [--json] check your setup - npx @sim/setup start | stop | restart bring your install up / down / cycle - npx @sim/setup update pull and apply Compose images - npx @sim/setup status show what's installed and healthy - npx @sim/setup logs follow logs - npx @sim/setup down remove containers (data kept) - npx @sim/setup reset archive .env + wipe managed data + sim-setup [--quick] [--dir ] [--mode compose|dev|k8s] + sim-setup config show configured capabilities and integrations + sim-setup add configure ${SETUP_FEATURES} + sim-setup doctor [--fix] [--json] check your setup + sim-setup start | stop | restart bring your install up / down / cycle + sim-setup update pull/rebuild and apply Compose images + sim-setup status show what's installed and healthy + sim-setup logs follow logs + sim-setup down remove containers (data kept) + sim-setup reset archive .env + wipe managed data -Inside a Sim source checkout, use the repository command: - bun run sim-setup [--quick] [--mode compose|dev|k8s] - bun run sim-setup config show configured capabilities and integrations - bun run sim-setup add configure ${SETUP_FEATURES} - bun run sim-setup doctor [--fix] [--json] check your setup - bun run sim-setup start | stop | restart bring your install up / down / cycle - bun run sim-setup update pull/rebuild and apply Compose images - bun run sim-setup status what's installed and healthy - bun run sim-setup logs follow logs - bun run sim-setup down remove containers (data kept) - bun run sim-setup reset archive .env + wipe managed data` +Note: dev and k8s modes require a Sim source checkout.` async function main(): Promise { const invocation = parseSetupArguments(process.argv.slice(2)) diff --git a/packages/sim-setup/src/lifecycle.test.ts b/packages/sim-setup/src/lifecycle.test.ts index 2cca78faeea..3ba26e86d01 100644 --- a/packages/sim-setup/src/lifecycle.test.ts +++ b/packages/sim-setup/src/lifecycle.test.ts @@ -4,7 +4,12 @@ import { tmpdir } from 'node:os' import path from 'node:path' import { describe, expect, it } from 'vitest' import { ensureProductionComposeFile } from './compose-asset' -import { getComposeUpdateMode, isLifecycleCommand, refreshComposeFileForUpdate } from './lifecycle' +import { + composeInstallFromDirectory, + getComposeUpdateMode, + isLifecycleCommand, + refreshComposeFileForUpdate, +} from './lifecycle' describe('setup lifecycle', () => { it('recognizes update as a lifecycle command', () => { @@ -38,4 +43,33 @@ describe('setup lifecycle', () => { rmSync(root, { recursive: true, force: true }) } }) + + it('restores a downed standalone install from its persisted project name', () => { + const root = mkdtempSync(path.join(tmpdir(), 'sim-setup-lifecycle-')) + try { + ensureProductionComposeFile({ kind: 'standalone', root, existing: false }) + writeFileSync(path.join(root, '.env'), 'COMPOSE_PROJECT_NAME=sim-a1b2c3d4e5f6\n') + + expect(composeInstallFromDirectory(root, [])).toEqual({ + kind: 'compose', + file: path.join(root, 'docker-compose.prod.yml'), + dir: root, + project: 'sim-a1b2c3d4e5f6', + }) + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) + + it('does not duplicate a running install restored from its directory', () => { + const root = mkdtempSync(path.join(tmpdir(), 'sim-setup-lifecycle-')) + try { + const file = ensureProductionComposeFile({ kind: 'standalone', root, existing: false }) + const active = [{ kind: 'compose', file, dir: root, project: 'sim-live' }] as const + + expect(composeInstallFromDirectory(root, active)).toBeNull() + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) }) diff --git a/packages/sim-setup/src/lifecycle.ts b/packages/sim-setup/src/lifecycle.ts index 4a2a6500804..2c2fa65355e 100644 --- a/packages/sim-setup/src/lifecycle.ts +++ b/packages/sim-setup/src/lifecycle.ts @@ -1,10 +1,11 @@ import { spawnSync } from 'node:child_process' -import { readFileSync } from 'node:fs' +import { existsSync, readFileSync } from 'node:fs' import path from 'node:path' import { ensureProductionComposeFile } from './compose-asset' -import { resolveSetupContextAtRoot } from './context' +import { legacyComposeProjectName } from './compose-project' +import { directoryOverride, resolveSetupContextAtRoot, SETUP_CONTEXT } from './context' import { DB_CONTAINER, type Detection, REDIS_CONTAINER, runDetection } from './detect' -import { archiveEnvFile, archiveFile, ROOT } from './env-files' +import { archiveEnvFile, archiveFile, parseEnv, ROOT } from './env-files' import { SetupError } from './errors' import { forwardCommands, isLocalKubeContext } from './modes/k8s' import { httpHealth } from './probes' @@ -67,7 +68,7 @@ function dockerRun(args: string[], failMessage: string, cwd: string = ROOT): voi } } -interface ComposeInstall { +export interface ComposeInstall { kind: 'compose' /** Absolute path to the compose file Docker recorded for the project. */ file: string @@ -158,6 +159,28 @@ function composeInstalls(): ComposeInstall[] { return installs } +/** Restores a setup-managed Compose target from disk even when `down` removed every container. */ +export function composeInstallFromDirectory( + root: string, + activeInstalls: readonly ComposeInstall[] +): ComposeInstall | null { + const file = path.join(root, 'docker-compose.prod.yml') + if (!isSimComposeFile(file)) return null + if (activeInstalls.some((install) => path.resolve(install.file) === path.resolve(file))) + return null + + const envFile = path.join(root, '.env') + const configuredProject = existsSync(envFile) + ? parseEnv(readFileSync(envFile, 'utf8')).get('COMPOSE_PROJECT_NAME') + : undefined + return { + kind: 'compose', + file, + dir: root, + project: configuredProject || legacyComposeProjectName(root), + } +} + /** * Compose args for an op on a detected install. `-p` is not optional: without it * Compose re-derives the project from the working directory, and that name is @@ -203,7 +226,12 @@ function k8sInstall(detection: Detection): K8sInstall | null { } function detectInstalls(detection: Detection): Install[] { - const installs: Install[] = [...composeInstalls()] + const compose = composeInstalls() + if (SETUP_CONTEXT.kind === 'standalone' && SETUP_CONTEXT.existing) { + const fromDirectory = composeInstallFromDirectory(SETUP_CONTEXT.root, compose) + if (fromDirectory) compose.push(fromDirectory) + } + const installs: Install[] = [...compose] const dev = devInstall(detection) if (dev) installs.push(dev) const k8s = k8sInstall(detection) @@ -211,6 +239,16 @@ function detectInstalls(detection: Detection): Install[] { return installs } +/** Limits an explicit standalone `--dir` command to that installation. */ +function scopeInstallsToInvocation(installs: Install[]): Install[] { + if (SETUP_CONTEXT.kind !== 'standalone' || directoryOverride(process.argv.slice(2)) === null) { + return installs + } + return installs.filter( + (install) => install.kind === 'compose' && path.resolve(install.dir) === SETUP_CONTEXT.root + ) +} + function describeInstall(install: Install): string { if (install.kind === 'compose') return `Docker Compose (project ${install.project} in ${install.dir})` @@ -501,7 +539,7 @@ async function reset(install: Install | null): Promise { async function status(): Promise { const detection = await runDetection() - const installs = detectInstalls(detection) + const installs = scopeInstallsToInvocation(detectInstalls(detection)) const docker = dockerReachable() console.log(`\n${theme.heading('◆ Sim status')}\n`) // Every container probe goes through Docker, so when the daemon is down the @@ -540,7 +578,7 @@ async function status(): Promise { export async function runLifecycle(command: LifecycleCommand): Promise { if (command === 'status') return status() - const installs = detectInstalls(await runDetection()) + const installs = scopeInstallsToInvocation(detectInstalls(await runDetection())) // Reset stays useful with nothing running — it still archives stray .env files. if (command === 'reset') return reset(await resolveInstall(installs)) diff --git a/packages/sim-setup/src/modes/compose.ts b/packages/sim-setup/src/modes/compose.ts index 98176e438e7..24c53c19557 100644 --- a/packages/sim-setup/src/modes/compose.ts +++ b/packages/sim-setup/src/modes/compose.ts @@ -3,6 +3,7 @@ import path from 'node:path' import { EMAIL_SETUP, STORAGE_SETUP } from '../capability-config' import { promptCapabilitySetup, stageCapabilitySetupTransition } from '../capability-setup' import { ensureProductionComposeFile } from '../compose-asset' +import { legacyComposeProjectName, standaloneComposeProjectName } from '../compose-project' import { SETUP_CONTEXT } from '../context' import type { Detection } from '../detect' import { ensureDocker } from '../docker' @@ -27,14 +28,54 @@ import { APP_SIGNUP_URL, APP_URL } from '../urls' const REQUIRED_PORTS = [3000, 3002] as const +interface ComposeProject { + Name: string + ConfigFiles: string +} + +/** Returns the active Compose project already associated with this exact configuration file. */ +function activeComposeProject(composeFile: string): string | null { + const result = spawnSync('docker', ['compose', 'ls', '-a', '--format', 'json'], { + encoding: 'utf8', + }) + if (result.status !== 0) return null + + let projects: ComposeProject[] + try { + projects = JSON.parse(result.stdout) + } catch { + return null + } + const target = path.resolve(composeFile) + return ( + projects.find((project) => + project.ConfigFiles.split(',').some((file) => path.resolve(file.trim()) === target) + )?.Name ?? null + ) +} + +/** Builds a Compose command pinned to the selected standalone project when one is present. */ +function composeArgs( + composeFile: string, + project: string | undefined, + ...args: string[] +): string[] { + return ['compose', ...(project ? ['-p', project] : []), '-f', composeFile, ...args] +} + +/** Formats the pinned Compose prefix used in copyable diagnostics. */ +function composeCommand(composeFile: string, project: string | undefined): string { + return `docker compose${project ? ` -p ${project}` : ''} -f ${composeFile}` +} + /** * Host ports this compose project currently publishes. Read from the containers * rather than assumed from the file, because what matters is what is bound right * now — a project with only db/redis up publishes neither app port, so those * still need the conflict check. */ -function composePublishedPorts(composeFile: string): Set { - const ids = spawnSync('docker', ['compose', '-f', composeFile, 'ps', '-q'], { +function composePublishedPorts(composeFile: string, project: string | undefined): Set { + const ids = spawnSync('docker', composeArgs(composeFile, project, 'ps', '-q'), { cwd: ROOT, encoding: 'utf8', }) @@ -65,14 +106,17 @@ function composePublishedPorts(composeFile: string): Set { * instead of letting `docker compose up` die halfway through startup. Aborting * is fatal here: compose can't come up while the ports are held. */ -async function ensureComposePortsFree(composeFile: string): Promise { +async function ensureComposePortsFree( + composeFile: string, + project: string | undefined +): Promise { // A port this stack already publishes is not a conflict — `docker compose up // -d` reconciles its own containers, and reporting the install's own realtime // container as a blocker (offering to kill Docker's listener) is never right. // Skip only the ports this project actually publishes: leftover db/redis // containers must not wave through a foreign process sitting on 3000, which // would otherwise surface as a raw compose bind error instead of the prompt. - const ours = composePublishedPorts(composeFile) + const ours = composePublishedPorts(composeFile, project) const toCheck = REQUIRED_PORTS.filter((port) => !ours.has(port)) if (toCheck.length < REQUIRED_PORTS.length) { const skipped = REQUIRED_PORTS.filter((port) => ours.has(port)) @@ -115,6 +159,26 @@ export async function runComposeMode(detection: Detection, quick: boolean): Prom const root = readEnvFile('root') const values = collectSecrets(root) const remove = new Set() + const configuredComposeProject = root.vars.get('COMPOSE_PROJECT_NAME') + const activeProject = + SETUP_CONTEXT.kind === 'standalone' ? activeComposeProject(composeFile) : null + if (configuredComposeProject && activeProject && configuredComposeProject !== activeProject) { + throw new SetupError( + `COMPOSE_PROJECT_NAME is ${configuredComposeProject}, but ${composeFile} is running as project ${activeProject}.`, + ['stop the active project or restore its name in .env before running setup again'] + ) + } + const composeProject = + SETUP_CONTEXT.kind === 'standalone' + ? (configuredComposeProject ?? + activeProject ?? + (SETUP_CONTEXT.existing + ? legacyComposeProjectName(ROOT) + : standaloneComposeProjectName(ROOT))) + : undefined + if (composeProject && !configuredComposeProject) { + values.COMPOSE_PROJECT_NAME = composeProject + } // Before the key is minted: a half-set override mints against one environment // and validates against the other, and warning afterwards is too late — the // bad key is already stored, and the next run offers to keep it. @@ -155,7 +219,7 @@ export async function runComposeMode(detection: Detection, quick: boolean): Prom reconcileEnvValues('root', [...remove], values) p.log.step('Wrote .env (compose reads it for variable substitution)') - const validation = spawnSync('docker', ['compose', '-f', composeFile, 'config'], { + const validation = spawnSync('docker', composeArgs(composeFile, composeProject, 'config'), { cwd: ROOT, encoding: 'utf8', }) @@ -165,18 +229,19 @@ export async function runComposeMode(detection: Detection, quick: boolean): Prom ) } - await ensureComposePortsFree(composeFile) + await ensureComposePortsFree(composeFile, composeProject) - p.log.step(`Running docker compose -f ${composeFile} up -d`) - const result = spawnSync('docker', ['compose', '-f', composeFile, 'up', '-d'], { + const command = composeCommand(composeFile, composeProject) + p.log.step(`Running ${command} up -d`) + const result = spawnSync('docker', composeArgs(composeFile, composeProject, 'up', '-d'), { cwd: ROOT, stdio: 'inherit', }) if (result.status !== 0) { throw new SetupError(`docker compose exited with ${result.status}.`, [ - `inspect what failed: ${theme.command(`docker compose -f ${composeFile} logs --tail 50`)}`, - `container status: ${theme.command(`docker compose -f ${composeFile} ps`)}`, - `clean slate: ${theme.command(`docker compose -f ${composeFile} down`)} then re-run the wizard`, + `inspect what failed: ${theme.command(`${command} logs --tail 50`)}`, + `container status: ${theme.command(`${command} ps`)}`, + `clean slate: ${theme.command(`${command} down`)} then re-run the wizard`, ]) } @@ -190,7 +255,7 @@ export async function runComposeMode(detection: Detection, quick: boolean): Prom throw new SetupError( `${!appHealthy ? 'the app (:3000)' : 'realtime (:3002)'} never answered its health check.`, [ - `follow the logs: ${theme.command(`docker compose -f ${composeFile} logs -f`)}`, + `follow the logs: ${theme.command(`${command} logs -f`)}`, `first boots on slow disks can exceed the wait — if containers are still starting, just wait and open ${APP_SIGNUP_URL}`, ] ) From da603e9adb1c2de4fef1ec5bf72a7523a7dd569b Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 19 Aug 2026 16:53:09 -0700 Subject: [PATCH 7/7] fix(setup): restore default stopped installs --- packages/sim-setup/src/context.test.ts | 16 ++++++++++++++++ packages/sim-setup/src/context.ts | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/packages/sim-setup/src/context.test.ts b/packages/sim-setup/src/context.test.ts index b5bf0e1262e..8b251077301 100644 --- a/packages/sim-setup/src/context.test.ts +++ b/packages/sim-setup/src/context.test.ts @@ -55,6 +55,22 @@ describe('resolveSetupContext', () => { }) }) + it('finds an existing installation in the default child directory', () => { + const root = tempRoot() + const installRoot = path.join(root, 'sim') + mkdirSync(installRoot) + writeFileSync( + path.join(installRoot, 'docker-compose.prod.yml'), + 'image: ghcr.io/simstudioai/simstudio:latest\n' + ) + + expect(resolveSetupContext(root, [])).toEqual({ + kind: 'standalone', + root: installRoot, + existing: true, + }) + }) + it('fails on a partial Sim checkout', () => { const root = tempRoot() writePackage(root, 'package.json', 'simstudio') diff --git a/packages/sim-setup/src/context.ts b/packages/sim-setup/src/context.ts index b074d23216d..1a11aee54f5 100644 --- a/packages/sim-setup/src/context.ts +++ b/packages/sim-setup/src/context.ts @@ -112,7 +112,7 @@ export function resolveSetupContext( } } - return { kind: 'standalone', root: path.join(path.resolve(start), 'sim'), existing: false } + return resolveSetupContextAtRoot(path.join(path.resolve(start), 'sim')) } export const SETUP_CONTEXT = resolveSetupContext()