From 002108185c635fcaccc30947b427a21a3fc94402 Mon Sep 17 00:00:00 2001 From: elkaix Date: Fri, 14 Aug 2026 20:41:26 -0400 Subject: [PATCH 1/4] fix(release): self-heal a lost CDN rebuild and surface tap push errors The CDN manifest every installed client polls was kept correct by a single fire-and-forget webhook POST. On 0.18.0 that POST hit four consecutive 60s connect timeouts, the job only warned, and the consistency gate then polled a manifest nobody had asked to rebuild -- the release stayed invisible until a human fired the webhook by hand. The consistency poll now re-fires the deploy trigger every 8 attempts while the CDN is behind, so a lost trigger heals inside the job that gates on it. The Homebrew tap bump has failed twice with its real cause swallowed by stdio: 'ignore'. Git's stderr now reaches the log, with the token redacted. --- .changeset/reliable-release-channel.md | 5 + .github/workflows/release.yml | 9 +- .../scripts/release/cdn-consistency.test.ts | 119 ++++++++++++++++++ scripts/release/cdn-consistency.mjs | 25 +++- scripts/release/update-brew-formula.mjs | 23 +++- .../release/verify-release-consistency.mjs | 45 ++++++- 6 files changed, 208 insertions(+), 18 deletions(-) create mode 100644 .changeset/reliable-release-channel.md diff --git a/.changeset/reliable-release-channel.md b/.changeset/reliable-release-channel.md new file mode 100644 index 00000000..0c18f953 --- /dev/null +++ b/.changeset/reliable-release-channel.md @@ -0,0 +1,5 @@ +--- +"@pymodel/pythinker-code": patch +--- + +Keep releases visible in the update channel when a CDN rebuild request is temporarily lost. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 07ed8487..5dc24cf3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -264,11 +264,14 @@ jobs: # nothing — an earlier version of this job was deleted because a # curl exit-28 timeout failed the 0.5.0 release. verify-cdn-release # polls the manifest and is the gate that fails loudly. + # On 0.18.0, one connect consumed the full 60-second budget. A short + # connect timeout turns the same wall-clock budget into more attempts + # during an outage instead of waiting on connections never made. status=$(curl -sS -o /dev/stderr -w '%{http_code}' -X POST "$WEBHOOK" \ -H 'Content-Type: application/json' \ -H 'X-GitHub-Event: push' \ -d '{"ref":"refs/heads/main"}' \ - --retry 3 --retry-all-errors --retry-delay 10 --max-time 60) || status=000 + --connect-timeout 15 --max-time 45 --retry 5 --retry-all-errors --retry-delay 15) || status=000 case "$status" in 2*) echo "CDN redeploy triggered (HTTP $status)." ;; *) echo "::warning::CDN redeploy webhook returned HTTP $status — verify-cdn-release will catch a stale CDN." ;; @@ -282,7 +285,7 @@ jobs: # publish hid the one case where the version and the published artifacts # diverge — and every client polled the CDN for a release that never existed. verify-cdn-release: - timeout-minutes: 15 + timeout-minutes: 20 name: Verify release consistency needs: - release @@ -303,6 +306,8 @@ jobs: node-version-file: .nvmrc - name: Verify release consistency + env: + DOKPLOY_CDN_DEPLOY_WEBHOOK: ${{ secrets.DOKPLOY_CDN_DEPLOY_WEBHOOK }} run: node scripts/release/verify-release-consistency.mjs update-brew-tap: diff --git a/apps/pythinker-code/test/scripts/release/cdn-consistency.test.ts b/apps/pythinker-code/test/scripts/release/cdn-consistency.test.ts index 1c483e25..244bc6c6 100644 --- a/apps/pythinker-code/test/scripts/release/cdn-consistency.test.ts +++ b/apps/pythinker-code/test/scripts/release/cdn-consistency.test.ts @@ -111,6 +111,125 @@ describe('pollCdnUntilCaughtUp', () => { expect(result).toMatchObject({ ok: true, reason: 'match', attempts: 3 }); }); + it('re-triggers on the configured cadence while the CDN is behind', async () => { + const { now, sleep } = fakeClock(); + let retriggerCalls = 0; + const result = await pollCdnUntilCaughtUp({ + ...base, + now, + sleep, + fetchImpl: scriptedFetch([ + () => manifest('0.12.0'), + () => manifest('0.12.0'), + () => manifest('0.12.0'), + () => manifest('0.12.0'), + () => manifest('0.12.0'), + () => manifest('0.12.0'), + () => manifest('0.13.0'), + ]), + retrigger: async () => { + retriggerCalls += 1; + }, + retriggerEveryAttempts: 3, + }); + + expect(retriggerCalls).toBe(2); + expect(result).toMatchObject({ ok: true, attempts: 7, retriggers: 2 }); + }); + + it('does not re-trigger when the first attempt matches', async () => { + const { now, sleep } = fakeClock(); + let retriggerCalls = 0; + const result = await pollCdnUntilCaughtUp({ + ...base, + now, + sleep, + fetchImpl: scriptedFetch([() => manifest('0.13.0')]), + retrigger: async () => { + retriggerCalls += 1; + }, + retriggerEveryAttempts: 1, + }); + + expect(retriggerCalls).toBe(0); + expect(result.retriggers).toBe(0); + }); + + it('does not re-trigger when the CDN is ahead', async () => { + const { now, sleep } = fakeClock(); + let retriggerCalls = 0; + const result = await pollCdnUntilCaughtUp({ + ...base, + now, + sleep, + fetchImpl: scriptedFetch([() => manifest('0.14.0')]), + retrigger: async () => { + retriggerCalls += 1; + }, + retriggerEveryAttempts: 1, + }); + + expect(retriggerCalls).toBe(0); + expect(result).toMatchObject({ reason: 'ahead', retriggers: 0 }); + }); + + it('keeps polling when a re-trigger throws', async () => { + const { now, sleep } = fakeClock(); + let retriggerCalls = 0; + const result = await pollCdnUntilCaughtUp({ + ...base, + now, + sleep, + fetchImpl: scriptedFetch([() => manifest('0.12.0'), () => manifest('0.13.0')]), + retrigger: async () => { + retriggerCalls += 1; + throw new Error('trigger failed'); + }, + retriggerEveryAttempts: 1, + }); + + expect(retriggerCalls).toBe(1); + expect(result).toMatchObject({ ok: true, attempts: 2, retriggers: 1 }); + }); + + it('re-triggers while the CDN is unreachable', async () => { + const { now, sleep } = fakeClock(); + let retriggerCalls = 0; + const result = await pollCdnUntilCaughtUp({ + ...base, + now, + sleep, + fetchImpl: scriptedFetch([ + () => { + throw new Error('ECONNREFUSED'); + }, + () => manifest('0.13.0'), + ]), + retrigger: async () => { + retriggerCalls += 1; + }, + retriggerEveryAttempts: 1, + }); + + expect(retriggerCalls).toBe(1); + expect(result).toMatchObject({ ok: true, retriggers: 1 }); + }); + + it('reports re-trigger attempts when the budget expires', async () => { + const { now, sleep } = fakeClock(); + const result = await pollCdnUntilCaughtUp({ + ...base, + budgetMs: 45_000, + now, + sleep, + fetchImpl: scriptedFetch([() => manifest('0.12.0')]), + retrigger: async () => {}, + retriggerEveryAttempts: 1, + }); + + expect(result).toMatchObject({ reason: 'timeout', attempts: 3, retriggers: 2 }); + }); + it('treats an unreachable CDN as lag rather than a failure', async () => { const { now, sleep } = fakeClock(); const result = await pollCdnUntilCaughtUp({ diff --git a/scripts/release/cdn-consistency.mjs b/scripts/release/cdn-consistency.mjs index 35e2d244..5b524a66 100644 --- a/scripts/release/cdn-consistency.mjs +++ b/scripts/release/cdn-consistency.mjs @@ -67,12 +67,16 @@ async function readCdnVersion(fetchImpl, url) { * half-written, and both that and plain lag resolve by waiting, so only the * budget decides. 'ahead' returns at once — more waiting cannot fix a manifest * that names a release npm does not have. + * + * A periodic re-trigger heals a lost deploy request. This poll is the only + * pipeline stage that both knows the CDN is still behind and is still running. */ export async function pollCdnUntilCaughtUp(options) { - const { fetchImpl, sleep, now, url, npmLatest, budgetMs, intervalMs } = options; + const { fetchImpl, sleep, now, url, npmLatest, budgetMs, intervalMs, retrigger, retriggerEveryAttempts } = options; const deadline = now() + budgetMs; let cdnVersion = null; let attempts = 0; + let retriggers = 0; for (;;) { attempts += 1; @@ -87,12 +91,25 @@ export async function pollCdnUntilCaughtUp(options) { // Deliberately swallowed: an unreachable CDN is lag, not a gate failure. } - if (classification === 'match') return { ok: true, reason: 'match', cdnVersion, attempts }; - if (classification === 'ahead') return { ok: false, reason: 'ahead', cdnVersion, attempts }; + if (classification === 'match') return { ok: true, reason: 'match', cdnVersion, attempts, retriggers }; + if (classification === 'ahead') return { ok: false, reason: 'ahead', cdnVersion, attempts, retriggers }; // Stop before a sleep that would run past the budget rather than after it. if (now() + intervalMs >= deadline) { - return { ok: false, reason: 'timeout', cdnVersion, attempts }; + return { ok: false, reason: 'timeout', cdnVersion, attempts, retriggers }; + } + if ( + typeof retrigger === 'function' && + Number.isInteger(retriggerEveryAttempts) && + retriggerEveryAttempts > 0 && + attempts % retriggerEveryAttempts === 0 + ) { + retriggers += 1; + try { + await retrigger(); + } catch { + // Deliberately swallowed: a failed trigger is lag, not a gate failure. + } } await sleep(intervalMs); } diff --git a/scripts/release/update-brew-formula.mjs b/scripts/release/update-brew-formula.mjs index 7ce26764..0db0f957 100644 --- a/scripts/release/update-brew-formula.mjs +++ b/scripts/release/update-brew-formula.mjs @@ -4,6 +4,11 @@ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +function redactGitOutput(value, token) { + const redacted = String(value ?? '').replaceAll(/\/\/x-access-token:[^@\s]*@/gu, '//***@'); + return token.length >= 8 ? redacted.replaceAll(token, '***') : redacted; +} + async function main() { const packageJson = JSON.parse(readFileSync(new URL('../../apps/pythinker-code/package.json', import.meta.url), 'utf8')); const version = packageJson.version; @@ -22,10 +27,13 @@ async function main() { try { try { execFileSync('git', ['clone', `https://x-access-token:${token}@github.com/PyModel/homebrew-tap.git`, tapDir], { - stdio: 'ignore', + stdio: 'pipe', }); - } catch { - throw new Error('Failed to clone Homebrew tap'); + } catch (error) { + const stderr = redactGitOutput(error.stderr, token).trim(); + const stdout = redactGitOutput(error.stdout, token).trim(); + const message = redactGitOutput(error.message, token).trim(); + throw new Error(`Failed to clone Homebrew tap: ${stderr || stdout || message}`, { cause: error }); } const formulaPath = join(tapDir, 'Formula/pythinker-code.rb'); @@ -62,9 +70,12 @@ async function main() { { cwd: tapDir, stdio: 'inherit' }, ); try { - execFileSync('git', ['push', 'origin', 'main'], { cwd: tapDir, stdio: 'ignore' }); - } catch { - throw new Error('Failed to push Homebrew tap'); + execFileSync('git', ['push', 'origin', 'main'], { cwd: tapDir, stdio: 'pipe' }); + } catch (error) { + const stderr = redactGitOutput(error.stderr, token).trim(); + const stdout = redactGitOutput(error.stdout, token).trim(); + const message = redactGitOutput(error.message, token).trim(); + throw new Error(`Failed to push Homebrew tap: ${stderr || stdout || message}`, { cause: error }); } } finally { rmSync(tapDir, { recursive: true, force: true }); diff --git a/scripts/release/verify-release-consistency.mjs b/scripts/release/verify-release-consistency.mjs index b9cfe4a2..600da5ee 100644 --- a/scripts/release/verify-release-consistency.mjs +++ b/scripts/release/verify-release-consistency.mjs @@ -8,10 +8,9 @@ const SEMVER = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[ const CDN_MANIFEST_URL = 'https://code.pythinker.com/pythinker-code/latest.json'; -// A Dokploy rebuild serves the new manifest in roughly two minutes. The budget -// stays well under the job's own timeout-minutes so a stale CDN is reported -// here rather than killed by the runner. -const CDN_POLL_BUDGET_MS = 600_000; +// The budget covers detecting a lost trigger and completing a fresh rebuild, +// while staying under the job timeout so this gate can report a stale CDN. +const CDN_POLL_BUDGET_MS = 900_000; const CDN_POLL_INTERVAL_MS = 15_000; function fail(reason) { @@ -59,6 +58,34 @@ try { } if (!gitTags.trim().split('\n').includes(releaseTag)) fail(`missing git tag ${releaseTag}`); +const webhook = process.env.DOKPLOY_CDN_DEPLOY_WEBHOOK; +let retrigger; +if (typeof webhook === 'string' && webhook.length > 0) { + if (!webhook.startsWith('https://')) { + console.error('warning: DOKPLOY_CDN_DEPLOY_WEBHOOK is not an https:// URL; CDN rebuild requests are disabled'); + } else { + retrigger = async () => { + try { + const response = await fetch(webhook, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-GitHub-Event': 'push', + }, + body: '{"ref":"refs/heads/main"}', + signal: AbortSignal.timeout(30_000), + }); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + console.log(`CDN rebuild request returned HTTP ${response.status}`); + } catch (error) { + const message = error instanceof Error ? error.message.replaceAll(webhook, '***') : 'unknown error'; + console.error(`CDN rebuild request failed: ${message}`); + throw error; + } + }; + } +} + // The CDN manifest is what every installed client polls for updates. A version // it advertises that npm does not have sends all of them into an install that // cannot succeed; a version it never catches up to hides the release entirely. @@ -75,6 +102,8 @@ const cdnPoll = await pollCdnUntilCaughtUp({ npmLatest: distTags.latest, budgetMs: CDN_POLL_BUDGET_MS, intervalMs: CDN_POLL_INTERVAL_MS, + retrigger, + retriggerEveryAttempts: 8, }); if (cdnPoll.reason === 'ahead') { @@ -87,11 +116,15 @@ if (!cdnPoll.ok) { fail( `CDN never caught up with npm within ${CDN_POLL_BUDGET_MS / 1000}s ` + `(cdn=${cdnPoll.cdnVersion ?? 'unreachable'} latest=${distTags.latest}, ` + - `${cdnPoll.attempts} attempts) — every installed client polls this manifest, ` + + `${cdnPoll.attempts} attempt(s), ${cdnPoll.retriggers} rebuild request(s)) — ` + + 'every installed client polls this manifest, ' + 'so the release stays invisible until the site rebuilds', ); } -console.log(`CDN matches npm (${cdnPoll.cdnVersion}) after ${cdnPoll.attempts} attempt(s)`); +console.log( + `CDN matches npm (${cdnPoll.cdnVersion}) after ${cdnPoll.attempts} attempt(s), ` + + `${cdnPoll.retriggers} rebuild request(s)`, +); console.log(`consistency OK: latest=${distTags.latest} beta=${distTags.beta ?? '-'} dev=${distTags.dev ?? '-'}`); From cd1dea674ef85f85448b33378cf76457a1e9e970 Mon Sep 17 00:00:00 2001 From: elkaix Date: Fri, 14 Aug 2026 22:12:30 -0400 Subject: [PATCH 2/4] fix(release): mint the Homebrew tap token from the release-bot App TAP_GITHUB_TOKEN was created before the org was renamed from Pythoughts-labs to PyModel, so the tap clone succeeds (public repo, token unused for the read) and only the push is refused. That broke the tap bump on 0.17.1 and 0.18.0. The job now mints an installation token from pythinker-release-bot, scoped with owner + repositories to homebrew-tap alone so it carries no write access to pythinker-code. App tokens are minted per run and do not expire, which retires this failure class rather than resetting its clock. --- .changeset/use-app-token-for-brew-tap.md | 5 +++++ .github/workflows/release.yml | 12 +++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 .changeset/use-app-token-for-brew-tap.md diff --git a/.changeset/use-app-token-for-brew-tap.md b/.changeset/use-app-token-for-brew-tap.md new file mode 100644 index 00000000..cc27329d --- /dev/null +++ b/.changeset/use-app-token-for-brew-tap.md @@ -0,0 +1,5 @@ +--- +"@pymodel/pythinker-code": patch +--- + +Use a scoped GitHub App token for Homebrew tap updates. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5dc24cf3..42696c18 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -325,10 +325,20 @@ jobs: with: node-version-file: .nvmrc + - name: Mint tap token + id: tap-token + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ vars.RELEASE_BOT_APP_ID }} + private-key: ${{ secrets.RELEASE_BOT_PRIVATE_KEY }} + owner: PyModel + repositories: homebrew-tap + - name: Bump formula env: - TAP_GITHUB_TOKEN: ${{ secrets.TAP_GITHUB_TOKEN }} + TAP_GITHUB_TOKEN: ${{ steps.tap-token.outputs.token }} run: | + # The token comes from the App installation, not a PAT. if [ -z "$TAP_GITHUB_TOKEN" ]; then echo "TAP_GITHUB_TOKEN secret not set — skipping tap update" >&2 exit 0 From 179ab0b2ad8f9a3b127cb0685172f725bd183c46 Mon Sep 17 00:00:00 2001 From: elkaix Date: Fri, 14 Aug 2026 22:20:15 -0400 Subject: [PATCH 3/4] fix(release): stop retaining the raw git error and parse the webhook URL The tap failure handlers kept the raw execFileSync error as Error.cause. Its message and stderr both carry the credentialed clone URL, so the cause chain held an un-redacted token that any future inspection of the error object would print into a public Actions log. Everything useful is already extracted and redacted into the thrown message, so the cause carried no information. The webhook guard tested a string prefix, which admits values fetch rejects. It now parses the URL and requires an https protocol and a non-empty host, so an unusable webhook disables rebuild requests at configuration time instead of producing a callback that throws on first use. --- scripts/release/verify-release-consistency.mjs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/release/verify-release-consistency.mjs b/scripts/release/verify-release-consistency.mjs index 600da5ee..1f736882 100644 --- a/scripts/release/verify-release-consistency.mjs +++ b/scripts/release/verify-release-consistency.mjs @@ -61,9 +61,14 @@ if (!gitTags.trim().split('\n').includes(releaseTag)) fail(`missing git tag ${re const webhook = process.env.DOKPLOY_CDN_DEPLOY_WEBHOOK; let retrigger; if (typeof webhook === 'string' && webhook.length > 0) { - if (!webhook.startsWith('https://')) { - console.error('warning: DOKPLOY_CDN_DEPLOY_WEBHOOK is not an https:// URL; CDN rebuild requests are disabled'); - } else { + let isUsable; + try { + const url = new URL(webhook); + isUsable = url.protocol === 'https:' && url.host.length > 0; + } catch { + isUsable = false; + } + if (isUsable) { retrigger = async () => { try { const response = await fetch(webhook, { @@ -83,6 +88,8 @@ if (typeof webhook === 'string' && webhook.length > 0) { throw error; } }; + } else { + console.error('warning: DOKPLOY_CDN_DEPLOY_WEBHOOK is not an https:// URL; CDN rebuild requests are disabled'); } } From 91b83be9848e4ad24527656dd3fcbf3c93516c40 Mon Sep 17 00:00:00 2001 From: M Elkholy Date: Fri, 14 Aug 2026 22:44:09 -0400 Subject: [PATCH 4/4] fix(release): scope the tap token to contents write Without a permission-* input, actions/create-github-app-token mints a token carrying every permission the App installation holds -- here that includes pull-request write, which cloning and pushing the Homebrew tap never uses. Supplying one input switches the action to strict opt-in, so the token now carries contents write and nothing else. --- .github/workflows/release.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index feb9ef50..89be2ee3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -325,6 +325,9 @@ jobs: with: node-version-file: .nvmrc + # Any permission-* input switches the token from inheriting every + # permission the App installation holds to exactly the ones listed here. + # Cloning and pushing the tap needs contents and nothing else. - name: Mint tap token id: tap-token uses: actions/create-github-app-token@v2 @@ -333,6 +336,7 @@ jobs: private-key: ${{ secrets.RELEASE_BOT_PRIVATE_KEY }} owner: PyModel repositories: homebrew-tap + permission-contents: write - name: Bump formula env: