diff --git a/.changeset/reliable-release-channel.md b/.changeset/reliable-release-channel.md new file mode 100644 index 00000000..0c18f953 --- /dev/null +++ b/.changeset/reliable-release-channel.md @@ -0,0 +1,5 @@ +--- +"@pymodel/pythinker-code": patch +--- + +Keep releases visible in the update channel when a CDN rebuild request is temporarily lost. diff --git a/.changeset/use-app-token-for-brew-tap.md b/.changeset/use-app-token-for-brew-tap.md new file mode 100644 index 00000000..cc27329d --- /dev/null +++ b/.changeset/use-app-token-for-brew-tap.md @@ -0,0 +1,5 @@ +--- +"@pymodel/pythinker-code": patch +--- + +Use a scoped GitHub App token for Homebrew tap updates. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bd5153e3..89be2ee3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -264,11 +264,14 @@ jobs: # nothing — an earlier version of this job was deleted because a # curl exit-28 timeout failed the 0.5.0 release. verify-cdn-release # polls the manifest and is the gate that fails loudly. + # On 0.18.0, one connect consumed the full 60-second budget. A short + # connect timeout turns the same wall-clock budget into more attempts + # during an outage instead of waiting on connections never made. status=$(curl -sS -o /dev/stderr -w '%{http_code}' -X POST "$WEBHOOK" \ -H 'Content-Type: application/json' \ -H 'X-GitHub-Event: push' \ -d '{"ref":"refs/heads/main"}' \ - --retry 3 --retry-all-errors --retry-delay 10 --max-time 60) || status=000 + --connect-timeout 15 --max-time 45 --retry 5 --retry-all-errors --retry-delay 15) || status=000 case "$status" in 2*) echo "CDN redeploy triggered (HTTP $status)." ;; *) echo "::warning::CDN redeploy webhook returned HTTP $status — verify-cdn-release will catch a stale CDN." ;; @@ -282,7 +285,7 @@ jobs: # publish hid the one case where the version and the published artifacts # diverge — and every client polled the CDN for a release that never existed. verify-cdn-release: - timeout-minutes: 15 + timeout-minutes: 20 name: Verify release consistency needs: - release @@ -303,6 +306,8 @@ jobs: node-version-file: .nvmrc - name: Verify release consistency + env: + DOKPLOY_CDN_DEPLOY_WEBHOOK: ${{ secrets.DOKPLOY_CDN_DEPLOY_WEBHOOK }} run: node scripts/release/verify-release-consistency.mjs update-brew-tap: @@ -320,10 +325,24 @@ jobs: with: node-version-file: .nvmrc + # Any permission-* input switches the token from inheriting every + # permission the App installation holds to exactly the ones listed here. + # Cloning and pushing the tap needs contents and nothing else. + - name: Mint tap token + id: tap-token + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ vars.RELEASE_BOT_APP_ID }} + private-key: ${{ secrets.RELEASE_BOT_PRIVATE_KEY }} + owner: PyModel + repositories: homebrew-tap + permission-contents: write + - name: Bump formula env: - TAP_GITHUB_TOKEN: ${{ secrets.TAP_GITHUB_TOKEN }} + TAP_GITHUB_TOKEN: ${{ steps.tap-token.outputs.token }} run: | + # The token comes from the App installation, not a PAT. if [ -z "$TAP_GITHUB_TOKEN" ]; then echo "TAP_GITHUB_TOKEN secret not set — skipping tap update" >&2 exit 0 diff --git a/apps/pythinker-code/test/scripts/release/cdn-consistency.test.ts b/apps/pythinker-code/test/scripts/release/cdn-consistency.test.ts index 1c483e25..244bc6c6 100644 --- a/apps/pythinker-code/test/scripts/release/cdn-consistency.test.ts +++ b/apps/pythinker-code/test/scripts/release/cdn-consistency.test.ts @@ -111,6 +111,125 @@ describe('pollCdnUntilCaughtUp', () => { expect(result).toMatchObject({ ok: true, reason: 'match', attempts: 3 }); }); + it('re-triggers on the configured cadence while the CDN is behind', async () => { + const { now, sleep } = fakeClock(); + let retriggerCalls = 0; + const result = await pollCdnUntilCaughtUp({ + ...base, + now, + sleep, + fetchImpl: scriptedFetch([ + () => manifest('0.12.0'), + () => manifest('0.12.0'), + () => manifest('0.12.0'), + () => manifest('0.12.0'), + () => manifest('0.12.0'), + () => manifest('0.12.0'), + () => manifest('0.13.0'), + ]), + retrigger: async () => { + retriggerCalls += 1; + }, + retriggerEveryAttempts: 3, + }); + + expect(retriggerCalls).toBe(2); + expect(result).toMatchObject({ ok: true, attempts: 7, retriggers: 2 }); + }); + + it('does not re-trigger when the first attempt matches', async () => { + const { now, sleep } = fakeClock(); + let retriggerCalls = 0; + const result = await pollCdnUntilCaughtUp({ + ...base, + now, + sleep, + fetchImpl: scriptedFetch([() => manifest('0.13.0')]), + retrigger: async () => { + retriggerCalls += 1; + }, + retriggerEveryAttempts: 1, + }); + + expect(retriggerCalls).toBe(0); + expect(result.retriggers).toBe(0); + }); + + it('does not re-trigger when the CDN is ahead', async () => { + const { now, sleep } = fakeClock(); + let retriggerCalls = 0; + const result = await pollCdnUntilCaughtUp({ + ...base, + now, + sleep, + fetchImpl: scriptedFetch([() => manifest('0.14.0')]), + retrigger: async () => { + retriggerCalls += 1; + }, + retriggerEveryAttempts: 1, + }); + + expect(retriggerCalls).toBe(0); + expect(result).toMatchObject({ reason: 'ahead', retriggers: 0 }); + }); + + it('keeps polling when a re-trigger throws', async () => { + const { now, sleep } = fakeClock(); + let retriggerCalls = 0; + const result = await pollCdnUntilCaughtUp({ + ...base, + now, + sleep, + fetchImpl: scriptedFetch([() => manifest('0.12.0'), () => manifest('0.13.0')]), + retrigger: async () => { + retriggerCalls += 1; + throw new Error('trigger failed'); + }, + retriggerEveryAttempts: 1, + }); + + expect(retriggerCalls).toBe(1); + expect(result).toMatchObject({ ok: true, attempts: 2, retriggers: 1 }); + }); + + it('re-triggers while the CDN is unreachable', async () => { + const { now, sleep } = fakeClock(); + let retriggerCalls = 0; + const result = await pollCdnUntilCaughtUp({ + ...base, + now, + sleep, + fetchImpl: scriptedFetch([ + () => { + throw new Error('ECONNREFUSED'); + }, + () => manifest('0.13.0'), + ]), + retrigger: async () => { + retriggerCalls += 1; + }, + retriggerEveryAttempts: 1, + }); + + expect(retriggerCalls).toBe(1); + expect(result).toMatchObject({ ok: true, retriggers: 1 }); + }); + + it('reports re-trigger attempts when the budget expires', async () => { + const { now, sleep } = fakeClock(); + const result = await pollCdnUntilCaughtUp({ + ...base, + budgetMs: 45_000, + now, + sleep, + fetchImpl: scriptedFetch([() => manifest('0.12.0')]), + retrigger: async () => {}, + retriggerEveryAttempts: 1, + }); + + expect(result).toMatchObject({ reason: 'timeout', attempts: 3, retriggers: 2 }); + }); + it('treats an unreachable CDN as lag rather than a failure', async () => { const { now, sleep } = fakeClock(); const result = await pollCdnUntilCaughtUp({ diff --git a/scripts/release/cdn-consistency.mjs b/scripts/release/cdn-consistency.mjs index 35e2d244..5b524a66 100644 --- a/scripts/release/cdn-consistency.mjs +++ b/scripts/release/cdn-consistency.mjs @@ -67,12 +67,16 @@ async function readCdnVersion(fetchImpl, url) { * half-written, and both that and plain lag resolve by waiting, so only the * budget decides. 'ahead' returns at once — more waiting cannot fix a manifest * that names a release npm does not have. + * + * A periodic re-trigger heals a lost deploy request. This poll is the only + * pipeline stage that both knows the CDN is still behind and is still running. */ export async function pollCdnUntilCaughtUp(options) { - const { fetchImpl, sleep, now, url, npmLatest, budgetMs, intervalMs } = options; + const { fetchImpl, sleep, now, url, npmLatest, budgetMs, intervalMs, retrigger, retriggerEveryAttempts } = options; const deadline = now() + budgetMs; let cdnVersion = null; let attempts = 0; + let retriggers = 0; for (;;) { attempts += 1; @@ -87,12 +91,25 @@ export async function pollCdnUntilCaughtUp(options) { // Deliberately swallowed: an unreachable CDN is lag, not a gate failure. } - if (classification === 'match') return { ok: true, reason: 'match', cdnVersion, attempts }; - if (classification === 'ahead') return { ok: false, reason: 'ahead', cdnVersion, attempts }; + if (classification === 'match') return { ok: true, reason: 'match', cdnVersion, attempts, retriggers }; + if (classification === 'ahead') return { ok: false, reason: 'ahead', cdnVersion, attempts, retriggers }; // Stop before a sleep that would run past the budget rather than after it. if (now() + intervalMs >= deadline) { - return { ok: false, reason: 'timeout', cdnVersion, attempts }; + return { ok: false, reason: 'timeout', cdnVersion, attempts, retriggers }; + } + if ( + typeof retrigger === 'function' && + Number.isInteger(retriggerEveryAttempts) && + retriggerEveryAttempts > 0 && + attempts % retriggerEveryAttempts === 0 + ) { + retriggers += 1; + try { + await retrigger(); + } catch { + // Deliberately swallowed: a failed trigger is lag, not a gate failure. + } } await sleep(intervalMs); } diff --git a/scripts/release/update-brew-formula.mjs b/scripts/release/update-brew-formula.mjs index 7ce26764..0db0f957 100644 --- a/scripts/release/update-brew-formula.mjs +++ b/scripts/release/update-brew-formula.mjs @@ -4,6 +4,11 @@ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +function redactGitOutput(value, token) { + const redacted = String(value ?? '').replaceAll(/\/\/x-access-token:[^@\s]*@/gu, '//***@'); + return token.length >= 8 ? redacted.replaceAll(token, '***') : redacted; +} + async function main() { const packageJson = JSON.parse(readFileSync(new URL('../../apps/pythinker-code/package.json', import.meta.url), 'utf8')); const version = packageJson.version; @@ -22,10 +27,13 @@ async function main() { try { try { execFileSync('git', ['clone', `https://x-access-token:${token}@github.com/PyModel/homebrew-tap.git`, tapDir], { - stdio: 'ignore', + stdio: 'pipe', }); - } catch { - throw new Error('Failed to clone Homebrew tap'); + } catch (error) { + const stderr = redactGitOutput(error.stderr, token).trim(); + const stdout = redactGitOutput(error.stdout, token).trim(); + const message = redactGitOutput(error.message, token).trim(); + throw new Error(`Failed to clone Homebrew tap: ${stderr || stdout || message}`, { cause: error }); } const formulaPath = join(tapDir, 'Formula/pythinker-code.rb'); @@ -62,9 +70,12 @@ async function main() { { cwd: tapDir, stdio: 'inherit' }, ); try { - execFileSync('git', ['push', 'origin', 'main'], { cwd: tapDir, stdio: 'ignore' }); - } catch { - throw new Error('Failed to push Homebrew tap'); + execFileSync('git', ['push', 'origin', 'main'], { cwd: tapDir, stdio: 'pipe' }); + } catch (error) { + const stderr = redactGitOutput(error.stderr, token).trim(); + const stdout = redactGitOutput(error.stdout, token).trim(); + const message = redactGitOutput(error.message, token).trim(); + throw new Error(`Failed to push Homebrew tap: ${stderr || stdout || message}`, { cause: error }); } } finally { rmSync(tapDir, { recursive: true, force: true }); diff --git a/scripts/release/verify-release-consistency.mjs b/scripts/release/verify-release-consistency.mjs index b9cfe4a2..1f736882 100644 --- a/scripts/release/verify-release-consistency.mjs +++ b/scripts/release/verify-release-consistency.mjs @@ -8,10 +8,9 @@ const SEMVER = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[ const CDN_MANIFEST_URL = 'https://code.pythinker.com/pythinker-code/latest.json'; -// A Dokploy rebuild serves the new manifest in roughly two minutes. The budget -// stays well under the job's own timeout-minutes so a stale CDN is reported -// here rather than killed by the runner. -const CDN_POLL_BUDGET_MS = 600_000; +// The budget covers detecting a lost trigger and completing a fresh rebuild, +// while staying under the job timeout so this gate can report a stale CDN. +const CDN_POLL_BUDGET_MS = 900_000; const CDN_POLL_INTERVAL_MS = 15_000; function fail(reason) { @@ -59,6 +58,41 @@ try { } if (!gitTags.trim().split('\n').includes(releaseTag)) fail(`missing git tag ${releaseTag}`); +const webhook = process.env.DOKPLOY_CDN_DEPLOY_WEBHOOK; +let retrigger; +if (typeof webhook === 'string' && webhook.length > 0) { + let isUsable; + try { + const url = new URL(webhook); + isUsable = url.protocol === 'https:' && url.host.length > 0; + } catch { + isUsable = false; + } + if (isUsable) { + retrigger = async () => { + try { + const response = await fetch(webhook, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-GitHub-Event': 'push', + }, + body: '{"ref":"refs/heads/main"}', + signal: AbortSignal.timeout(30_000), + }); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + console.log(`CDN rebuild request returned HTTP ${response.status}`); + } catch (error) { + const message = error instanceof Error ? error.message.replaceAll(webhook, '***') : 'unknown error'; + console.error(`CDN rebuild request failed: ${message}`); + throw error; + } + }; + } else { + console.error('warning: DOKPLOY_CDN_DEPLOY_WEBHOOK is not an https:// URL; CDN rebuild requests are disabled'); + } +} + // The CDN manifest is what every installed client polls for updates. A version // it advertises that npm does not have sends all of them into an install that // cannot succeed; a version it never catches up to hides the release entirely. @@ -75,6 +109,8 @@ const cdnPoll = await pollCdnUntilCaughtUp({ npmLatest: distTags.latest, budgetMs: CDN_POLL_BUDGET_MS, intervalMs: CDN_POLL_INTERVAL_MS, + retrigger, + retriggerEveryAttempts: 8, }); if (cdnPoll.reason === 'ahead') { @@ -87,11 +123,15 @@ if (!cdnPoll.ok) { fail( `CDN never caught up with npm within ${CDN_POLL_BUDGET_MS / 1000}s ` + `(cdn=${cdnPoll.cdnVersion ?? 'unreachable'} latest=${distTags.latest}, ` + - `${cdnPoll.attempts} attempts) — every installed client polls this manifest, ` + + `${cdnPoll.attempts} attempt(s), ${cdnPoll.retriggers} rebuild request(s)) — ` + + 'every installed client polls this manifest, ' + 'so the release stays invisible until the site rebuilds', ); } -console.log(`CDN matches npm (${cdnPoll.cdnVersion}) after ${cdnPoll.attempts} attempt(s)`); +console.log( + `CDN matches npm (${cdnPoll.cdnVersion}) after ${cdnPoll.attempts} attempt(s), ` + + `${cdnPoll.retriggers} rebuild request(s)`, +); console.log(`consistency OK: latest=${distTags.latest} beta=${distTags.beta ?? '-'} dev=${distTags.dev ?? '-'}`);