From 6954017674ae4534836b065e972da6c015cbc23e Mon Sep 17 00:00:00 2001 From: Gabriele Battimelli Date: Sun, 16 Aug 2026 08:26:54 -0400 Subject: [PATCH] Roll back automatically when a deploy fails its health check On 2026-08-16 the indexing run deployed an image that never served traffic. The health check correctly detected it and failed the run, but the broken release stayed live: the site was down from 03:41 until a human noticed at 08:12. Verifying after the fact is not enough if nothing undoes the damage. Capture the currently-serving release before deploying, and restore it if the new one does not return 200 within ~5 minutes. Then re-verify the rollback itself and say plainly in the job summary whether the site came back, so a failure that needs a human is distinguishable from one that self-healed. Also stop recording LAST_PHYSLIB_SHA unless the deploy verified. It was being set unconditionally, so a failed deploy marked that PhysLib revision as done and the next run would skip it -- stranding the indexed work silently. Select the rollback target by release description rather than the slug field: container releases carry no slug, so the previous check matched nothing and would have crashed at the moment it was needed. --- .github/workflows/weekly-index.yml | 75 ++++++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 4 deletions(-) diff --git a/.github/workflows/weekly-index.yml b/.github/workflows/weekly-index.yml index f8184bc..1629a34 100644 --- a/.github/workflows/weekly-index.yml +++ b/.github/workflows/weekly-index.yml @@ -286,6 +286,30 @@ jobs: echo "::warning title=ChromaDB approaching memory budget::chroma/ is ${SIZE_MB} MB (threshold ${CHROMA_MAX_MB} MB). The web dyno loads this into RAM at startup; upgrade the dyno or prune the index before it OOMs." fi + # Record what is currently serving, so a bad deploy can be undone. + - name: Record current release for rollback + id: prev_release + if: steps.gate.outputs.proceed == 'true' + env: + HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }} + run: | + # Roll back onto the last release that actually shipped an image. Config-var + # releases don't carry one, so rolling back onto them would not change the + # running code. Container releases are identified by their description. + PREV=$(heroku releases --app physlibsearch --num 30 --json | python3 -c ' + import json, sys + releases = json.load(sys.stdin) + for r in releases: + desc = r.get("description", "") + if desc.startswith("Deployed web") or desc.startswith("Rollback to"): + print(r["version"]) + break + else: + sys.exit("no image-bearing release found in the last 30") + ') + echo "prev=$PREV" >> "$GITHUB_OUTPUT" + echo "Will roll back to v$PREV if the new release is unhealthy." + # Rebuild the Docker image with the updated chroma/ and deploy - name: Build and release Docker image if: steps.gate.outputs.proceed == 'true' @@ -294,11 +318,16 @@ jobs: run: | heroku container:push web --app physlibsearch heroku container:release web --app physlibsearch - heroku config:set LAST_PHYSLIB_SHA=${{ steps.check.outputs.current_sha }} --app physlibsearch - # Fail loudly if the deploy did not actually come up. - - name: Verify deployment is healthy + # Verify the new release actually serves traffic, and put the previous one + # back if it does not. A failed index is an inconvenience; a site that stays + # down until someone notices is not, and that is exactly what happened on + # 2026-08-16 (deployed 03:41, still down at 08:12). + - name: Verify deployment, roll back if unhealthy + id: verify if: steps.gate.outputs.proceed == 'true' + env: + HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }} run: | echo "Waiting for the new release to serve traffic..." for attempt in $(seq 1 20); do @@ -310,5 +339,43 @@ jobs: echo "attempt $attempt: HTTP $CODE — retrying in 15s" sleep 15 done - echo "::error title=Deploy unhealthy::Site did not return HTTP 200 within ~5 minutes of release. Check 'heroku logs --app physlibsearch' — a common cause is the dyno running out of memory loading chroma/." + + echo "::error title=Deploy unhealthy — rolling back::New release never returned HTTP 200. Restoring v${{ steps.prev_release.outputs.prev }}." + heroku rollback "v${{ steps.prev_release.outputs.prev }}" --app physlibsearch + + RESTORED="" + for attempt in $(seq 1 20); do + CODE=$(curl -s -o /dev/null -w "%{http_code}" -m 20 "$HEALTHCHECK_URL" || echo 000) + if [ "$CODE" = "200" ]; then + echo "Rollback healthy (HTTP 200) after $attempt attempt(s)." + RESTORED=1 + break + fi + sleep 15 + done + + { + echo "## Deploy failed — rolled back" + echo "" + echo "The new image did not serve traffic, so v${{ steps.prev_release.outputs.prev }} was restored." + if [ -n "$RESTORED" ]; then + echo "" + echo "**The site is back up on the previous release.** The index work is safe in" + echo "Postgres; only the image that ships it failed. Re-running is safe." + else + echo "" + echo "**THE SITE IS STILL DOWN after rollback — this needs a human now.**" + echo "Check \`heroku logs --app physlibsearch\` and \`heroku releases\`." + fi + } >> "$GITHUB_STEP_SUMMARY" exit 1 + + # Only record the indexed SHA once the deploy is verified. Otherwise a failed + # deploy would mark this PhysLib revision as done and the next run would skip + # it, silently stranding the work. + - name: Mark PhysLib revision as indexed + if: steps.gate.outputs.proceed == 'true' && steps.verify.outcome == 'success' + env: + HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }} + run: | + heroku config:set LAST_PHYSLIB_SHA=${{ steps.check.outputs.current_sha }} --app physlibsearch