ci: stop storing the package tables in git

.github/workflows/deploy.yml

@@ -3,9 +3,11 @@ on:
   push:
     branches:
       - main
-  workflow_run:
-    workflows: ["Update package table"]
-    types: [completed]
+  # The package tables live only in the deployed site, not in git; the schedule
+  # is what keeps them fresh.
+  schedule:
+    - cron: "0 */6 * * *"
+  workflow_dispatch:
 
 # ghp-import force-pushes gh-pages; two runs racing each other could publish
 # the older build last.
@@ -16,14 +18,25 @@ concurrency:
 jobs:
   deploy:
     runs-on: ubuntu-latest
-    # Only redeploy for a table refresh that actually succeeded; a push to main
-    # deploys unconditionally.
-    if: github.event_name == 'push' || github.event.workflow_run.conclusion == 'success'
+    # workflow_dispatch can be pointed at any branch; only main may deploy.
+    if: github.ref == 'refs/heads/main'
     # `ghp-import --push` pushes the built site to the gh-pages branch
     permissions:
       contents: write
     steps:
       # ghp-import pushes with the credentials persisted by checkout
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - uses: prefix-dev/setup-pixi@f00437f565399d418b0acc85936d12c1fb668347 # v0.10.1
+      # Regenerates every distro with a dataChannel in src/data/distros.json;
+      # a distro without one (foxy, galactic) is a committed snapshot. A failed
+      # distro does not stop the deploy: the ensure-tables task inside
+      # build-docs falls back to the currently deployed copy for whatever is
+      # missing, and the run is marked failed at the end.
+      - name: Update tables
+        id: update
+        continue-on-error: true
+        run: pixi run update-tables
       - run: pixi run gh-deploy
+      - name: Surface update failures
+        if: steps.update.outcome == 'failure'
+        run: exit 1

.gitignore

@@ -12,3 +12,10 @@ dist
 
 # pnpm package store
 .pnpm-store
+
+# Package tables; regenerated by the deploy workflow, downloaded from the
+# deployed site everywhere else. foxy and galactic have no data channel left
+# to regenerate from, so their snapshots stay committed.
+public/data/*.json
+!public/data/foxy.json
+!public/data/galactic.json

README.md

@@ -10,7 +10,10 @@ the common header:
   `src/components/home/`
 - the package pages - one per distro, generated by `src/pages/[distro].astro`
   from `src/data/distros.ts`; the tables render in the browser from
-  `public/data/<distro>.json`
+  `public/data/<distro>.json`, which the deploy workflow regenerates from the
+  conda channels every six hours and which `serve` and `build-docs` download
+  from the deployed site when the local copy is missing or older than six
+  hours (only the foxy and galactic snapshots are committed)
 - the documentation - the Markdown files in `src/content/docs/`
 
 The header is Starlight's own, with the wordmark supplied by the

pixi.toml

@@ -14,12 +14,12 @@ pnpm-install = {
 }
 serve = {
   cmd = "pnpm run dev",
-  depends-on = ["pnpm-install"],
+  depends-on = ["pnpm-install", "ensure-tables"],
   description = "Serve the site locally",
 }
 build-docs = {
   cmd = "pnpm run build",
-  depends-on = ["pnpm-install"],
+  depends-on = ["pnpm-install", "ensure-tables"],
   description = "Build the site",
 }
 gh-deploy = {
@@ -95,6 +95,10 @@ update-tables = {
   cmd = "python scripts/compare_pkg_completeness.py --all",
   description = "Regenerate the package tables for every distro the pipeline maintains",
 }
+ensure-tables = {
+  cmd = "python scripts/ensure_tables.py",
+  description = "Download the deployed package tables where the local copy is missing or older than 6 hours",
+}
 
 [environments]
 default = ["site", "lint", "scripts"]

scripts/compare_pkg_completeness.py

@@ -388,12 +388,12 @@ def slots(per_mutex: dict[str, Slot]) -> list[Any]:
 def write(document: dict[str, Any], path: Path) -> None:
     """Write the document with one package per line.
 
-    Compact JSON on a single line would make every rebuild a one-line diff covering
-    the whole file. One package per line costs a byte each and keeps the six-hourly
-    CI commits reviewable.
+    Compact JSON on a single line would turn any comparison into a one-line diff
+    covering the whole file. One package per line costs a byte each and keeps the
+    committed snapshots and local before/after diffs reviewable.
 
     `newline="\\n"` matters too. Without it Python writes CRLF on Windows and the
-    file diffs in full against what CI generates on Linux.
+    file diffs in full against what Linux generates.
     """
     head = {k: v for k, v in document.items() if k != "packages"}
     body = ",\n".join(json.dumps(p, separators=(",", ":")) for p in document["packages"])

scripts/ensure_tables.py

@@ -0,0 +1,76 @@
+"""Fetch the deployed package tables when the local copies are missing or stale.
+
+`public/data/<distro>.json` is not stored in git for any distro that still has a
+data channel; the deploy workflow regenerates those tables from the channels with
+scripts/compare_pkg_completeness.py every six hours. Everywhere else - local
+development and the checks workflow - this script downloads the currently
+deployed copy from the live site instead, skipping any file younger than
+`--max-age-hours`. Distros without a `dataChannel` in src/data/distros.json
+(foxy, galactic) are committed snapshots and are left alone.
+
+A stale copy that fails to redownload is kept with a warning; a distro that ends
+up with no file at all is an error, because the site cannot build without it.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+from pathlib import Path
+
+import niquests
+from urllib3.util.retry import Retry
+
+DISTROS_JSON = Path(__file__).parent.parent / "src" / "data" / "distros.json"
+DATA_DIR = Path(__file__).parent.parent / "public" / "data"
+SITE = "https://robostack.github.io/data"
+
+session = niquests.Session(
+    retries=Retry(total=3, backoff_factor=1, status_forcelist=[429, 500, 502, 503, 504])
+)
+
+
+def main() -> None:
+    """Download every regenerable distro's table that is missing or stale."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--max-age-hours",
+        type=float,
+        default=6.0,
+        help="redownload a table once the local copy is older than this (default: 6)",
+    )
+    args = parser.parse_args()
+    max_age_seconds = args.max_age_hours * 3600
+
+    missing: list[str] = []
+    for entry in json.loads(DISTROS_JSON.read_text())["distros"]:
+        if not entry["dataChannel"]:
+            continue
+        name = entry["name"]
+        path = DATA_DIR / f"{name}.json"
+        if path.exists() and time.time() - path.stat().st_mtime < max_age_seconds:
+            print(f"{name}: fresh", file=sys.stderr)
+            continue
+        try:
+            response = session.get(f"{SITE}/{name}.json")
+            response.raise_for_status()
+            body = response.content or b""
+        except Exception as error:  # noqa: BLE001 - stale data is usable, absent data is not
+            if path.exists():
+                print(f"{name}: keeping stale copy, download failed ({error})", file=sys.stderr)
+            else:
+                print(f"{name}: no local copy and download failed ({error})", file=sys.stderr)
+                missing.append(name)
+            continue
+        DATA_DIR.mkdir(parents=True, exist_ok=True)
+        path.write_bytes(body)
+        print(f"{name}: downloaded", file=sys.stderr)
+
+    if missing:
+        sys.exit(f"no table data for: {', '.join(missing)}")
+
+
+if __name__ == "__main__":
+    main()

src/components/home/Stats.astro

@@ -5,8 +5,9 @@ import { DISTROS, PLATFORMS } from "../../data/distros";
 /**
  * The stats strip: a row of figures in a card, divided by hairlines.
  *
- * The numbers are computed at build time from the committed table data, so
- * the six-hourly refresh moves them instead of them going stale. A package
+ * The numbers are computed at build time from the table data the ensure-tables
+ * task puts in public/data, so the six-hourly deploy moves them instead of
+ * them going stale. A package
  * counts once per distro when any mutex generation has a build for it.
  */
 const maintained = DISTROS.filter((d) => d.maintained);

zizmor.yml

@@ -1,9 +1,5 @@
 rules:
-  # deploy.yml intentionally reruns after "Update package table" completes,
-  # and its checkout keeps credentials because `ghp-import --push` uses them.
-  dangerous-triggers:
-    ignore:
-      - deploy.yml
+  # deploy.yml's checkout keeps credentials because `ghp-import --push` uses them.
   artipacked:
     ignore:
       - deploy.yml