From baec79a28e042ca5fd6c2cfd949b91e139428c96 Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 27 Jul 2026 10:14:53 -0400 Subject: [PATCH 1/2] fix(manifest): correct published digests and make drift detectable platform-manifest.json is served publicly from raw.githubusercontent and is OpenAdapt's only cryptographic claim about the current release. It pinned openadapt-flow 1.23.0 with that release's wheel and sdist sha256, so anyone verifying today's published artifact against it failed. Regenerate from the live PyPI state (flow 1.24.0). All eight artifact digests were re-verified by downloading each file and hashing it, not by trusting a metadata field. The more important bug is that the drift detector missed the drift for hours and could have missed it for a week: * The staleness originates in ANOTHER repository, so no commit lands here and only the schedule can notice. It was weekly. It is now daily, plus a `component-released` repository_dispatch receiver so a component release can trigger the check immediately. * The check failed benignly after every single release, which is how a real failure became invisible: the semantic-release version commit leaves pyproject.toml ahead of the not-yet-reconciled manifest, and PyPI's info.version lags an upload by minutes. Main was already red from the 1.9.0 release when the genuine flow drift arrived. Both conditions now warn. Every filename, URL, and digest comparison stays fatal, and an unreachable PyPI warns rather than fails (an index outage is not evidence of drift) except at the release gate, which passes --require-network. * A red scheduled run on a repository nobody has open is not a signal, so a failed scheduled or dispatched run now files or comments on a drift issue. tests/test_platform_manifest_drift.py proves the guard FAILS when it should: against a simulated future release, a tampered digest, a tampered URL, and a version PyPI never published -- and that it does not fail on the two transient release-time conditions above. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01NyCHrzA1psrKMFfroYbzaM --- .github/workflows/platform-manifest.yml | 65 +++++- .github/workflows/release-and-publish.yml | 5 +- docs/platform-manifest.md | 49 ++++- platform-manifest.json | 16 +- scripts/validate_platform_manifest.py | 243 +++++++++++++++++---- tests/test_platform_manifest_drift.py | 254 ++++++++++++++++++++++ 6 files changed, 570 insertions(+), 62 deletions(-) create mode 100644 tests/test_platform_manifest_drift.py diff --git a/.github/workflows/platform-manifest.yml b/.github/workflows/platform-manifest.yml index 43e064387..17ae50759 100644 --- a/.github/workflows/platform-manifest.yml +++ b/.github/workflows/platform-manifest.yml @@ -12,10 +12,25 @@ on: branches: - main schedule: - # Weekly drift check: a component release without a manifest regeneration - # should fail here, loudly, even if no PR touches this repo. - - cron: '17 6 * * 1' + # DAILY drift check. The manifest goes stale because openadapt-flow, + # -capture, or -desktop released in ANOTHER repository, so no commit + # lands here and only the schedule can notice. This was weekly, which + # meant up to seven days of serving wrong sha256 digests from + # raw.githubusercontent; flow 1.24.0 published on 2026-07-27 and the + # manifest still advertised 1.23.0's digests. One small stdlib-only job + # costs seconds a day. + - cron: '17 6 * * *' workflow_dispatch: + # Receiver for a component repository to announce its release immediately + # rather than waiting for the daily sweep. The sending side is a one-step + # `gh api repos/OpenAdaptAI/OpenAdapt/dispatches -f event_type=component-released` + # in openadapt-flow/-capture/-desktop's release workflow. + repository_dispatch: + types: [component-released] + +concurrency: + group: platform-manifest-${{ github.ref }} + cancel-in-progress: true permissions: contents: read @@ -32,9 +47,53 @@ jobs: with: python-version: '3.12' + # Cheapest decisive check first: structure, signature honesty, and + # repo agreement need no network and cannot be flaky. + - name: Validate platform manifest structure (offline) + run: python scripts/validate_platform_manifest.py --offline + + # Standard library only: no dependency install, no lockfile, no cache. - name: Validate platform manifest against published artifacts run: python scripts/validate_platform_manifest.py + - name: Prove the drift guard fails when it should + run: | + python -m pip install --quiet 'pytest>=8.0.0' + python -m pytest tests/test_platform_manifest_drift.py -q + + report-manifest-drift: + # A red scheduled run in a repository nobody has open is not a signal. + # The 2026-07-27 drift DID turn the cron red and still shipped, because + # main was already red from the previous night's release. File an issue + # so drift has an owner and a paper trail instead of a stale red dot. + needs: validate-platform-manifest + if: >- + ${{ always() && needs.validate-platform-manifest.result == 'failure' && + (github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + github.event_name == 'repository_dispatch') }} + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - name: File or update the manifest drift issue + env: + GH_TOKEN: ${{ github.token }} + run: | + TITLE="platform-manifest.json has drifted from the published releases" + BODY="The scheduled platform manifest check failed: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + \`platform-manifest.json\` is served publicly from raw.githubusercontent and is OpenAdapt's only cryptographic claim about the current release. While it is stale, anyone verifying a downloaded wheel or sdist against it fails. + + Fix: \`python scripts/generate_platform_manifest.py && python scripts/validate_platform_manifest.py\`, then open a PR." + EXISTING=$(gh issue list --repo "${{ github.repository }}" --state open \ + --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty') + if [ -n "$EXISTING" ]; then + gh issue comment "$EXISTING" --repo "${{ github.repository }}" --body "$BODY" + else + gh issue create --repo "${{ github.repository }}" --title "$TITLE" --body "$BODY" + fi + check-source-boundary: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/release-and-publish.yml b/.github/workflows/release-and-publish.yml index bababcf87..47d559b58 100644 --- a/.github/workflows/release-and-publish.yml +++ b/.github/workflows/release-and-publish.yml @@ -192,7 +192,10 @@ jobs: echo "Published artifacts are not visible yet; retrying in 15 seconds." sleep 15 done - python scripts/validate_platform_manifest.py + # --require-network: at the release gate an unreachable PyPI must + # fail. Elsewhere it only warns, because an index outage is not + # evidence of drift and a flaky guard stops being read. + python scripts/validate_platform_manifest.py --require-network python scripts/validate_platform_manifest.py --offline - name: Commit the reconciled manifest diff --git a/docs/platform-manifest.md b/docs/platform-manifest.md index 06da5f57e..677a41aae 100644 --- a/docs/platform-manifest.md +++ b/docs/platform-manifest.md @@ -44,7 +44,7 @@ If publication or reconciliation fails, `main` remains red and the release workflow opens or updates a failure issue rather than weakening validation. Regenerate and commit the manifest manually after other component releases; -the scheduled validator catches component or public-status drift. +the daily scheduled validator catches component or public-status drift. ## How it is validated @@ -52,9 +52,39 @@ the scheduled validator catches component or public-status drift. python scripts/validate_platform_manifest.py ``` -CI runs this on every pull request and on a schedule +CI runs this on every pull request, on pushes to `main`, and DAILY (`.github/workflows/platform-manifest.yml`), so drift between the committed -manifest and the actually published artifacts fails loudly. +manifest and the actually published artifacts fails loudly. A component +repository can also trigger the check immediately via a +`component-released` `repository_dispatch`. + +### Why the schedule is daily and why false reds are not tolerated + +This manifest went stale once, and both causes are worth stating because they +are properties of the check rather than of the manifest: + +1. **The staleness originates in another repository.** `openadapt-flow` + released 1.24.0 on 2026-07-27; nothing was committed here, so only the + schedule could notice. It was weekly, so the public raw.githubusercontent + copy could advertise superseded sha256 digests for up to seven days. + The schedule is now daily and the job is stdlib-only (no install, no cache), + costing seconds a day. +2. **The check failed benignly after every release, so a real failure was + invisible.** Two conditions were classified as errors when they are normal: + the semantic-release version commit leaves `pyproject.toml` ahead of the + not-yet-reconciled manifest, and PyPI's `info.version` lags an upload by + minutes. Both now warn, while every digest, URL, and filename comparison + stays fatal. A guard that cries wolf at every release gets ignored, and + that is exactly what happened. + +A failed scheduled or dispatched run also files (or comments on) a +`platform-manifest.json has drifted` issue, so drift has an owner rather than +a stale red dot on a repository nobody has open. + +`tests/test_platform_manifest_drift.py` proves the guard FAILS on a simulated +future release, a tampered digest, a tampered URL, and a version PyPI never +published — and that it does NOT fail on the two transient release-time +conditions above. Validated today: @@ -63,10 +93,17 @@ Validated today: - Signature honesty: while `signature.value` is null, `signature.status` must read `unsigned (signing infrastructure pending)`. A non-null signature value fails validation because no verification path exists yet. -- Repo agreement: launcher version and openadapt-* compatibility ranges match - `pyproject.toml`. +- Repo agreement: openadapt-* compatibility ranges match `pyproject.toml`, and + the manifest's launcher version is not ahead of `pyproject.toml`'s. A + `pyproject.toml` ahead of the manifest is the normal in-flight-release state + and only warns. - Published-artifact agreement: component versions, artifact filenames, URLs, - and sha256 digests match PyPI exactly (`--offline` skips this class). + and sha256 digests match PyPI exactly (`--offline` skips this class). A + manifest version behind PyPI's latest, or naming a release PyPI never + published, fails; one ahead of `info.version` but already present in + `releases` is index propagation lag and warns, with digests still verified. + An unreachable PyPI warns rather than fails — it is not evidence of drift — + unless `--require-network` is passed. - Status skew: disagreement with `status.json` versions is a warning by default (status.json regenerates on its own cadence in `openadapt-web`); `--strict-status` escalates it to a failure. diff --git a/platform-manifest.json b/platform-manifest.json index d0d91ecd6..a6b6feb2c 100644 --- a/platform-manifest.json +++ b/platform-manifest.json @@ -1,7 +1,7 @@ { "manifest_kind": "openadapt-platform-release-manifest", "schema_version": "1.0.0", - "generated_at": "2026-07-26T23:59:24+00:00", + "generated_at": "2026-07-27T14:10:57+00:00", "release_channel": "beta", "components": { "launcher": { @@ -26,21 +26,21 @@ }, "flow": { "package": "openadapt-flow", - "version": "1.23.0", + "version": "1.24.0", "source": "pypi", "requires_python": "<3.13,>=3.10", "artifacts": [ { "type": "bdist_wheel", - "filename": "openadapt_flow-1.23.0-py3-none-any.whl", - "url": "https://files.pythonhosted.org/packages/54/f7/632b13c60ba805cfd22614dc4c73285c06c2b7d698c124d9c45104623c18/openadapt_flow-1.23.0-py3-none-any.whl", - "sha256": "b5d10dfc294479866d6dddd4c1a6afc9164414f0634703173b7e440dc0133bec" + "filename": "openadapt_flow-1.24.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/12/fb/86a58376c9bb7e587c45f3d1a149486053efb0ae86069b46d7581696983a/openadapt_flow-1.24.0-py3-none-any.whl", + "sha256": "170fdac154794292c99dc6eea6486e7a2c3fdf321bcd87976d924bccd3db4aef" }, { "type": "sdist", - "filename": "openadapt_flow-1.23.0.tar.gz", - "url": "https://files.pythonhosted.org/packages/f6/e5/8957790965a0fc5cde4e8f8dc8664c3db682d1b09881abf05f013fd3f495/openadapt_flow-1.23.0.tar.gz", - "sha256": "b16e181ac1f8b84f825b418ca9c6608cd408bde9ccd94d0a7e62db515d2145be" + "filename": "openadapt_flow-1.24.0.tar.gz", + "url": "https://files.pythonhosted.org/packages/4c/db/5b570f6511bcaed68134be094e9fc2b745f4f77c3cadf13a5d7518ac883f/openadapt_flow-1.24.0.tar.gz", + "sha256": "2d4702e5ccbdfed0f78063ca510dc82894a2833edbed71d77edffbd0ffebd67d" } ] }, diff --git a/scripts/validate_platform_manifest.py b/scripts/validate_platform_manifest.py index a7eb912d1..3f58cf305 100644 --- a/scripts/validate_platform_manifest.py +++ b/scripts/validate_platform_manifest.py @@ -13,12 +13,20 @@ "unsigned (signing infrastructure pending)". A non-null signature value FAILS validation, because no verification infrastructure exists yet and a claimed-but-unverifiable signature is worse than an honest null. -* Repo agreement (offline): the launcher version and the openadapt-* - compatibility ranges in the manifest match this checkout's pyproject.toml. +* Repo agreement (offline): the openadapt-* compatibility ranges in the + manifest match this checkout's pyproject.toml, and the manifest's launcher + version is not AHEAD of pyproject.toml's. A pyproject version ahead of the + manifest is the normal in-flight-release state and only warns; the PyPI + comparison below is what authoritatively detects staleness. * Published-artifact agreement (network): for every component, the manifest version equals the latest version on PyPI, and every artifact filename, - URL, and sha256 digest matches what PyPI reports. Run with --offline to - skip only this class of check (for example in an airgapped environment). + URL, and sha256 digest matches what PyPI reports. A manifest version that + is BEHIND PyPI's latest is drift and fails; one that is AHEAD but already + published is PyPI index propagation lag and warns (its digests are still + verified); one PyPI never published fails. An unreachable PyPI warns rather + than fails -- it is not evidence of drift, and a flaky guard gets ignored -- + unless --require-network is passed. Run with --offline to skip only this + class of check (for example in an airgapped environment). * Status-document agreement (network): version skew against https://openadapt.ai/status.json is reported as a WARNING by default, because status.json is regenerated on its own cadence in openadapt-web. @@ -141,17 +149,58 @@ def check_signature_honesty(manifest: dict, report: Report) -> None: ) +def compare_launcher_to_pyproject( + manifest_version: str | None, pyproject_version: str | None, report: Report +) -> None: + """Compare the manifest's launcher version to this checkout's pyproject. + + The manifest records the PUBLISHED launcher, while ``pyproject.toml`` + records the version this checkout would publish next. Those legitimately + disagree for the ~90 seconds between python-semantic-release pushing its + version-bump commit and the release workflow's reconcile job regenerating + the manifest, so: + + * equal -> silent. + * pyproject AHEAD of the manifest -> WARNING. A release is in flight or + the bump has not been published yet. Treating this as an error made the + guard fail on main after every single release, which is precisely how + a real drift failure became invisible. Staleness against the real world + is detected by ``compare_component_to_pypi``, which is authoritative. + * pyproject BEHIND the manifest, or either version unparseable -> ERROR. + The manifest claims a launcher this repository never produced. + """ + if manifest_version == pyproject_version: + return + + manifest_release = _release_tuple(str(manifest_version)) + pyproject_release = _release_tuple(str(pyproject_version)) + if ( + manifest_release is not None + and pyproject_release is not None + and pyproject_release > manifest_release + ): + report.warning( + f"launcher: pyproject.toml has {pyproject_version!r} but the " + f"manifest records the published {manifest_version!r}. A release " + "is in flight or unpublished; the manifest deliberately records " + "only published versions. PyPI comparison remains authoritative." + ) + return + + report.error( + f"launcher version drift: manifest has {manifest_version!r} " + f"but pyproject.toml has {pyproject_version!r}. The manifest names a " + "launcher this repository does not produce. Regenerate with " + "scripts/generate_platform_manifest.py." + ) + + def check_against_pyproject(manifest: dict, report: Report) -> None: pyproject = tomllib.loads((ROOT / "pyproject.toml").read_text()) project = pyproject["project"] launcher = manifest.get("components", {}).get("launcher", {}) - if launcher.get("version") != project["version"]: - report.error( - f"launcher version drift: manifest has {launcher.get('version')!r} " - f"but pyproject.toml has {project['version']!r}. Regenerate with " - "scripts/generate_platform_manifest.py after releasing." - ) + compare_launcher_to_pyproject(launcher.get("version"), project["version"], report) compatibility = manifest.get("compatibility", {}) if compatibility.get("python") != project["requires-python"]: @@ -192,45 +241,142 @@ def check_against_pyproject(manifest: dict, report: Report) -> None: ) -def check_against_pypi(manifest: dict, report: Report) -> None: +def _release_tuple(version: str) -> tuple[int, ...] | None: + """Return the numeric release segment of a version, or None if unparseable. + + Only the release segment is compared. Any pre/post/dev suffix makes the + version unparseable here, which callers treat conservatively (as drift) + rather than guessing an ordering. + """ + parts = version.split(".") + if not parts or not all(part.isdigit() for part in parts): + return None + return tuple(int(part) for part in parts) + + +def _files_for_version(doc: dict, version: str) -> list[dict] | None: + """Return PyPI's file records for ``version``, or None if it has none. + + ``info.version`` lags behind an upload on some PyPI CDN edges, so + ``urls`` (which describes ``info.version``) is not always the right list. + ``releases`` is keyed by version and is authoritative for a named one. + """ + releases = doc.get("releases") + if isinstance(releases, dict): + files = releases.get(version) + if files: + return list(files) + if version in releases: + return [] + return None + # Older/partial payloads without a `releases` map: fall back to `urls`, + # which only ever describes `info.version`. + if doc.get("info", {}).get("version") == version: + return list(doc.get("urls", [])) + return None + + +def compare_component_to_pypi( + role: str, component: dict, doc: dict, report: Report +) -> None: + """Compare one manifest component against PyPI's published state. + + Pure (no I/O) so the drift guard can be tested against simulated future + releases and tampered digests. Classification: + + * manifest version OLDER than PyPI's latest -> ERROR. This is staleness: + the manifest advertises a superseded release and its digests, so anyone + verifying today's artifact against it fails. This is the exact bug this + guard exists to catch. + * manifest version NEWER than PyPI's latest but already present in + ``releases`` -> WARNING. PyPI's ``info.version`` lags an upload by up to + a few minutes on some CDN edges, and a release-time false red trains + people to ignore this check. Digests are still verified against the + named version, so a wrong digest still fails. + * manifest version absent from ``releases`` entirely -> ERROR. The + manifest names a release that was never published. + * any filename/URL/digest mismatch -> ERROR, always. + """ + package = component.get("package") + manifest_version = component.get("version") + latest_version = doc.get("info", {}).get("version") + + files = _files_for_version(doc, manifest_version) + if not files: + report.error( + f"{role} version drift: manifest has {manifest_version!r} but " + f"PyPI publishes no files for {package}=={manifest_version} " + f"(PyPI's latest is {latest_version!r}). The manifest names a " + "release that does not exist. Regenerate the manifest." + ) + return + + if manifest_version != latest_version: + manifest_release = _release_tuple(str(manifest_version)) + latest_release = _release_tuple(str(latest_version)) + ahead = ( + manifest_release is not None + and latest_release is not None + and manifest_release > latest_release + ) + if ahead: + report.warning( + f"{role}: manifest has {manifest_version!r} but PyPI's " + f"latest {package} still reads {latest_version!r}. The " + "manifest version is published, so this is PyPI index " + "propagation lag, not drift; digests are verified against " + f"{manifest_version!r}." + ) + else: + report.error( + f"{role} version drift: manifest has {manifest_version!r} " + f"but PyPI's latest {package} is {latest_version!r}. The " + "manifest advertises a superseded release and its digests. " + "Regenerate it with scripts/generate_platform_manifest.py." + ) + return + + published = { + entry["filename"]: (entry["url"], entry["digests"]["sha256"]) for entry in files + } + for artifact in component.get("artifacts", []): + filename = artifact.get("filename") + if filename not in published: + report.error( + f"{role} artifact {filename} is not among PyPI's files " + f"for {package}=={manifest_version}" + ) + continue + url, sha256 = published[filename] + if artifact.get("url") != url: + report.error( + f"{role} artifact {filename} URL drift: manifest has " + f"{artifact.get('url')} but PyPI has {url}" + ) + if artifact.get("sha256") != sha256: + report.error( + f"{role} artifact {filename} sha256 drift: manifest has " + f"{artifact.get('sha256')} but PyPI has {sha256}. A consumer " + "verifying the published artifact against this manifest " + "would fail." + ) + + +def check_against_pypi(manifest: dict, report: Report, require_network: bool) -> None: for role, component in manifest.get("components", {}).items(): package = component.get("package") try: doc = _fetch_json(PYPI_URL_TEMPLATE.format(package=package)) except (urllib.error.URLError, TimeoutError) as exc: - report.error(f"could not fetch PyPI metadata for {package}: {exc}") - continue - published_version = doc["info"]["version"] - if component.get("version") != published_version: - report.error( - f"{role} version drift: manifest has " - f"{component.get('version')!r} but PyPI's latest {package} is " - f"{published_version!r}. Regenerate the manifest." - ) + # An unreachable index is not evidence of drift. Failing here + # makes the guard flaky, and a flaky guard gets ignored -- which + # is how the stale manifest survived. Release gates pass + # --require-network so the comparison cannot be silently skipped + # where it is load-bearing. + emit = report.error if require_network else report.warning + emit(f"could not fetch PyPI metadata for {package}: {exc}") continue - published = { - entry["filename"]: (entry["url"], entry["digests"]["sha256"]) - for entry in doc.get("urls", []) - } - for artifact in component.get("artifacts", []): - filename = artifact.get("filename") - if filename not in published: - report.error( - f"{role} artifact {filename} is not among PyPI's files " - f"for {package}=={published_version}" - ) - continue - url, sha256 = published[filename] - if artifact.get("url") != url: - report.error( - f"{role} artifact {filename} URL drift: manifest has " - f"{artifact.get('url')} but PyPI has {url}" - ) - if artifact.get("sha256") != sha256: - report.error( - f"{role} artifact {filename} sha256 drift: manifest has " - f"{artifact.get('sha256')} but PyPI has {sha256}" - ) + compare_component_to_pypi(role, component, doc, report) def check_against_status(manifest: dict, report: Report, strict: bool) -> None: @@ -276,6 +422,15 @@ def main() -> int: action="store_true", help="Treat status.json skew as a failure instead of a warning.", ) + parser.add_argument( + "--require-network", + action="store_true", + help=( + "Treat an unreachable PyPI as a failure instead of a warning. Use " + "at release gates, where skipping the comparison is not " + "acceptable." + ), + ) args = parser.parse_args() if not args.manifest.exists(): @@ -292,7 +447,7 @@ def main() -> int: check_signature_honesty(manifest, report) check_against_pyproject(manifest, report) if not args.offline: - check_against_pypi(manifest, report) + check_against_pypi(manifest, report, require_network=args.require_network) check_against_status(manifest, report, strict=args.strict_status) for warning in report.warnings: diff --git a/tests/test_platform_manifest_drift.py b/tests/test_platform_manifest_drift.py new file mode 100644 index 000000000..35de2daff --- /dev/null +++ b/tests/test_platform_manifest_drift.py @@ -0,0 +1,254 @@ +"""The platform-manifest drift guard must FAIL when the manifest is stale. + +`platform-manifest.json` is served from raw.githubusercontent and is the only +public *cryptographic* claim OpenAdapt makes about the current release: a +consumer verifying a downloaded wheel against it must get the right digest. + +It went wrong once. On 2026-07-26 the manifest pinned openadapt-flow 1.23.0 +with that release's wheel/sdist digests; openadapt-flow 1.24.0 published at +03:58 UTC the next morning and the manifest kept advertising the superseded +digests. Three things let that survive: + +1. The drift check only ran on pull requests, pushes to this repo's main, and + a WEEKLY cron. Staleness here is caused by a release in *another* + repository, so nothing in this repo changes and only the cron can catch it + -- a window of up to seven days. +2. The manifest's only automatic regeneration path lives in THIS repository's + release workflow, so a flow/capture/desktop release has no way to fix it. +3. Both runs triggered by the launcher's own 1.9.0 release went red for + benign, guaranteed-at-every-release reasons: the semantic-release version + commit puts pyproject.toml ahead of the not-yet-reconciled manifest, and + PyPI's `info.version` still read the previous version minutes after upload. + Main was therefore already red when the genuine drift arrived, so the real + signal was indistinguishable from the noise. + +These tests pin the behaviour that matters: the guard must fail loudly on a +simulated future release and on a tampered digest, and must NOT fail on the +transient conditions that produce false reds -- a guard that cries wolf every +release is the reason a real failure went unnoticed. +""" + +from __future__ import annotations + +import json +import sys +from copy import deepcopy +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +from validate_platform_manifest import ( # noqa: E402 + Report, + compare_component_to_pypi, + compare_launcher_to_pyproject, +) + +MANIFEST_PATH = REPO_ROOT / "platform-manifest.json" + + +@pytest.fixture() +def manifest() -> dict: + return json.loads(MANIFEST_PATH.read_text()) + + +@pytest.fixture() +def flow_component(manifest: dict) -> dict: + return deepcopy(manifest["components"]["flow"]) + + +def pypi_doc(version: str, component: dict, latest: str | None = None) -> dict: + """Build a PyPI JSON payload that publishes exactly `component`'s files. + + `latest` defaults to `version`; pass it explicitly to simulate PyPI's + `info.version` disagreeing with the release being described. + """ + files = [ + { + "filename": artifact["filename"], + "url": artifact["url"], + "digests": {"sha256": artifact["sha256"]}, + "packagetype": artifact["type"], + } + for artifact in component["artifacts"] + ] + return { + "info": {"version": latest or version}, + "urls": files, + "releases": {version: files}, + } + + +def errors_for(component: dict, doc: dict) -> list[str]: + report = Report() + compare_component_to_pypi("flow", component, doc, report) + return report.errors + + +def warnings_for(component: dict, doc: dict) -> list[str]: + report = Report() + compare_component_to_pypi("flow", component, doc, report) + return report.warnings + + +def test_truthful_manifest_passes(flow_component: dict) -> None: + """The committed manifest must validate against its own published state.""" + doc = pypi_doc(flow_component["version"], flow_component) + assert errors_for(flow_component, doc) == [] + + +def test_guard_fails_when_a_newer_release_supersedes_the_manifest( + flow_component: dict, +) -> None: + """A future release must make the guard fail -- the exact 1.23.0 bug. + + The manifest still describes the version it was generated for, with + correct digests for THAT version, and PyPI has moved on. This must fail: + the manifest's digests no longer describe the current release. + """ + stale = deepcopy(flow_component) + future = deepcopy(flow_component) + future_version = "99.0.0" + future["version"] = future_version + for artifact in future["artifacts"]: + artifact["filename"] = artifact["filename"].replace( + stale["version"], future_version + ) + artifact["url"] = artifact["url"].replace(stale["version"], future_version) + artifact["sha256"] = "f" * 64 + + doc = pypi_doc(future_version, future) + doc["releases"][stale["version"]] = [ + { + "filename": artifact["filename"], + "url": artifact["url"], + "digests": {"sha256": artifact["sha256"]}, + "packagetype": artifact["type"], + } + for artifact in stale["artifacts"] + ] + + errors = errors_for(stale, doc) + assert errors, "a superseded manifest version must be reported as drift" + assert any("version drift" in error for error in errors) + assert any(future_version in error for error in errors) + + +def test_guard_fails_on_a_tampered_digest(flow_component: dict) -> None: + """A correct version with a wrong digest must still fail. + + Version-only checking would pass this. The digest is the load-bearing + claim, so it is checked independently. + """ + doc = pypi_doc(flow_component["version"], flow_component) + tampered = deepcopy(flow_component) + tampered["artifacts"][0]["sha256"] = "0" * 64 + + errors = errors_for(tampered, doc) + assert any("sha256 drift" in error for error in errors) + + +def test_guard_fails_on_a_tampered_url(flow_component: dict) -> None: + doc = pypi_doc(flow_component["version"], flow_component) + tampered = deepcopy(flow_component) + tampered["artifacts"][0]["url"] = "https://example.invalid/evil.whl" + + errors = errors_for(tampered, doc) + assert any("URL drift" in error for error in errors) + + +def test_guard_fails_on_a_version_pypi_never_published( + flow_component: dict, +) -> None: + """A manifest may not name a release that does not exist.""" + doc = pypi_doc(flow_component["version"], flow_component) + invented = deepcopy(flow_component) + invented["version"] = "1.99.99" + + errors = errors_for(invented, doc) + assert any("no files" in error for error in errors) + + +def test_index_propagation_lag_warns_but_does_not_fail( + flow_component: dict, +) -> None: + """The release-time false red must not recur. + + Minutes after an upload, PyPI's `info.version` can still report the + previous release while the new one is already in `releases`. That is not + drift. A false red at release time is what trained everyone to ignore this + check, so it warns instead -- while still verifying digests. + """ + doc = pypi_doc(flow_component["version"], flow_component, latest="1.0.0") + + assert errors_for(flow_component, doc) == [] + assert any("propagation lag" in w for w in warnings_for(flow_component, doc)) + + +def test_propagation_lag_still_verifies_digests(flow_component: dict) -> None: + """The lag allowance must not become a hole that hides a bad digest.""" + doc = pypi_doc(flow_component["version"], flow_component, latest="1.0.0") + tampered = deepcopy(flow_component) + tampered["artifacts"][0]["sha256"] = "0" * 64 + + assert any("sha256 drift" in error for error in errors_for(tampered, doc)) + + +def test_release_in_flight_warns_instead_of_failing() -> None: + """The second guaranteed false red must not recur either. + + python-semantic-release pushes a version-bump commit to main before the + reconcile job regenerates the manifest. For that window pyproject.toml is + ahead of the manifest, which is correct -- the manifest records only + PUBLISHED versions. Failing here made main red after every release. + """ + report = Report() + compare_launcher_to_pyproject("1.8.0", "1.9.0", report) + + assert report.errors == [] + assert any("release is in flight" in w for w in report.warnings) + + +def test_manifest_ahead_of_pyproject_still_fails() -> None: + """A manifest naming a launcher this repo never produced is an error. + + The lenient direction is one-way. This is the case the pyproject check + still exists to catch. + """ + report = Report() + compare_launcher_to_pyproject("2.0.0", "1.9.0", report) + + assert any("version drift" in error for error in report.errors) + + +def test_unparseable_launcher_versions_fail_closed() -> None: + """No ordering can be inferred, so do not guess it is benign.""" + report = Report() + compare_launcher_to_pyproject("1.9.0", "1.10.0rc1", report) + + assert report.errors, "an unorderable version pair must not be waved through" + + +def test_matching_launcher_versions_are_silent() -> None: + report = Report() + compare_launcher_to_pyproject("1.9.0", "1.9.0", report) + + assert report.errors == [] + assert report.warnings == [] + + +def test_every_manifest_component_is_covered(manifest: dict) -> None: + """All four published components are guarded, not just flow.""" + assert set(manifest["components"]) == { + "launcher", + "flow", + "capture", + "desktop", + } + for role, component in manifest["components"].items(): + doc = pypi_doc(component["version"], component) + report = Report() + compare_component_to_pypi(role, component, doc, report) + assert report.errors == [], f"{role} failed self-consistency" From 08019e33e919f90d143e8f9008bdbae68ab08299 Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 27 Jul 2026 10:24:03 -0400 Subject: [PATCH 2/2] ci(manifest): keep the scheduled drift probe stdlib-only The daily and repository_dispatch runs exist to compare the committed manifest against PyPI, and that comparison needs nothing but the standard library. Gate the characterization suite to pull requests and pushes, where the guard's own source can change, so the remote drift probe stays fast and installs nothing. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01NyCHrzA1psrKMFfroYbzaM --- .github/workflows/platform-manifest.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/platform-manifest.yml b/.github/workflows/platform-manifest.yml index 17ae50759..8ac86c99c 100644 --- a/.github/workflows/platform-manifest.yml +++ b/.github/workflows/platform-manifest.yml @@ -57,6 +57,10 @@ jobs: run: python scripts/validate_platform_manifest.py - name: Prove the drift guard fails when it should + # The daily/dispatched production check is intentionally stdlib-only. + # This characterization suite belongs on source changes, not every + # remote drift probe, so scheduled checks stay fast and network-light. + if: github.event_name == 'pull_request' || github.event_name == 'push' run: | python -m pip install --quiet 'pytest>=8.0.0' python -m pytest tests/test_platform_manifest_drift.py -q