From edccb81f26668fa02d8a6b45dca1ad53bf4741e1 Mon Sep 17 00:00:00 2001 From: Pengyu Zhang Date: Sat, 22 Aug 2026 15:47:22 -0700 Subject: [PATCH] test(discovery): add the adr-e2e command line Four subcommands over the pieces already here: check, run, score, report. score and report never touch a VM, which is the property that matters day to day. A scoring change is replayed over every recorded run in milliseconds, and somebody debugging a false positive needs nothing but the run directory. check delegates to the manifest_check module rather than repeating it. CI runs that module directly and does not need the rest of this file; two copies of the same rules would eventually disagree, and the copy CI runs is the one that matters. A guest that cannot scan is recorded as a failed run rather than as an empty inventory. A host that reported nothing and a host that never reported are different facts, and conflating them would score every entry a miss. --- Discovery/tests/HARNESS.md | 155 ++++++++++++++++++++++++++++++++ Discovery/tests/cli.py | 180 +++++++++++++++++++++++++++++++++++++ 2 files changed, 335 insertions(+) create mode 100644 Discovery/tests/HARNESS.md create mode 100644 Discovery/tests/cli.py diff --git a/Discovery/tests/HARNESS.md b/Discovery/tests/HARNESS.md new file mode 100644 index 0000000..91a90f6 --- /dev/null +++ b/Discovery/tests/HARNESS.md @@ -0,0 +1,155 @@ +# The end-to-end harness + +Implements the method in [../README.md](../README.md): install a known set of AI +tools on a clean VM, scan, and score what the collector reported against what +was actually installed. + +``` +manifests/*.toml ──► provision ──► install ──► scan ──► scoring ──► score.json + 120 entries clean guest by id twice pure fn report.html +``` + +Everything here is standard-library Python and lives under `Discovery/tests/`. +Nothing outside this directory is imported, including the collector: see +[Why the collector is not imported](#why-the-collector-is-not-imported). + +## Running it + +```bash +cd Discovery # anything below runs from here + +python3 -m tests.manifest_check # static checks - no VM, what CI runs +python3 -m tests.cli check --catalog adr_discovery/catalog.json +python3 -m tests.cli score tests/recorded/synthetic-linux --report +python3 -m tests.cli run --os linux --out runs/2026-08-linux --driver lima +python3 -m tests.cli run --os mac --out runs/2026-08-mac \ + --driver tart --image adr-macos --user admin --identity ~/.ssh/adr_e2e + +python3 -m unittest discover -s tests -t . # the harness's own tests +python3 -m tests.tools.faultcheck # does the scorer respond to known faults? +``` + +`--driver dry` runs the installer against a guest that records commands instead +of executing them. It exercises ordering, canary substitution and the recorded +outcomes with no hypervisor; it has no collector in it, so the score it produces +is meaningless by construction and says so (every entry a miss, scan failed). + +## What is here + +| Path | What it is | +| --- | --- | +| `manifests/` | The 120 entries, plus canary shapes and vendor sources. TOML, reviewed like code. | +| `manifest.py` | The only reader of those files. Validates at load; three static checks. | +| `provision/` | `restore`/`run`/`push`/`pull`, and the drivers behind it. | +| `install/` | Executes entries by id in dependency order; writes `manifest.actual.json`. | +| `scoring/` | The product: `(before, after, manifest.actual) → score.json`. | +| `report/` | One self-contained HTML page per run. | +| `recorded/` | Captured runs, checked in - the scoring engine's own fixtures. | +| `test_*.py` | 88 tests for the harness itself. Milliseconds, no VM. | +| `tools/` | `synthesize.py` builds a run directory without a guest; `faultcheck.py` proves the scorer responds to known defects. | + +## Status + +Built: the manifest, the scoring engine, the report, the Linux and macOS +drivers, the runner, and three of the fourteen recipe families - `declare-mcp` +(27 entries), `artifact` (24) and `npm-global` (9). + +Validated against real guests, not only in simulation: + +| | guest | applicable | installed | failed | +| --- | --- | ---: | ---: | ---: | +| Linux | Ubuntu 24.04.4 aarch64 (lima) | 105 | 52 | 0 | +| macOS | 15.7.7 arm64 (tart) | 110 | 50 | 1 | +| Windows | — | 103 | — | not validated | + +The macOS failure is real and not the harness's: Kilo CLI's own postinstall +fails on macOS arm64 while the same entry installs cleanly on Linux arm64. + +The six-entry gap against the plan's P1 target of 60 is the `M-SITE` rows that +declare inside an application's own config directory (`M-SITE-03`..`M-SITE-08`), +which the runner correctly defers until `app-installer` exists. + +Windows has no driver here. This host is Apple silicon and cannot run a Windows +guest without a hypervisor that is not installed, so the QEMU driver the plan +describes is deliberately absent rather than shipped unexercised. + +The other eleven families record `unimplemented`. That is a deliberate fourth +status alongside `installed` / `unavailable` / `failed`: it keeps those entries +out of the denominator and keeps them loud. `PENDING` in +`install/recipes/__init__.py` names each one and the phase it belongs to. + +Two things need a human before a real run: + +- **Pins are unconfirmed.** `pins_confirmed = false` in `manifests/tools.toml`. + The versions there are placeholders; each needs checking against the registry + it installs from when the golden images are built. +- **57 vendor descriptors are unresolved.** `sources.toml` carries `url = ""` + for every app installer and vendor binary. `check` lists them; the recipes + refuse to run an entry whose source is unresolved rather than reporting a + vendor that stopped shipping. + +## Why the collector is not imported + +The plan reaches for `diff_snapshots` from the collector to compute the delta. +This harness re-derives it in `scoring/snapshot.py` instead, for two reasons. + +The harness must run from this directory alone, so scoring a recorded run needs +nothing installed beside it. That is the practical reason. + +The load-bearing one is that a test which imports the thing it measures stops +being able to catch a whole class of defect. If the scorer computed "what +arrived" with the collector's own diff, a diff that dropped assets would drop +them from the measurement too, and the run would score a clean sheet while +quietly measuring less. Re-deriving means the two definitions can disagree - and +a disagreement is exactly the finding worth having. + +The cost is that `scoring/snapshot.py` encodes an expectation about the snapshot +format. That is deliberate: the format is the collector's published contract, +and a test that fails when it changes silently is the correct outcome. + +## The recorded runs + +`recorded/` is the load-bearing directory. A run captured there becomes a +scoring fixture: change the scorer, replay every recorded run, see exactly which +verdicts moved. + +`synthetic-linux` is **generated, not captured** - `tools/synthesize.py` built +it from the manifest with four defects injected on purpose (two misses, one +duplicate spanning five entries, one attributed invention, one unattributed). +Its `manifest.actual.json` says `"synthetic": true`, and a test asserts that it +does. It proves the scorer computes what we think it computes. Only a captured +run says anything about the collector. + +## Checking the instrument + +Two different questions, and both need answering before a score means anything. + +`python3 -m unittest discover -s tests -t .` asks whether each piece behaves - +88 tests, no VM, about 70ms. + +`python3 -m tests.tools.faultcheck` asks whether the instrument as a whole +responds correctly to known faults. It builds a defect-free control run, +confirms it scores 1.0/1.0 with the gate passing, then injects one real failure +mode at a time - a missed declaration site, one tool reported twice, an +invention, a lookalike believed, a wrong version, a wrong scope, a mutable tag +read as pinned, a leaked credential, silence about an unknown tool, an +unexplained error, and recall below the last accepted run - and checks that the +predicted signal appears and that nothing else moves. + +Both halves matter. A scorer that misses a planted duplicate is broken; so is +one that reports defects nobody planted, because every false alarm costs +somebody the afternoon it takes to prove the collector was fine. Run it for each +OS: `--os mac`, `--os win`. + +## Scoring, in one paragraph + +Entries are matched by shape: an installed tool by catalog id, a declared server +by what it launches *and where it was declared*, an artifact by its path, a +state by the asset it attaches to. One entry matched once is a TP; matched by +two or more assets is a DUP, tracked separately because a duplicate is not a +partial success; matched by none is an FN. An asset no entry claims is an FP, +attributed to a negative control where one explains it. Field accuracy is +computed over true positives only and reported per field, never blended. A +canary leak, a dirty baseline, an unexplained error, any duplicate, a missed +review-queue entry, or recall below the last accepted run for that OS fails the +gate. diff --git a/Discovery/tests/cli.py b/Discovery/tests/cli.py new file mode 100644 index 0000000..abd961e --- /dev/null +++ b/Discovery/tests/cli.py @@ -0,0 +1,180 @@ +"""``adr-e2e`` - the harness from a terminal. + + python3 -m tests.cli check # static checks, no VM + python3 -m tests.cli run --os linux --out DIR # restore, install, scan, score + python3 -m tests.cli score DIR # re-score a recorded run + python3 -m tests.cli report DIR # scorecard from score.json + +``score`` and ``report`` never touch a VM, which is the property that matters +day to day: a scoring change is replayed over every recorded run in +milliseconds, and somebody debugging a false positive needs nothing but the run +directory. +""" + +import argparse +import json +import os +import sys +from typing import Any, Dict, List, Optional + +from . import manifest as manifest_module +from . import manifest_check +from .report import html as report_html +from .scoring import score_run + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser(prog="adr-e2e", description=__doc__.splitlines()[0]) + sub = parser.add_subparsers(dest="command", required=True) + + check = sub.add_parser("check", help="static manifest checks - no VM, per-commit safe") + check.add_argument("--catalog", help="path to the collector's catalog.json, to check coverage") + + run = sub.add_parser("run", help="restore a guest, install, scan and score") + run.add_argument("--os", dest="platform", required=True, choices=("mac", "linux", "win")) + run.add_argument("--out", required=True, help="run directory to write") + run.add_argument("--driver", default="dry", choices=("dry", "lima", "tart")) + run.add_argument("--image", default="", help="backing image or golden VM name") + run.add_argument("--collector", default="", help="version of the collector under test") + run.add_argument("--ssh-port", type=int, default=2222) + run.add_argument("--user", default="tester") + run.add_argument("--identity", default=None, help="ssh key for the guest") + run.add_argument("--home", default="") + + score = sub.add_parser("score", help="score a run directory") + score.add_argument("run_dir") + score.add_argument("--previous", help="score.json of the last accepted run, for the gate") + score.add_argument("--report", action="store_true", help="also write report.html") + + report = sub.add_parser("report", help="render report.html from an existing score.json") + report.add_argument("run_dir") + + args = parser.parse_args(argv) + return {"check": _check, "run": _run, "score": _score, "report": _report}[args.command](args) + + +# -- commands ---------------------------------------------------------- + + +def _check(args: Any) -> int: + """The static checks, which are also a module of their own. + + CI runs them directly as ``python3 -m tests.manifest_check`` and does not + need the rest of this file, so the checks live there and this delegates. + Two copies of the same rules would eventually disagree, and the copy CI + runs is the one that matters. + """ + return manifest_check.main(["--catalog", args.catalog] if args.catalog else []) + + +def _run(args: Any) -> int: + from .install import Context, Runner + from .install.runner import write_actual + + manifest = manifest_module.load() + driver = _driver(args) + home = args.home or _default_home(args.platform, args.user) + + os.makedirs(args.out, exist_ok=True) + print("restore %s" % args.driver) + driver.restore() + + print("baseline scan") + before = _scan(driver, os.path.join(args.out, "before.json")) + + context = Context(driver, manifest, args.platform, home) + _write_json(os.path.join(args.out, "canaries.json"), context.canaries) + + print("install %d applicable entries" % len(manifest.for_platform(args.platform))) + actual = Runner(context).run() + actual["collector"] = args.collector + actual["run_id"] = os.path.basename(os.path.normpath(args.out)) + write_actual(actual, args.out) + print(" %d installed, %d unavailable, %d failed, %d unimplemented" + % (actual["installed"], actual["unavailable"], actual["failed"], actual["unimplemented"])) + + print("scan") + _scan(driver, os.path.join(args.out, "after.json")) + del before + + return _score(argparse.Namespace(run_dir=args.out, previous=None, report=True)) + + +def _score(args: Any) -> int: + manifest = manifest_module.load() + previous = None + if getattr(args, "previous", None): + with open(args.previous, encoding="utf-8") as handle: + previous = json.load(handle) + result = score_run(args.run_dir, manifest, previous=previous) + _write_json(os.path.join(args.run_dir, "score.json"), result) + + totals = result["totals"] + print("score tp=%d fp=%d fn=%d dup=%d recall=%s precision=%s" + % (totals["tp"], totals["fp"], totals["fn"], totals["dup"], + totals["recall"], totals["precision"])) + print("canaries %d planted, %d leaked" % (result["canaries"]["planted"], + result["canaries"]["leaked"])) + if getattr(args, "report", False): + print("report %s" % report_html.write(result, args.run_dir)) + if result["gate"]["passed"]: + print("gate passed") + return 0 + print("gate FAILED: %s" % ", ".join(result["gate"]["reasons"]), file=sys.stderr) + return 2 + + +def _report(args: Any) -> int: + with open(os.path.join(args.run_dir, "score.json"), encoding="utf-8") as handle: + result = json.load(handle) + print(report_html.write(result, args.run_dir)) + return 0 + + +# -- plumbing ---------------------------------------------------------- + + +def _driver(args: Any) -> Any: + if args.driver == "dry": + from .provision import DryRunDriver + return DryRunDriver(args.platform, args.home or _default_home(args.platform, args.user)) + if args.driver == "lima": + from .provision.lima import LimaDriver + return LimaDriver(instance=args.image or "adr-disco-linux") + from .provision.tart import TartDriver + return TartDriver(golden=args.image, user=args.user, identity=args.identity) + + +def _scan(driver: Any, destination: str) -> Dict[str, Any]: + """Run the collector in the guest and bring its snapshot back. + + The collector under test is whatever the guest has been given - built + locally and pushed in by whoever provisioned the image, never installed from + a registry, because a run that fetched a published artifact would be testing + a release rather than the change in front of it. + """ + result = driver.run(["adr-discovery", "--json"], timeout=900) + if result.ok and result.text(): + payload = json.loads(result.text()) + else: + # A guest that cannot scan is a failed run, not an empty inventory: a + # host that reported nothing and a host that never reported are + # different facts, and conflating them would score every entry a miss. + payload = {"hostname": "", "assets": [], "errors": [ + {"probe": "harness", "path": "", "message": "scan failed: %s" % result.stderr[:200]}], + "stats": {"asset_count": 0, "error_count": 1}} + _write_json(destination, payload) + return payload + + +def _default_home(platform: str, user: str) -> str: + return {"mac": "/Users/%s", "win": "C:/Users/%s"}.get(platform, "/home/%s") % user + + +def _write_json(path: str, payload: Any) -> None: + with open(path, "w", encoding="utf-8") as handle: + json.dump(payload, handle, indent=2, sort_keys=True) + handle.write("\n") + + +if __name__ == "__main__": + raise SystemExit(main())