diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index f98cdc5..8a2f5f2 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -86,6 +86,11 @@ jobs: timeout-minutes: 20 run: python tools/execution_spikes/recipe_worker_e2e_qualification.py --json --strict + - name: Run durable recipe coordinator qualification + shell: pwsh + timeout-minutes: 35 + run: python tools/execution_spikes/recipe_coordinator_e2e_qualification.py --json --strict + - name: Prepare signed WebView2 bootstrapper shell: pwsh run: ./packaging/prepare_webview2.ps1 diff --git a/docs/adr/0001-capability-tiered-agentic-execution-harness.md b/docs/adr/0001-capability-tiered-agentic-execution-harness.md index c00cdb5..f2d7b01 100644 --- a/docs/adr/0001-capability-tiered-agentic-execution-harness.md +++ b/docs/adr/0001-capability-tiered-agentic-execution-harness.md @@ -1007,12 +1007,15 @@ ready. Attachment staging stores only bounded, MIME-sniffed bytes behind an opaque artifact ID. `build_native_recipe_coordinator_factory` composes a fresh signed/native launcher, broker binder, protected pipe identity, and authenticated client for each recipe job; it requires an explicit reviewed process-factory -injection and never falls back to a host process. The next core slice is an -end-to-end durable-coordinator run through the packaged signed worker, including -cancellation, retention, atomic publication, and native cleanup. The separate -`recipe.release-review.v1` verifier is available as optional release hardening; -it remains observation-only and no approval record or production trust material -is committed. +injection and never falls back to a host process. The durable-coordinator +packaged-worker slice is now implemented as the strict Windows-only +`recipe_coordinator_e2e_qualification.py` gate. It stages an attachment, runs +the real signed/native coordinator, verifies owner isolation, retention expiry, +cancellation, atomic publication, and complete native cleanup, and never falls +back to a host process. Quality CI runs it after the existing worker release +qualification. The separate `recipe.release-review.v1` verifier is available as +optional release hardening; it remains observation-only and no approval record or +production trust material is committed. ### Phase 3 — `scratch.auto.v1` arbitrary WebAssembly code diff --git a/docs/adr/0001-phase2-evidence.md b/docs/adr/0001-phase2-evidence.md index 5cf5de6..bfcd83f 100644 --- a/docs/adr/0001-phase2-evidence.md +++ b/docs/adr/0001-phase2-evidence.md @@ -1,7 +1,7 @@ # ADR-0001 Phase 2 evidence log - **Phase:** 2 — signed image recipes and calculator/check primitives -- **Status:** Open-source qualification track complete through the fixed-function worker boundary; optional official-release hardening is separate +- **Status:** Open-source qualification track complete through the durable packaged-worker coordinator boundary; optional official-release hardening is separate - **Scope:** Provider-independent contracts plus a qualification-only fixed-function core - **Source decision:** [Capability-tiered agentic execution harness](0001-capability-tiered-agentic-execution-harness.md) - **Contract ADR:** [Phase 2 typed recipe and primitive contract](0001-phase2-recipe-contract.md) @@ -35,9 +35,10 @@ | Fixed-function image provider core | **Complete (qualification-only)** | `RecipeImageProvider` validates allowlisted PNG/JPEG/WebP bytes, verifies/loads one frame with Pillow bomb/resource limits, applies only parsed steps, strips metadata, revalidates encoded output, and checks cancellation. The provider remains separate from the API; the typed coordinator route is qualification-only and default-off. | | Windows recipe sandbox qualification harness | **Complete (signed launch/broker/hostile/cancellation/resource/watchdog)** | `recipe_worker_e2e_qualification.py` signs a disposable package with an in-memory key, installs/verifies one immutable generation, binds the live AppContainer identity to the broker, and exercises the fixed PNG transform, truncated-PNG decoder rejection, active-SVG decoder rejection, and in-flight cancellation corpus. `resource_watchdog_qualification.py` separately proves immutable budgets, actual Job Object accounting, and kill-on-close tree reaping. The native broker uses bounded availability polling before reads so the cancellation reader remains live while the packaged provider transforms. | | Suspended native launcher/resource policy | **Complete (factory + binder + ACL cleanup + qualification evidence)** | `NativeWin32ProcessFactory` grants only inherited read/execute access to the fresh AppContainer SID on the verified package root, applies and verifies Job Object policy before resume, and removes the per-launch ACE during cleanup. `NativeBrokerIdentityBinder` pins the live server to the worker PID/AppContainer SID and launcher cleanup closes it on failure. | -| OS sandbox provider and provider-produced image outputs | **Complete (qualification; default-off in the app)** | Signed installation, provenance, AppContainer/job identity, broker handshake, `prepare`, `input_chunk`, `input_complete`, `collect_output`, hostile decoder rejection, in-flight cancellation acknowledgement, artifact security review, resource accounting, and watchdog tree reaping are qualified. The explicit lifecycle composition, durable recipe coordinator/publication path, typed qualification-only API, trusted attachment staging, and per-job signed/native attempt factory are wired behind injection; the next gate is a durable-coordinator run through the packaged worker. Official-release review/signing remains optional hardening. | +| OS sandbox provider and provider-produced image outputs | **Complete (qualification; default-off in the app)** | Signed installation, provenance, AppContainer/job identity, broker handshake, `prepare`, `input_chunk`, `input_complete`, `collect_output`, hostile decoder rejection, in-flight cancellation acknowledgement, artifact security review, resource accounting, watchdog tree reaping, and the durable packaged-worker coordinator run are qualified. Official-release review/signing remains optional hardening. | | Explicit qualification-profile lifecycle wiring | **Complete (local/CI composition; default-off in the app)** | `build_execution_lifecycle()` accepts only exact `disabled`/`qualification` selection, requires a qualification release gate, coordinator factory, and provider-health probe, and composes them in a fail-closed order. `Cortex_Preview.build_preview_app` exposes this only as an explicit injection; the normal app supplies no profile or controls. | | Durable recipe coordinator/request, attachment staging, and artifact publication | **Complete (qualification-only API; default-off in the app)** | `RecipeExecutionCoordinator` persists only opaque artifact IDs and canonical plan digests, enforces owner-scoped input reads and idempotency conflicts, leases/recoveries/cancels attempts, validates worker envelopes/chunks, and publishes exactly one output through `ArtifactBoundary.collect_outputs`. `POST /api/v1/execution/attachments` creates an idempotent owner-scoped `attachment.stage.v1` record through `ArtifactBoundary.stage_bytes`; `POST /api/v1/execution/recipe/image` consumes its opaque artifact ID after a ready `qualification` lifecycle. `build_native_recipe_coordinator_factory` binds a fresh signed/native launcher/broker/client attempt per job without host fallback. `tests/test_phase2_recipe_coordinator.py`, `tests/test_phase2_recipe_api.py`, `tests/test_phase2_attachment_staging.py`, `tests/test_phase2_native_recipe_attempt.py`, and `tests/test_phase2_qualification_lifecycle.py` cover the hostile, cancellation, API, native-composition, and lifecycle paths. | +| Durable packaged-worker coordinator qualification | **Complete (local + hosted CI; default-off)** | `recipe_coordinator_e2e_qualification.py --json --strict --timeout-seconds 300` passed on Windows using an in-memory ephemeral signing key and a disposable immutable installer generation. The single coordinator corpus staged a PNG attachment, completed a real signed/native transform, verified digest/MIME/size and atomic publication, rejected a foreign owner, expired and purged a one-second artifact, cancelled an in-flight transform with no result artifact, and verified native worker/process and temporary-root cleanup. Final Quality run [30466976650](https://github.com/dovvnloading/Cortex/actions/runs/30466976650), job [90627154917](https://github.com/dovvnloading/Cortex/actions/runs/30466976650/job/90627154917), passed on commit `0c7b350` in 4m15s after the packaged worker release qualification. | ## Security invariants @@ -144,6 +145,7 @@ python -m pytest tests/test_recipe_sandbox_qualification.py -q python tools/execution_spikes/native_launcher_qualification.py python tools/execution_spikes/recipe_sandbox_qualification.py --json --strict python tools/execution_spikes/recipe_worker_e2e_qualification.py --json --strict +python tools/execution_spikes/recipe_coordinator_e2e_qualification.py --json --strict --timeout-seconds 300 python tools/execution_spikes/recipe_parser_fuzz.py --json --strict python tools/execution_spikes/artifact_security_review.py --json --strict python tools/execution_spikes/resource_watchdog_qualification.py --json --strict @@ -302,8 +304,9 @@ provider-health probe. The normal application remains default-off. The durable recipe-specific coordinator/request and artifact-publication path is now implemented as an explicit qualification-only API surface with a typed TypeScript client method; no outside reviewer or production key was required for -it. The next core slice is trusted attachment staging and binding the real -signed/native broker worker into the injected attempt factory. +it. Trusted attachment staging and binding the real signed/native broker worker +into the injected attempt factory are now complete; the durable packaged-worker +coordinator qualification is the hosted CI gate for this stage. **Qualification API stage verification (2026-07-29):** The explicit `POST /api/v1/execution/recipe/image` surface is available only from a ready @@ -330,6 +333,19 @@ the explicit coordinator helper requires a reviewed process-factory injection and creates no process during configuration. Focused verification passed 52 tests across artifact boundary, attachment staging, native composition, API, and launcher suites; the qualification route remains unavailable in the normal -default-off app. The next gate is an end-to-end durable-coordinator run through -the packaged signed worker, including cancellation, retention, publication, and -native cleanup. +default-off app. Hosted Quality CI verification of the durable packaged-worker +coordinator gate is complete. + +**Durable packaged-worker coordinator qualification result (2026-07-29):** The +strict Windows probe +`recipe_coordinator_e2e_qualification.py --json --strict --timeout-seconds 300` +passed locally using only an in-memory ephemeral Ed25519 key and a disposable +installer store. In one coordinator instance it staged a fixed PNG attachment, +completed a signed/native transform with digest/MIME/size checks and atomic +publication, rejected a foreign-owner request before worker launch, expired and +purged a one-second attachment, cancelled an in-flight resize transform with no +published result, and verified native process plus temporary workspace cleanup. +The probe remains qualification-only and does not enable the normal application; +hosted Quality CI run [30466976650](https://github.com/dovvnloading/Cortex/actions/runs/30466976650) +and job [90627154917](https://github.com/dovvnloading/Cortex/actions/runs/30466976650/job/90627154917) +passed the same gate on commit `0c7b350` in 4m15s. diff --git a/docs/adr/0001-phase2-native-launcher.md b/docs/adr/0001-phase2-native-launcher.md index 4edf0c3..a003669 100644 --- a/docs/adr/0001-phase2-native-launcher.md +++ b/docs/adr/0001-phase2-native-launcher.md @@ -82,19 +82,18 @@ The fixed probe and watchdog corpus do not yet prove enforcement against a real signed recipe worker, resource-exhaustion behavior across supported Windows versions, or external launcher review. -## Remaining blockers - -- The fixed signed `recipe_worker.exe` package is not shipped at the immutable - installer generation used by the launcher. -- The fixed signed `recipe_worker.exe` bundle is not installed in the immutable - generation used by the launcher, so no real worker has completed the broker - handshake yet. -- The packaged worker loop is now broker-only and fail-closed; the disposable - signed-worker qualification covers authenticated input/output, hostile decoder, - and cancellation cases, but the immutable production generation is not shipped - at the launcher path. -- Real signed-worker resource enforcement, external security review, and lifecycle - health-gated wiring remain separate release gates. +## Remaining release-hardening boundaries + +- The open-source qualification path uses a disposable ephemeral signing key and + temporary immutable installer generation; a production signed package and + pinned trust root are still optional official-release material. +- External security review and release-specific resource/enforcement review remain + separate official-release gates. + +The disposable signed-worker qualification and the strict durable-coordinator +qualification now prove a real broker handshake and native process lifecycle on +Windows. Missing or unverifiable worker material still blocks the profile; there +is no host-process or weaker-sandbox fallback. If any control is missing or cannot be verified, the provider remains unavailable; there is no host-process or weaker-sandbox fallback. diff --git a/docs/adr/0001-phase2-qualification-lifecycle.md b/docs/adr/0001-phase2-qualification-lifecycle.md index 8a8e97c..d3a7738 100644 --- a/docs/adr/0001-phase2-qualification-lifecycle.md +++ b/docs/adr/0001-phase2-qualification-lifecycle.md @@ -68,10 +68,11 @@ The recipe-specific coordinator/request, trusted attachment staging, native attempt composition, and artifact-publication path plus their typed qualification-only API surfaces are now implemented behind this lifecycle boundary. It preserves the same release gate, native launcher, broker identity, -resource/watchdog, and trusted artifact controls. The next slice is an -end-to-end durable-coordinator run against the packaged signed worker, including -cancellation, retention, publication, and native cleanup; it must remain -default-off in the application. +resource/watchdog, and trusted artifact controls. The durable-coordinator run +against the packaged signed worker is now implemented as a separate strict +qualification probe; it exercises cancellation, retention, publication, +owner-isolation, and native cleanup while remaining default-off in the +application. ## Verification @@ -82,3 +83,16 @@ diagnostics, clean stop, and repository-bound worker-attempt factory wiring. `tests/test_phase2_recipe_api.py` covers both routes' ready-lifecycle gates and owner/idempotency/error behavior. `tests/test_phase2_native_recipe_attempt.py` covers explicit native composition without launch during configuration. +The packaged-worker coordinator gate is +`tools/execution_spikes/recipe_coordinator_e2e_qualification.py`; its local +Windows run passed the transform, owner-isolation, retention, cancellation, +atomic-publication, and native-cleanup cases. Quality CI runs it after the +packaged worker release qualification. + +## Next stage + +Hosted Quality CI run [30466976650](https://github.com/dovvnloading/Cortex/actions/runs/30466976650) +and job [90627154917](https://github.com/dovvnloading/Cortex/actions/runs/30466976650/job/90627154917) +passed the coordinator gate on commit `0c7b350`. The immediate remaining action +is review/merge of PR #64; the application remains disabled unless an explicit +caller injects a ready qualification profile. diff --git a/docs/adr/0001-phase2-recipe-contract.md b/docs/adr/0001-phase2-recipe-contract.md index d65946f..df57c4e 100644 --- a/docs/adr/0001-phase2-recipe-contract.md +++ b/docs/adr/0001-phase2-recipe-contract.md @@ -69,15 +69,16 @@ profile is now deliberately composable through an explicit injected lifecycle builder. Outside review and production trust are optional official-release hardening, not open-source prerequisites. -## Required next gates - -1. Add trusted attachment staging and bind the real signed/native broker worker - into the injected attempt factory behind the passing lifecycle health check. - The qualification-only request surface and implemented coordinator are - documented in [the coordinator ADR](0001-phase2-recipe-coordinator.md); the - lifecycle builder intentionally does not create processes or discover - artifacts. Official prebuilt releases may add external review and production - trust as optional hardening. +## Required next gate + +The trusted attachment boundary, signed/native attempt factory, and durable +packaged-worker coordinator qualification are implemented behind the passing +lifecycle health check. Hosted Quality run [30466976650](https://github.com/dovvnloading/Cortex/actions/runs/30466976650) +passed `recipe_coordinator_e2e_qualification.py` on commit `0c7b350`; the +immediate remaining action is review and merge. The lifecycle builder still +intentionally does not create processes or discover artifacts. Official +prebuilt releases may add external review and production trust as optional +hardening. ## Verification diff --git a/docs/adr/0001-phase2-recipe-coordinator.md b/docs/adr/0001-phase2-recipe-coordinator.md index c382c75..ac889c2 100644 --- a/docs/adr/0001-phase2-recipe-coordinator.md +++ b/docs/adr/0001-phase2-recipe-coordinator.md @@ -2,7 +2,8 @@ - **Status:** Implemented and verified behind an explicit qualification-only API boundary, including trusted attachment staging and signed/native attempt - composition; the normal application remains default-off + composition and a durable packaged-worker qualification corpus; the normal + application remains default-off - **Phase:** 2 - fixed-function image recipe - **Parent:** [Capability-tiered agentic execution harness](0001-capability-tiered-agentic-execution-harness.md) - **Depends on:** [typed recipe contract](0001-phase2-recipe-contract.md), @@ -136,13 +137,22 @@ composition, and bounded cleanup. The generated OpenAPI and TypeScript contracts include both typed envelopes, and `frontend/src/api/client.ts` exposes the qualification routes without enabling them in the UI by default. +The qualification-only packaged-worker gate is +`tools/execution_spikes/recipe_coordinator_e2e_qualification.py`. It signs the +already-built one-folder worker with an in-memory ephemeral key, installs one +disposable immutable generation, stages a PNG through `AttachmentStagingService`, +and drives the real `RecipeExecutionCoordinator` through the native +AppContainer/broker/client attempt. Its strict corpus verifies successful +publication and digest/MIME/size binding, foreign-owner rejection, retention +expiry and purge, in-flight cancellation with no result artifact, absence of +temporary publication directories, and complete native process cleanup. The +probe is Windows-only, bounded, redacted, and has no host-process fallback. + ## Next stage -The next implementation decision is an end-to-end qualification run through the -durable coordinator using the packaged signed worker: stage an attachment, -launch/bind the native attempt, exercise transform and cancellation, and verify -retention, owner isolation, atomic publication, and complete native cleanup. -That run must preserve this API/coordinator contract and keep the normal -application default-off. No external reviewer or production key is required to -continue the open-source qualification path; production trust remains optional -release hardening. +The coordinator implementation and hosted Quality CI gates are complete. The +immediate remaining action is review/merge of PR #64. Any follow-on Phase 3 +arbitrary-code profile requires its own ADR and qualification gates; this +API/coordinator contract must remain default-off. No external reviewer or +production key is required for the open-source qualification path; production +trust remains optional release hardening. diff --git a/docs/adr/0001-phase2-recipe-provider.md b/docs/adr/0001-phase2-recipe-provider.md index 6e7585b..76e150a 100644 --- a/docs/adr/0001-phase2-recipe-provider.md +++ b/docs/adr/0001-phase2-recipe-provider.md @@ -120,9 +120,14 @@ health-gated builder. The durable recipe coordinator binds that worker contract to owner-scoped staged artifacts and atomic publication, and the typed API route is available only after the lifecycle is ready. Trusted attachment staging and signed/native attempt composition are now present behind that same lifecycle. -The next gate is an end-to-end durable coordinator run through the packaged -worker: the provider must launch out of process, complete a staged transform, -cancel in flight, and prove retention, publication, and native cleanup. It must -never fall back to a host process or in-process execution. +The packaged-worker coordinator gate is now implemented by +`tools/execution_spikes/recipe_coordinator_e2e_qualification.py`. Its strict +Windows corpus launches the provider out of process, completes a staged +transform, rejects a foreign owner, expires and purges a short-retention +attachment, cancels in flight, verifies atomic publication, and proves native +cleanup. It never falls back to a host process or in-process execution. Hosted +Quality run [30466976650](https://github.com/dovvnloading/Cortex/actions/runs/30466976650) +passed this gate on commit `0c7b350`; the immediate remaining action is review +and merge. External review and production signing remain optional official-release hardening. diff --git a/docs/adr/0001-phase2-sandbox-qualification.md b/docs/adr/0001-phase2-sandbox-qualification.md index 1b65101..909f9ee 100644 --- a/docs/adr/0001-phase2-sandbox-qualification.md +++ b/docs/adr/0001-phase2-sandbox-qualification.md @@ -109,6 +109,7 @@ default-off; the explicit qualification profile is now a deliberate, injected lifecycle composition rather than an application default. Official production readiness, signing, and external review are separate optional maintainer concerns and are not prerequisites for open-source development. The recipe -coordinator/request path and typed qualification-only API are implemented behind -this boundary; the next implementation slice is trusted attachment staging and -binding the real qualified attempt factory. +coordinator/request path, trusted attachment staging, and qualified native +attempt factory are implemented behind this boundary. The durable packaged +worker coordinator corpus now provides the next composite evidence gate while +the normal application remains default-off. diff --git a/docs/adr/0001-phase2-worker-protocol.md b/docs/adr/0001-phase2-worker-protocol.md index 8980cb1..81afa34 100644 --- a/docs/adr/0001-phase2-worker-protocol.md +++ b/docs/adr/0001-phase2-worker-protocol.md @@ -84,7 +84,7 @@ On the controlled Windows host (2026-07-23), the PyInstaller build produced `dist/recipe-runtime/recipe_worker.exe`; direct launch without the exact native broker arguments returned exit code `78` as required. -## Remaining qualification gate +## Coordinator qualification gate The durable coordinator now has an explicit native attempt factory that installs no fallback transport: each job creates a fresh pipe/broker identity, launches a @@ -94,9 +94,15 @@ and closes the client, broker, and process tree on failure or cancellation. Attachment staging is likewise available only through the owner-scoped artifact boundary and qualification lifecycle. +The strict coordinator qualification probe now runs the durable path through the +packaged process as a separate Windows-only gate. It stages an attachment, +completes a typed transform, rejects a foreign owner, proves retention expiry, +cancels an in-flight transform, checks atomic publication/no-result cleanup, and +verifies native process teardown. The existing worker probe continues to own the +hostile decoder corpus; the coordinator probe composes that same signed worker +without duplicating provider authority. + This ADR still does not authorize the normal application to execute providers. -The next gate must run the full hostile decoder/cancellation corpus through the -durable coordinator using that packaged process and verify retention, atomic -publication, and cleanup. Lifecycle/UI enablement remains behind those gates; -external security review and production trust remain optional official-release -hardening. +Quality CI runs both qualification probes with bounded timeouts; lifecycle/UI +enablement remains behind the explicit qualification profile, and external +security review and production trust remain optional official-release hardening. diff --git a/docs/adr/0001-phase2-worker-release-qualification.md b/docs/adr/0001-phase2-worker-release-qualification.md index 23ea056..a7bd3bb 100644 --- a/docs/adr/0001-phase2-worker-release-qualification.md +++ b/docs/adr/0001-phase2-worker-release-qualification.md @@ -23,6 +23,13 @@ unsigned qualification artifact; the ephemeral signature is discarded with the temporary store. A green run therefore proves the packaged boundary on the runner, not production release authorization. +Quality CI follows this worker-level corpus with +`tools/execution_spikes/recipe_coordinator_e2e_qualification.py`. That separate +probe composes the same freshly installed generation through the durable +coordinator and verifies owner-scoped attachment staging, publication, +retention, cancellation, and native cleanup; it does not add a second provider +or trust root. + ## Bounded execution The package build has a 15-minute step timeout and the end-to-end qualification diff --git a/tests/test_phase2_recipe_coordinator_qualification.py b/tests/test_phase2_recipe_coordinator_qualification.py new file mode 100644 index 0000000..6e34cb0 --- /dev/null +++ b/tests/test_phase2_recipe_coordinator_qualification.py @@ -0,0 +1,37 @@ +"""Qualification-only coordinator probe remains explicit and fail-closed.""" + +from __future__ import annotations + +from pathlib import Path + +from tools.execution_spikes import recipe_coordinator_e2e_qualification as qualification + + +def test_coordinator_probe_blocks_without_native_windows(monkeypatch, tmp_path: Path): + monkeypatch.setattr(qualification.os, "name", "posix") + + result = qualification.qualify( + tmp_path / "recipe-runtime", + timeout_seconds=10, + ) + + assert result == { + "status": "blocked", + "code": "native_windows_required", + "stages": [], + "cases": {}, + } + + +def test_coordinator_probe_reports_missing_package_without_fallback(monkeypatch, tmp_path: Path): + monkeypatch.setattr(qualification.os, "name", "nt") + + result = qualification.qualify( + tmp_path / "missing-recipe-runtime", + timeout_seconds=10, + ) + + assert result["status"] == "blocked" + assert result["code"] == "package_missing" + assert result["stages"] == [] + assert result["cases"] == {} diff --git a/tools/execution_spikes/recipe_coordinator_e2e_qualification.py b/tools/execution_spikes/recipe_coordinator_e2e_qualification.py new file mode 100644 index 0000000..c63661d --- /dev/null +++ b/tools/execution_spikes/recipe_coordinator_e2e_qualification.py @@ -0,0 +1,468 @@ +"""Durable recipe-coordinator qualification against the signed native worker. + +This probe is intentionally separate from normal Cortex startup. It creates a +disposable repository, signs and installs the already-built fixed worker with +an ephemeral key, composes the explicit native coordinator factory, and then +drives the same owner-scoped attachment, transform, cancellation, retention, +publication, and cleanup boundaries that a qualified caller would use. + +The probe is Windows-only by design. A non-Windows invocation is reported as +``native_windows_required``; it never substitutes a host process, an in-memory +worker, or an alternate transport. No source path or executable authority is +placed in a job payload, and all temporary state is removed before the probe +returns. +""" + +from __future__ import annotations + +import argparse +from hashlib import sha256 +import json +import os +from pathlib import Path +import shutil +import sys +import time +from tempfile import mkdtemp +from typing import Any, Callable + +ROOT = Path(__file__).resolve().parents[2] +SPIKE_ROOT = Path(__file__).resolve().parent +sys.path.insert(0, str(ROOT / "backend")) +sys.path.insert(0, str(SPIKE_ROOT)) + +from cortex_backend.execution.artifact_boundary import ArtifactBoundary # noqa: E402 +from cortex_backend.execution.attachment_staging import ( # noqa: E402 + AttachmentStagingService, +) +from cortex_backend.execution.bundle_installer import SignedBundleInstaller # noqa: E402 +from cortex_backend.execution.native_win32 import NativeWin32ProcessFactory # noqa: E402 +from cortex_backend.execution.qualification import ( # noqa: E402 + build_native_recipe_coordinator_factory, +) +from cortex_backend.execution.recipe_coordinator import ( # noqa: E402 + RecipeExecutionCoordinator, + RecipeExecutionError, + RecipeImageRequest, +) +from cortex_backend.execution.repository import ( # noqa: E402 + ExecutionRepository, + ExecutionRepositoryError, +) +from cortex_backend.execution.worker_provenance import verify_active_worker # noqa: E402 +from recipe_worker_e2e_qualification import ( # noqa: E402 + _bounded_cleanup, + _current_user_sid, + _fixed_png, + _install_ephemeral, + _plan, + _process_executable, + _slow_png, +) + + +DEFAULT_TIMEOUT_SECONDS = 300.0 +ATTACHMENT_RETENTION_SECONDS = 300 +OUTPUT_RETENTION_SECONDS = 300 +SHORT_RETENTION_SECONDS = 1 +FOREIGN_OWNER = "f" * 64 + + +class QualificationFailure(RuntimeError): + """A deterministic, redacted coordinator qualification failure.""" + + def __init__(self, code: str) -> None: + self.code = code + super().__init__(code) + + +class _RecordingProcessFactory: + """Record native workers so the qualification can prove they were closed.""" + + def __init__(self, workers: list[Any]) -> None: + self._workers = workers + self._delegate = NativeWin32ProcessFactory() + + def create_suspended(self, plan: Any) -> Any: + worker = self._delegate.create_suspended(plan) + self._workers.append(worker) + return worker + + +def _require(condition: bool, code: str) -> None: + if not condition: + raise QualificationFailure(code) + + +def _wait_until(predicate: Callable[[], bool], timeout_seconds: float, code: str) -> None: + deadline = time.monotonic() + timeout_seconds + while not predicate(): + if time.monotonic() >= deadline: + raise QualificationFailure(code) + time.sleep(0.01) + + +def _artifact_ids_for_job(repository: ExecutionRepository, job_id: str) -> list[str]: + with repository.connect() as connection: + rows = connection.execute( + "SELECT artifact_id FROM execution_artifacts WHERE job_id = ? ORDER BY artifact_id", + (job_id,), + ).fetchall() + return [str(row["artifact_id"]) for row in rows] + + +def _temporary_recipe_directories(repository: ExecutionRepository) -> list[Path]: + return [ + path + for path in repository.artifact_root.iterdir() + if path.name.startswith(".recipe-") + ] + + +def _wait_for_worker( + repository: ExecutionRepository, + job_id: str, + workers: list[Any], + minimum_worker_count: int, + timeout_seconds: float, +) -> None: + """Wait until the coordinator has both entered the worker phase and launched it.""" + + def ready() -> bool: + job = repository.get_job(job_id) + if job is None: + return False + phases = {event.phase for event in repository.events(job_id)} + return "worker" in phases and len(workers) > minimum_worker_count + + _wait_until(ready, timeout_seconds, "worker_launch_timeout") + + +def _successful_transform( + repository: ExecutionRepository, + coordinator: RecipeExecutionCoordinator, + workers: list[Any], + owner: str, + source_artifact_id: str, + timeout_seconds: float, +) -> dict[str, Any]: + request = RecipeImageRequest( + owner=owner, + request_id="recipe-transform", + source_artifact_id=source_artifact_id, + plan=_plan(artifact_id=source_artifact_id), + retention_seconds=OUTPUT_RETENTION_SECONDS, + ) + accepted = coordinator.start_image_transform(request) + completed = coordinator.wait(accepted.job_id, timeout=timeout_seconds) + _require(completed.status == "succeeded", "coordinator_transform_not_succeeded") + _require(isinstance(completed.result, dict), "coordinator_result_missing") + result = completed.result + expected_keys = { + "schema_version", + "artifact_id", + "mime_type", + "format", + "size", + "sha256", + "width", + "height", + "plan_digest", + } + _require(set(result) == expected_keys, "coordinator_result_shape_invalid") + _require("path" not in result, "coordinator_result_path_leak") + output_id = result.get("artifact_id") + _require(isinstance(output_id, str), "coordinator_result_artifact_invalid") + artifact = repository.get_artifact(output_id, owner=owner) + _require(artifact is not None, "coordinator_output_unavailable") + _require(artifact.job_id == accepted.job_id, "coordinator_output_owner_mismatch") + content = repository.read_artifact(output_id) + _require(sha256(content).hexdigest() == result.get("sha256"), "coordinator_output_hash_invalid") + _require(artifact.sha256 == result.get("sha256"), "coordinator_output_digest_mismatch") + _require(artifact.mime_type == result.get("mime_type") == "image/png", "coordinator_output_mime_invalid") + _require(result.get("size") == len(content) == artifact.size, "coordinator_output_size_invalid") + _require(result.get("width") == 4 and result.get("height") == 3, "coordinator_output_dimensions_invalid") + _require(not _temporary_recipe_directories(repository), "atomic_publication_staging_leaked") + _require(bool(workers), "native_worker_not_launched") + return { + "status": "passed", + "job_id": accepted.job_id, + "source_artifact_id": source_artifact_id, + "output_artifact_id": output_id, + "output_sha256": artifact.sha256, + } + + +def _owner_isolation( + repository: ExecutionRepository, + coordinator: RecipeExecutionCoordinator, + source_artifact_id: str, +) -> dict[str, Any]: + _require( + repository.get_artifact(source_artifact_id, owner=FOREIGN_OWNER) is None, + "owner_artifact_metadata_leak", + ) + try: + coordinator.start_image_transform( + RecipeImageRequest( + owner=FOREIGN_OWNER, + request_id="foreign-transform", + source_artifact_id=source_artifact_id, + plan=_plan(artifact_id=source_artifact_id), + retention_seconds=OUTPUT_RETENTION_SECONDS, + ) + ) + except RecipeExecutionError as error: + _require(error.code == "input_artifact_unavailable", "owner_isolation_error_invalid") + else: + raise QualificationFailure("owner_isolation_bypass") + return {"status": "passed", "source_artifact_id": source_artifact_id} + + +def _retention( + repository: ExecutionRepository, + staging: AttachmentStagingService, + owner: str, +) -> dict[str, Any]: + attachment = staging.stage( + owner=owner, + request_id="attachment-short-retention", + content=_fixed_png(), + retention_seconds=SHORT_RETENTION_SECONDS, + ) + time.sleep(SHORT_RETENTION_SECONDS + 0.25) + try: + repository.read_artifact(attachment.artifact.artifact_id) + except ExecutionRepositoryError: + pass + else: + raise QualificationFailure("retention_expiry_not_enforced") + removed = repository.purge_expired() + _require(repository.get_artifact(attachment.artifact.artifact_id, owner=owner) is None, "retention_purge_failed") + return { + "status": "passed", + "artifact_id": attachment.artifact.artifact_id, + "purged_artifacts": removed, + } + + +def _cancellation( + repository: ExecutionRepository, + coordinator: RecipeExecutionCoordinator, + staging: AttachmentStagingService, + workers: list[Any], + owner: str, + timeout_seconds: float, +) -> dict[str, Any]: + attachment = staging.stage( + owner=owner, + request_id="attachment-cancellation", + content=_slow_png(), + retention_seconds=ATTACHMENT_RETENTION_SECONDS, + ) + source_id = attachment.artifact.artifact_id + accepted = coordinator.start_image_transform( + RecipeImageRequest( + owner=owner, + request_id="recipe-cancellation", + source_artifact_id=source_id, + plan=_plan(artifact_id=source_id, slow=True), + retention_seconds=OUTPUT_RETENTION_SECONDS, + ) + ) + worker_count_before = len(workers) + _wait_for_worker( + repository, + accepted.job_id, + workers, + worker_count_before, + timeout_seconds, + ) + coordinator.cancel(accepted.job_id, owner=owner) + completed = coordinator.wait(accepted.job_id, timeout=timeout_seconds) + _require(completed.status == "cancelled", "coordinator_cancellation_not_terminal") + _require(completed.error == "cancelled", "coordinator_cancellation_error_invalid") + _require(completed.result is None, "coordinator_cancellation_published_result") + _require(not _artifact_ids_for_job(repository, accepted.job_id), "coordinator_cancellation_artifact_leak") + _require(not _temporary_recipe_directories(repository), "coordinator_cancellation_staging_leaked") + return {"status": "passed", "job_id": accepted.job_id, "source_artifact_id": source_id} + + +def _native_cleanup( + workers: list[Any], + workspace: Path, +) -> None: + """Treat a still-running workspace worker as a qualification failure.""" + + leaked: list[int] = [] + root = workspace.resolve() + for worker in workers: + process_id = getattr(worker, "process_id", None) + if type(process_id) is not int: + continue + executable = _process_executable(process_id) + if executable is not None: + try: + if executable.is_relative_to(root): + leaked.append(process_id) + except (OSError, RuntimeError): + leaked.append(process_id) + _require(not leaked, "native_worker_process_leaked") + + +def qualify( + source_root: Path, + *, + timeout_seconds: float, + case_name: str | None = None, +) -> dict[str, Any]: + """Run the strict durable coordinator qualification corpus.""" + + if os.name != "nt": + return {"status": "blocked", "code": "native_windows_required", "stages": [], "cases": {}} + if not source_root.is_dir(): + return {"status": "blocked", "code": "package_missing", "stages": [], "cases": {}} + + workspace = Path(mkdtemp(prefix="cortex-recipe-coordinator-e2e-")) + stages: list[str] = [] + cases: dict[str, dict[str, Any]] = {} + workers: list[Any] = [] + repository: ExecutionRepository | None = None + coordinator: RecipeExecutionCoordinator | None = None + cleanup_error: str | None = None + result: dict[str, Any] | None = None + try: + installer: SignedBundleInstaller = _install_ephemeral(source_root, workspace / "store") + stages.extend(["signed_ephemeral_manifest", "installed_immutable_generation"]) + verify_active_worker(installer) + stages.extend(["provenance_verified", "no_host_fallback_configured"]) + + repository = ExecutionRepository( + workspace / "execution.sqlite", + workspace / "artifacts", + max_artifact_bytes=10 * 1024 * 1024, + ) + boundary = ArtifactBoundary(repository) + staging = AttachmentStagingService(repository, boundary) + owner = repository.installation_principal_id + _require(len(owner) == 64 and all(character in "0123456789abcdef" for character in owner), "installation_owner_invalid") + _require(owner != FOREIGN_OWNER, "owner_identity_collision") + + def process_factory_factory() -> _RecordingProcessFactory: + return _RecordingProcessFactory(workers) + + factory = build_native_recipe_coordinator_factory( + installer, + allowed_user_sids=frozenset({_current_user_sid()}), + process_factory_factory=process_factory_factory, + accept_timeout_seconds=min(timeout_seconds, 30.0), + worker_timeout_seconds=min(timeout_seconds, 120.0), + cancel_grace_seconds=5.0, + artifact_boundary_factory=lambda current: boundary if current is repository else ArtifactBoundary(current), + ) + coordinator = factory(repository) + stages.extend(["repository_ready", "attachment_boundary_ready", "native_coordinator_composed"]) + base_attachment = staging.stage( + owner=owner, + request_id="attachment-base", + content=_fixed_png(), + retention_seconds=ATTACHMENT_RETENTION_SECONDS, + ) + _require(base_attachment.job.status == "succeeded", "attachment_stage_not_succeeded") + source_artifact_id = base_attachment.artifact.artifact_id + stages.append("attachment_staged") + + selected = { + "transform": lambda: _successful_transform( + repository, + coordinator, + workers, + owner, + source_artifact_id, + timeout_seconds, + ), + "owner_isolation": lambda: _owner_isolation( + repository, + coordinator, + source_artifact_id, + ), + "retention": lambda: _retention(repository, staging, owner), + "cancellation": lambda: _cancellation( + repository, + coordinator, + staging, + workers, + owner, + timeout_seconds, + ), + } + for name, action in selected.items(): + if case_name is not None and case_name != name: + continue + try: + cases[name] = action() + stages.append(f"{name}_verified") + except QualificationFailure as error: + cases[name] = {"status": "blocked", "code": error.code} + raise + except Exception as error: + cases[name] = {"status": "blocked", "code": "qualification_case_failed", "error_type": type(error).__name__} + raise + _require(case_name is None or case_name in selected, "qualification_case_invalid") + result = {"status": "passed", "stages": stages, "cases": cases} + except QualificationFailure as error: + result = {"status": "blocked", "code": error.code, "stages": stages, "cases": cases} + except Exception: + result = {"status": "blocked", "code": "qualification_failed_closed", "stages": stages, "cases": cases} + finally: + if coordinator is not None: + try: + coordinator.shutdown(timeout=min(timeout_seconds, 30.0)) + except Exception: + cleanup_error = "coordinator_shutdown_failed" + if repository is not None: + try: + _native_cleanup(workers, workspace) + stages.append("native_worker_processes_closed") + except QualificationFailure as error: + cleanup_error = error.code + _bounded_cleanup(lambda: shutil.rmtree(workspace)) + if workspace.exists(): + cleanup_error = cleanup_error or "qualification_workspace_cleanup_failed" + + assert result is not None + if cleanup_error is not None: + result = { + "status": "blocked", + "code": cleanup_error, + "stages": stages, + "cases": cases, + } + return result + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-root", type=Path, default=ROOT / "dist" / "recipe-runtime") + parser.add_argument("--timeout-seconds", type=float, default=DEFAULT_TIMEOUT_SECONDS) + parser.add_argument("--json", action="store_true") + parser.add_argument("--strict", action="store_true") + parser.add_argument( + "--case", + choices=("transform", "owner_isolation", "retention", "cancellation"), + help="qualify one coordinator case instead of the full corpus", + ) + args = parser.parse_args(argv) + if not 10 <= args.timeout_seconds <= 600: + parser.error("--timeout-seconds must be between 10 and 600") + result = qualify( + args.source_root.resolve(), + timeout_seconds=args.timeout_seconds, + case_name=args.case, + ) + print(json.dumps(result, sort_keys=True, separators=(",", ":") if args.json else None)) + return 2 if args.strict and result["status"] != "passed" else 0 + + +if __name__ == "__main__": + raise SystemExit(main())