diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index dd675577..64892638 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -317,6 +317,52 @@ jobs:
- name: Python 3.11 tests (native core)
run: .venv/bin/python -m pytest -q
+ notebook_smoke:
+ name: Notebook smoke (${{ matrix.python-version }})
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+ strategy:
+ fail-fast: false
+ matrix:
+ python-version: ["3.11", "3.13"]
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ # Full history and tags: the distribution version is derived
+ # from the latest `v*` tag, and a shallow clone has none.
+ fetch-depth: 0
+ - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+ with:
+ python-version: ${{ matrix.python-version }}
+ - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
+ with:
+ enable-cache: false
+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+ with:
+ node-version: "22"
+
+ - name: Build native core and render client
+ run: |
+ npm ci
+ node js/build.mjs
+ cargo build --release
+
+ - name: Install notebook smoke environment
+ env:
+ XY_SKIP_CARGO: "1"
+ run: |
+ uv venv .venv --python "$(command -v python)"
+ uv pip install -p .venv/bin/python \
+ --constraint benchmarks/requirements-ci.lock \
+ -e . matplotlib requests
+
+ - name: Execute deterministic public notebooks
+ env:
+ MPLBACKEND: Agg
+ XY_NOTEBOOK_DISPLAY: html
+ run: .venv/bin/python scripts/run_notebook_smoke.py --profile pr
+
# Cross-library scatter benchmark incl. time-to-first-render (spec/benchmarks/results.md).
# Each isolated group uses the same deterministic input sizes and runs in
# parallel. The report job below merges the partial results.
diff --git a/scripts/notebook_smoke_pr_oracle.json b/scripts/notebook_smoke_pr_oracle.json
new file mode 100644
index 00000000..7ec9c44e
--- /dev/null
+++ b/scripts/notebook_smoke_pr_oracle.json
@@ -0,0 +1,188 @@
+{
+ "basic-xy": [
+ {
+ "kind": "_repr_html_",
+ "sha256": "32948ca725e9a48a2c146dc9c3fa37efddad8e2c71f0d954761723214ed86717",
+ "size": 453331
+ }
+ ],
+ "pdsh-matplotlib-shim": [
+ {
+ "kind": "_repr_html_",
+ "sha256": "1fa4017618eb5b923ba98ddc0e832fa2cc94253b5000aad856f0f25e0eca0263",
+ "size": 452910
+ },
+ {
+ "kind": "_repr_html_",
+ "sha256": "1fa4017618eb5b923ba98ddc0e832fa2cc94253b5000aad856f0f25e0eca0263",
+ "size": 452910
+ },
+ {
+ "kind": "_repr_html_",
+ "sha256": "43c614dceb9980358bf04ab0fe59d3facabc62ff58516316ac7b356f89f951c2",
+ "size": 464467
+ },
+ {
+ "kind": "_repr_html_",
+ "sha256": "729ce1a70252d929180ce387b9348e88029c875ae96017c07297a4844ff9ca9f",
+ "size": 464279
+ },
+ {
+ "kind": "_repr_html_",
+ "sha256": "43c614dceb9980358bf04ab0fe59d3facabc62ff58516316ac7b356f89f951c2",
+ "size": 464467
+ },
+ {
+ "kind": "_repr_html_",
+ "sha256": "729ce1a70252d929180ce387b9348e88029c875ae96017c07297a4844ff9ca9f",
+ "size": 464279
+ },
+ {
+ "kind": "_repr_html_",
+ "sha256": "335319a249a04f4848e2235523465b016cb6ae446897004d379fd5b48bffbd8e",
+ "size": 475810
+ },
+ {
+ "kind": "_repr_html_",
+ "sha256": "5df34244280196fd91bb0340d0ed552844aa9067397c8c37054af0add441d81f",
+ "size": 475612
+ },
+ {
+ "kind": "repr",
+ "sha256": "6152722ba8866dafeff93f6731a30986f037e57191fc2979598a64caab5d53f4",
+ "size": 43
+ },
+ {
+ "kind": "_repr_html_",
+ "sha256": "d196093ae573f3c7b7e1711144a337452280521d13649b5c235912968ea235bf",
+ "size": 521103
+ },
+ {
+ "kind": "repr",
+ "sha256": "cb9b59b6542ec9b004af385aa8a8338d61ebf5ea0ed3b633a542413ecb57ab36",
+ "size": 45
+ },
+ {
+ "kind": "_repr_html_",
+ "sha256": "97c61bdec7cf771c2e5a6dd0a9f4d48c64df7035f3a28400545b55dbff198b87",
+ "size": 520925
+ },
+ {
+ "kind": "repr",
+ "sha256": "6152722ba8866dafeff93f6731a30986f037e57191fc2979598a64caab5d53f4",
+ "size": 43
+ },
+ {
+ "kind": "_repr_html_",
+ "sha256": "d6e9959f18264fcf645ac0f065fc3c3837689f03f4a5c2175f04b221c628783d",
+ "size": 543639
+ },
+ {
+ "kind": "repr",
+ "sha256": "cb9b59b6542ec9b004af385aa8a8338d61ebf5ea0ed3b633a542413ecb57ab36",
+ "size": 45
+ },
+ {
+ "kind": "_repr_html_",
+ "sha256": "a35b7a73f3b6185938caab9e87a031ec3e51934a1afeca4e26b74508807573f6",
+ "size": 543647
+ },
+ {
+ "kind": "repr",
+ "sha256": "6152722ba8866dafeff93f6731a30986f037e57191fc2979598a64caab5d53f4",
+ "size": 43
+ },
+ {
+ "kind": "_repr_html_",
+ "sha256": "fed9b4474a1bdc5683dabd09b7e33d087aa04500fd23c2265760ac9cc7fe2f1e",
+ "size": 498239
+ },
+ {
+ "kind": "repr",
+ "sha256": "cb9b59b6542ec9b004af385aa8a8338d61ebf5ea0ed3b633a542413ecb57ab36",
+ "size": 45
+ },
+ {
+ "kind": "_repr_html_",
+ "sha256": "207b453e2505575b8a03a00f7f98ea35b800e83bb5911be88978a87b85895018",
+ "size": 498225
+ },
+ {
+ "kind": "_repr_html_",
+ "sha256": "6d2c5986f8f5cf9c21c7726ce1e5a949a4aaf35a5114f9a734f12678ddaf1dde",
+ "size": 464319
+ },
+ {
+ "kind": "_repr_html_",
+ "sha256": "45b9329d931eb610c4b912797c6711f55916759361910940e1b13551626ddc05",
+ "size": 464311
+ },
+ {
+ "kind": "_repr_html_",
+ "sha256": "c9e288cfaffb27240893385ceefab256d1372b6f5410a40cab1415b9a98b5d39",
+ "size": 464414
+ },
+ {
+ "kind": "_repr_html_",
+ "sha256": "6295b660443a92941e97f1dc442907d30c54c0d4a93e561eba38a804b3990810",
+ "size": 464406
+ },
+ {
+ "kind": "_repr_html_",
+ "sha256": "43c614dceb9980358bf04ab0fe59d3facabc62ff58516316ac7b356f89f951c2",
+ "size": 464467
+ },
+ {
+ "kind": "_repr_html_",
+ "sha256": "6ebd1ed058d23e1cfa2a7d70f09d0157912c0099321e19cda0fa10d84c753845",
+ "size": 464459
+ },
+ {
+ "kind": "_repr_html_",
+ "sha256": "43c614dceb9980358bf04ab0fe59d3facabc62ff58516316ac7b356f89f951c2",
+ "size": 464467
+ },
+ {
+ "kind": "_repr_html_",
+ "sha256": "2148c64486ba05fe93e02ec422a9275430de299243ab029284a32b6c933c8bd3",
+ "size": 464451
+ },
+ {
+ "kind": "_repr_html_",
+ "sha256": "13a11af3ec9ed5c265b48994a42975b4ab4d4f28a38f29eb5a5f69a75bb1cfdb",
+ "size": 464738
+ },
+ {
+ "kind": "_repr_html_",
+ "sha256": "c90f206396ab2da5576aac81d1e08d06d187b045bfd7e26112f676d4b9155d76",
+ "size": 464606
+ },
+ {
+ "kind": "_repr_html_",
+ "sha256": "6994c103a917c8f956bb956bebbffc89a319643927ee45f57df4e9089a084c80",
+ "size": 475794
+ },
+ {
+ "kind": "_repr_html_",
+ "sha256": "98fd69d80ed8838f98a147d640615b7fed79a6e4eb36f47a3a0063515c2bab13",
+ "size": 476646
+ },
+ {
+ "kind": "_repr_html_",
+ "sha256": "cdfd74a3ad4ec778fbdd2b2849e47475f57742a4f26e27fe51f14ba76d309629",
+ "size": 464588
+ },
+ {
+ "kind": "_repr_html_",
+ "sha256": "949f9e4203b982a8134cb3d0566df92fdff5c688ff9a47ae3a193ca02ee25309",
+ "size": 464639
+ }
+ ],
+ "real-world-gaia-reduced": [
+ {
+ "kind": "_repr_html_",
+ "sha256": "b1b033a8c6fbf985b014df2582d1d68da72fca4aec4e3430c1e507ee913e8001",
+ "size": 1767870
+ }
+ ]
+}
diff --git a/scripts/run_notebook_smoke.py b/scripts/run_notebook_smoke.py
new file mode 100644
index 00000000..2880d8bc
--- /dev/null
+++ b/scripts/run_notebook_smoke.py
@@ -0,0 +1,389 @@
+#!/usr/bin/env python3
+"""Execute a bounded public-notebook smoke suite.
+
+The public examples include notebooks that intentionally download large remote
+datasets. This runner keeps the PR gate deterministic by selecting a small
+representative set and pre-seeding the real-world notebook cache with fixture
+data before executing its cells.
+"""
+
+from __future__ import annotations
+
+import argparse
+import ast
+import csv
+import hashlib
+import json
+import os
+import re
+import signal
+import sys
+import tempfile
+import time
+import traceback
+from collections.abc import Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from pathlib import Path
+from types import FrameType
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_ORACLE = ROOT / "scripts" / "notebook_smoke_pr_oracle.json"
+ORACLE_OUTPUT_KEYS = {"kind", "sha256", "size"}
+DISPLAY_OUTPUT_KINDS = {"repr", "_repr_mimebundle_", "_repr_html_", "_repr_svg_", "_repr_png_"}
+SHA256_HEX = re.compile(r"^[0-9a-f]{64}$")
+
+
+@dataclass(frozen=True)
+class NotebookCase:
+ name: str
+ path: Path
+ env: dict[str, str]
+
+
+@dataclass(frozen=True)
+class DisplayOutput:
+ kind: str
+ sha256: str
+ size: int
+
+ def as_dict(self) -> dict[str, object]:
+ return {"kind": self.kind, "sha256": self.sha256, "size": self.size}
+
+
+@dataclass(frozen=True)
+class NotebookResult:
+ code_cells: int
+ display_outputs: int
+ outputs: tuple[DisplayOutput, ...] = field(default=(), compare=False, repr=False)
+
+
+@dataclass(frozen=True)
+class CellResult:
+ outputs: tuple[DisplayOutput, ...]
+ displayed_ids: frozenset[int]
+
+
+def _write_gaia_fixture(data_dir: Path, rows: int) -> None:
+ data_dir.mkdir(parents=True, exist_ok=True)
+ csv_path = data_dir / f"gaia-dr3-hr-{rows}.csv"
+ with csv_path.open("w", newline="", encoding="utf-8") as file:
+ writer = csv.writer(file)
+ writer.writerow(["bp_rp", "phot_g_mean_mag", "parallax"])
+ for index in range(rows):
+ color = -0.25 + index * (2.6 / max(1, rows - 1))
+ magnitude = 5.8 + (index % 9) * 0.4
+ parallax = 4.0 + (index % 7) * 0.8
+ writer.writerow([f"{color:.6f}", f"{magnitude:.6f}", f"{parallax:.6f}"])
+
+
+def smoke_cases(tmp_dir: Path) -> list[NotebookCase]:
+ gaia_rows = 32
+ gaia_data = tmp_dir / "gaia"
+ _write_gaia_fixture(gaia_data, gaia_rows)
+ common_env = {
+ "MPLBACKEND": "Agg",
+ "XY_NOTEBOOK_DISPLAY": "html",
+ "XY_REAL_WORLD_DATA": str(gaia_data),
+ "GAIA_ROWS": str(gaia_rows),
+ }
+ return [
+ NotebookCase(
+ "basic-xy",
+ ROOT / "examples" / "symlog_axis.ipynb",
+ common_env,
+ ),
+ NotebookCase(
+ "pdsh-matplotlib-shim",
+ ROOT / "examples" / "pdsh" / "pdsh_04_01_simple_line_plots.ipynb",
+ common_env,
+ ),
+ NotebookCase(
+ "real-world-gaia-reduced",
+ ROOT / "examples" / "real_world" / "01_gaia_hr_diagram.ipynb",
+ common_env,
+ ),
+ ]
+
+
+@contextmanager
+def _patched_environment(values: dict[str, str]) -> Iterator[None]:
+ old = {key: os.environ.get(key) for key in values}
+ os.environ.update(values)
+ try:
+ yield
+ finally:
+ for key, value in old.items():
+ if value is None:
+ os.environ.pop(key, None)
+ else:
+ os.environ[key] = value
+
+
+@contextmanager
+def _cell_timeout(seconds: int) -> Iterator[None]:
+ if seconds <= 0:
+ raise ValueError("notebook cell timeout must be positive")
+ if not hasattr(signal, "SIGALRM"):
+ yield
+ return
+
+ def fail(_signum: int, _frame: FrameType | None) -> None:
+ raise TimeoutError(f"notebook cell exceeded {seconds}s")
+
+ previous = signal.signal(signal.SIGALRM, fail)
+ signal.alarm(seconds)
+ try:
+ yield
+ finally:
+ signal.alarm(0)
+ signal.signal(signal.SIGALRM, previous)
+
+
+def _validate_kernelspec(case: NotebookCase, notebook: dict[str, object]) -> None:
+ metadata = notebook.get("metadata", {})
+ kernelspec = metadata.get("kernelspec", {}) if isinstance(metadata, dict) else {}
+ if not isinstance(kernelspec, dict):
+ raise ValueError(f"{case.name}: metadata.kernelspec must be a mapping")
+ name = kernelspec.get("name")
+ language = kernelspec.get("language")
+ if name not in {"python", "python3"} or language != "python":
+ raise ValueError(f"{case.name}: unsupported kernelspec {kernelspec!r}; expected Python 3")
+
+
+def _stable_display_bytes(value: object) -> bytes:
+ if isinstance(value, bytes):
+ return value
+ if isinstance(value, str):
+ return value.encode("utf-8")
+ try:
+ return json.dumps(
+ value,
+ ensure_ascii=False,
+ sort_keys=True,
+ separators=(",", ":"),
+ ).encode("utf-8")
+ except TypeError:
+ return repr(value).encode("utf-8")
+
+
+def _display_output(kind: str, payload: object) -> DisplayOutput:
+ if kind == "repr":
+ payload = re.sub(r"0x[0-9a-fA-F]+", "0x...", str(payload))
+ data = _stable_display_bytes(payload)
+ return DisplayOutput(kind=kind, sha256=hashlib.sha256(data).hexdigest(), size=len(data))
+
+
+def _render_display_value(value: object) -> DisplayOutput | None:
+ if value is None:
+ return None
+ for method_name in ("_repr_mimebundle_", "_repr_html_", "_repr_svg_", "_repr_png_"):
+ method = getattr(value, method_name, None)
+ if callable(method):
+ return _display_output(method_name, method())
+ return _display_output("repr", repr(value))
+
+
+def _execute_cell(source: str, filename: str, namespace: dict[str, object]) -> CellResult:
+ module = ast.parse(source, filename=filename, mode="exec")
+ if module.body and isinstance(module.body[-1], ast.Expr) and not source.rstrip().endswith(";"):
+ prefix = ast.Module(body=module.body[:-1], type_ignores=module.type_ignores)
+ ast.fix_missing_locations(prefix)
+ exec(compile(prefix, filename, "exec"), namespace)
+ value = eval(compile(ast.Expression(module.body[-1].value), filename, "eval"), namespace)
+ output = _render_display_value(value)
+ return CellResult(
+ (() if output is None else (output,)),
+ frozenset({id(value)} if output is not None else ()),
+ )
+ exec(compile(module, filename, "exec"), namespace)
+ return CellResult((), frozenset())
+
+
+def _close_matplotlib_figures() -> None:
+ pyplot = sys.modules.get("matplotlib.pyplot")
+ close = getattr(pyplot, "close", None) if pyplot is not None else None
+ if callable(close):
+ close("all")
+
+
+def _close_notebook_figures() -> None:
+ """Release figures after a notebook while preserving cross-cell state."""
+ pyplot = sys.modules.get("xy.pyplot")
+ close = getattr(pyplot, "close", None) if pyplot is not None else None
+ if callable(close):
+ close("all")
+ _close_matplotlib_figures()
+
+
+def _flush_xy_pyplot_figures(*, skip_ids: frozenset[int]) -> tuple[DisplayOutput, ...]:
+ pyplot = sys.modules.get("xy.pyplot")
+ if pyplot is None:
+ return ()
+ all_figures = getattr(pyplot, "all_figures", None)
+ if not callable(all_figures):
+ return ()
+ figures = list(all_figures())
+ outputs = tuple(
+ output
+ for figure in figures
+ if id(figure) not in skip_ids
+ for output in (_render_display_value(figure),)
+ if output is not None
+ )
+ return outputs
+
+
+def _validate_oracle_output(
+ path: Path,
+ case_name: str,
+ index: int,
+ output: object,
+) -> dict[str, object]:
+ error = f"notebook smoke oracle {path} has invalid display-output entry {case_name}[{index}]"
+ if not isinstance(output, dict) or set(output) != ORACLE_OUTPUT_KEYS:
+ raise ValueError(error)
+ kind = output["kind"]
+ sha256 = output["sha256"]
+ size = output["size"]
+ if not isinstance(kind, str) or kind not in DISPLAY_OUTPUT_KINDS:
+ raise ValueError(error)
+ if not isinstance(sha256, str) or SHA256_HEX.fullmatch(sha256) is None:
+ raise ValueError(error)
+ if not isinstance(size, int) or isinstance(size, bool) or size < 0:
+ raise ValueError(error)
+ return {"kind": kind, "sha256": sha256, "size": size}
+
+
+def _load_oracle(path: Path) -> dict[str, list[dict[str, object]]]:
+ data = json.loads(path.read_text(encoding="utf-8"))
+ if not isinstance(data, dict):
+ raise ValueError(f"notebook smoke oracle {path} must be a mapping")
+ oracle: dict[str, list[dict[str, object]]] = {}
+ for name, outputs in data.items():
+ if not isinstance(name, str) or not isinstance(outputs, list):
+ raise ValueError(f"notebook smoke oracle {path} has invalid entry {name!r}")
+ oracle[name] = [
+ _validate_oracle_output(path, name, index, output)
+ for index, output in enumerate(outputs)
+ ]
+ return oracle
+
+
+def _assert_display_outputs_match(
+ case: NotebookCase,
+ actual_outputs: tuple[DisplayOutput, ...],
+ expected_outputs: list[dict[str, object]],
+) -> None:
+ actual = [output.as_dict() for output in actual_outputs]
+ if actual == expected_outputs:
+ return
+ raise AssertionError(
+ f"{case.name}: display output drift\n"
+ f"expected: {json.dumps(expected_outputs, sort_keys=True)}\n"
+ f"actual: {json.dumps(actual, sort_keys=True)}"
+ )
+
+
+def _execute_notebook(
+ case: NotebookCase,
+ *,
+ cell_timeout: int,
+ expected_outputs: list[dict[str, object]] | None = None,
+) -> NotebookResult:
+ notebook = json.loads(case.path.read_text(encoding="utf-8"))
+ _validate_kernelspec(case, notebook)
+ try:
+ display_path = case.path.relative_to(ROOT)
+ except ValueError:
+ display_path = case.path
+ namespace: dict[str, object] = {
+ "__name__": "__main__",
+ "__file__": str(case.path),
+ }
+ started = time.monotonic()
+ code_cells = [
+ (index, "".join(cell.get("source", [])))
+ for index, cell in enumerate(notebook.get("cells", []), start=1)
+ if cell.get("cell_type") == "code"
+ ]
+ display_outputs: list[DisplayOutput] = []
+ with _patched_environment(case.env):
+ old_cwd = Path.cwd()
+ os.chdir(ROOT)
+ try:
+ for index, source in code_cells:
+ if not source.strip():
+ continue
+ filename = f"{display_path}:cell-{index}"
+ try:
+ with _cell_timeout(cell_timeout):
+ result = _execute_cell(source, filename, namespace)
+ display_outputs.extend(result.outputs)
+ display_outputs.extend(
+ _flush_xy_pyplot_figures(skip_ids=result.displayed_ids)
+ )
+ except Exception as exc:
+ print(
+ f"FAIL {case.name}: {filename}: {exc.__class__.__name__}: {exc}",
+ file=sys.stderr,
+ )
+ traceback.print_exc()
+ raise
+ finally:
+ os.chdir(old_cwd)
+ _close_notebook_figures()
+ captured_outputs = tuple(display_outputs)
+ if expected_outputs is not None:
+ _assert_display_outputs_match(case, captured_outputs, expected_outputs)
+ elapsed = time.monotonic() - started
+ print(
+ f"PASS {case.name}: {len(code_cells)} code cells, "
+ f"{len(captured_outputs)} display outputs in {elapsed:.2f}s"
+ )
+ return NotebookResult(len(code_cells), len(captured_outputs), captured_outputs)
+
+
+def main(argv: list[str] | None = None) -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument(
+ "--profile",
+ choices=["pr"],
+ default="pr",
+ help="Notebook profile to execute.",
+ )
+ parser.add_argument("--cell-timeout", type=int, default=60)
+ parser.add_argument("--oracle", type=Path, default=DEFAULT_ORACLE)
+ parser.add_argument(
+ "--update-oracle",
+ action="store_true",
+ help="Rewrite the smoke profile display-output oracle from the current run.",
+ )
+ args = parser.parse_args(argv)
+
+ oracle = {} if args.update_oracle else _load_oracle(args.oracle)
+ updated_oracle: dict[str, list[dict[str, object]]] = {}
+ with tempfile.TemporaryDirectory(prefix="xy-notebook-smoke-") as tmp:
+ for case in smoke_cases(Path(tmp)):
+ expected_outputs = None
+ if not args.update_oracle:
+ expected_outputs = oracle.get(case.name)
+ if expected_outputs is None:
+ raise ValueError(f"notebook smoke oracle missing case {case.name!r}")
+ result = _execute_notebook(
+ case,
+ cell_timeout=args.cell_timeout,
+ expected_outputs=expected_outputs,
+ )
+ updated_oracle[case.name] = [output.as_dict() for output in result.outputs]
+ if args.update_oracle:
+ args.oracle.write_text(
+ json.dumps(updated_oracle, indent=2, sort_keys=True) + "\n",
+ encoding="utf-8",
+ )
+ print(f"Updated notebook smoke oracle: {args.oracle}")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/verify_ci_workflow.py b/scripts/verify_ci_workflow.py
index c3bf4b5e..b5dc0c2b 100644
--- a/scripts/verify_ci_workflow.py
+++ b/scripts/verify_ci_workflow.py
@@ -26,6 +26,7 @@
"matplotlib_reference",
"test",
"python_floor",
+ "notebook_smoke",
"benchmark_vs",
"benchmark_methodology",
"benchmark",
@@ -318,7 +319,7 @@ def _has_shell_init_environment(text: str) -> bool:
lines = _yaml_code_lines(text)
protected_jobs = _job_blocks(text)
protected_job_text = "\n".join(
- protected_jobs.get(job, "") for job in ("matplotlib_reference", "test")
+ protected_jobs.get(job, "") for job in ("matplotlib_reference", "test", "notebook_smoke")
)
if any(
re.search(r"\bGITHUB_(?:ENV|PATH)\b", line)
@@ -562,7 +563,12 @@ def _step_run_lines(step_block: str) -> list[str]:
def _require_step_runs_exactly(
- errors: list[str], job_text: str, step: str, description: str, *commands: str
+ errors: list[str],
+ job_text: str,
+ step: str,
+ description: str,
+ *commands: str,
+ allow_job_strategy: bool = False,
) -> None:
"""Require the exact active command list in a hard-gate named step."""
block = _named_step_blocks(job_text).get(step)
@@ -573,14 +579,9 @@ def _require_step_runs_exactly(
# an uninvoked function body, or one folded argument to another command.
forbidden_step_keys = ("if", "continue-on-error", "shell", "working-directory")
has_forbidden_step_key = any(_has_yaml_key(block, key, indent=8) for key in forbidden_step_keys)
- forbidden_job_keys = (
- "if",
- "continue-on-error",
- "defaults",
- "container",
- "needs",
- "strategy",
- )
+ forbidden_job_keys = ["if", "continue-on-error", "defaults", "container", "needs"]
+ if not allow_job_strategy:
+ forbidden_job_keys.append("strategy")
has_forbidden_job_key = any(
_has_yaml_key(job_text, key, indent=4) for key in forbidden_job_keys
)
@@ -611,6 +612,49 @@ def _require_job_contains(
errors.append(f"{workflow_label} {job} job missing {description}: {missing}")
+def _require_job_timeout_minutes(
+ errors: list[str],
+ jobs: dict[str, str],
+ job: str,
+ workflow_label: str,
+ minutes: int,
+) -> None:
+ block = jobs.get(job)
+ if block is None:
+ return
+ values, unsafe = _direct_yaml_key_values(block, "timeout-minutes", indent=4)
+ if unsafe or values != [str(minutes)]:
+ errors.append(f"{workflow_label} {job} job missing active timeout-minutes: {minutes}")
+
+
+def _require_simple_python_version_matrix(
+ errors: list[str],
+ jobs: dict[str, str],
+ job: str,
+ workflow_label: str,
+ expected: str,
+) -> None:
+ block = jobs.get(job)
+ if block is None:
+ return
+ strategy = _unique_mapping_block(block, "strategy", indent=4)
+ matrix = _unique_mapping_block(strategy or "", "matrix", indent=6)
+ if strategy is None or matrix is None:
+ errors.append(
+ f"{workflow_label} {job} job missing simple Python version matrix: {expected}"
+ )
+ return
+ values, unsafe = _direct_yaml_key_values(matrix, "python-version", indent=8)
+ has_filters = _has_yaml_key(matrix, "include", indent=8) or _has_yaml_key(
+ matrix, "exclude", indent=8
+ )
+ if unsafe or values != [expected] or has_filters:
+ errors.append(
+ f"{workflow_label} {job} job must run the exact Python version matrix "
+ f"{expected} without include/exclude filters"
+ )
+
+
def _step_block(job_text: str, step_needle: str) -> Optional[str]:
"""The indented block of the step whose `uses:`/`run:` line contains
`step_needle` — the step's own lines only, so a same-level sibling step
@@ -895,6 +939,40 @@ def validate_ci_workflow(path: Path = DEFAULT_CI_WORKFLOW) -> list[str]:
"scripts/check_python_floor.py",
"scripts/check_public_api.py",
)
+ _require_job_contains(
+ errors,
+ jobs,
+ "notebook_smoke",
+ "CI",
+ "deterministic public notebook smoke matrix",
+ "dtolnay/rust-toolchain@",
+ "actions/setup-python@",
+ 'node-version: "22"',
+ "npm ci",
+ "node js/build.mjs",
+ "cargo build --release",
+ "XY_SKIP_CARGO",
+ "--constraint benchmarks/requirements-ci.lock",
+ "matplotlib requests",
+ "MPLBACKEND: Agg",
+ "XY_NOTEBOOK_DISPLAY: html",
+ )
+ _require_job_timeout_minutes(errors, jobs, "notebook_smoke", "CI", 20)
+ _require_simple_python_version_matrix(
+ errors,
+ jobs,
+ "notebook_smoke",
+ "CI",
+ '["3.11", "3.13"]',
+ )
+ _require_step_runs_exactly(
+ errors,
+ jobs.get("notebook_smoke", ""),
+ "Execute deterministic public notebooks",
+ "notebook smoke runner command",
+ ".venv/bin/python scripts/run_notebook_smoke.py --profile pr",
+ allow_job_strategy=True,
+ )
_require_job_contains(
errors,
jobs,
diff --git a/tests/test_run_notebook_smoke.py b/tests/test_run_notebook_smoke.py
new file mode 100644
index 00000000..57095238
--- /dev/null
+++ b/tests/test_run_notebook_smoke.py
@@ -0,0 +1,296 @@
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+import scripts.run_notebook_smoke as notebook_smoke
+
+
+def _write_notebook(path: Path, sources: list[str]) -> None:
+ path.write_text(
+ json.dumps(
+ {
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": None,
+ "metadata": {},
+ "outputs": [],
+ "source": source.splitlines(keepends=True),
+ }
+ for source in sources
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3",
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5,
+ }
+ ),
+ encoding="utf-8",
+ )
+
+
+def test_execute_notebook_runs_cells_top_to_bottom(tmp_path: Path) -> None:
+ notebook = tmp_path / "ok.ipynb"
+ marker = tmp_path / "marker.txt"
+ _write_notebook(
+ notebook,
+ [
+ "value = 40\n",
+ f"from pathlib import Path\nPath({str(marker)!r}).write_text(str(value + 2));\n",
+ ],
+ )
+
+ case = notebook_smoke.NotebookCase("ok", notebook, {})
+
+ result = notebook_smoke._execute_notebook(case, cell_timeout=5)
+
+ assert marker.read_text(encoding="utf-8") == "42"
+ assert result == notebook_smoke.NotebookResult(code_cells=2, display_outputs=0)
+
+
+def test_execute_notebook_fails_on_cell_error(tmp_path: Path) -> None:
+ notebook = tmp_path / "broken.ipynb"
+ _write_notebook(notebook, ["answer = 42\n", "raise RuntimeError('boom')\n"])
+
+ case = notebook_smoke.NotebookCase("broken", notebook, {})
+
+ with pytest.raises(RuntimeError, match="boom"):
+ notebook_smoke._execute_notebook(case, cell_timeout=5)
+
+
+def test_execute_notebook_renders_final_expression(tmp_path: Path) -> None:
+ notebook = tmp_path / "display.ipynb"
+ marker = tmp_path / "display.txt"
+ _write_notebook(
+ notebook,
+ [
+ f"""
+class Displayed:
+ def _repr_html_(self):
+ from pathlib import Path
+ Path({str(marker)!r}).write_text("rendered")
+ return "ok"
+
+Displayed()
+""",
+ ],
+ )
+
+ case = notebook_smoke.NotebookCase("display", notebook, {})
+
+ result = notebook_smoke._execute_notebook(case, cell_timeout=5)
+
+ assert marker.read_text(encoding="utf-8") == "rendered"
+ assert result == notebook_smoke.NotebookResult(code_cells=1, display_outputs=1)
+
+
+def test_execute_notebook_rejects_display_output_drift(tmp_path: Path) -> None:
+ notebook = tmp_path / "display.ipynb"
+ _write_notebook(
+ notebook,
+ [
+ """
+class Displayed:
+ def _repr_html_(self):
+ return "changed"
+
+Displayed()
+""",
+ ],
+ )
+
+ case = notebook_smoke.NotebookCase("display", notebook, {})
+ expected = [notebook_smoke._display_output("_repr_html_", "original").as_dict()]
+
+ with pytest.raises(AssertionError, match="display output drift"):
+ notebook_smoke._execute_notebook(case, cell_timeout=5, expected_outputs=expected)
+
+
+@pytest.mark.parametrize(
+ "outputs",
+ [
+ [{"kind": "_repr_html_", "sha256": "0" * 64}],
+ [{"kind": "_repr_html_", "sha256": "not-a-hash", "size": 0}],
+ [{"kind": "_repr_html_", "sha256": "g" * 64, "size": 0}],
+ [{"kind": "_repr_html_", "sha256": "0" * 64, "size": -1}],
+ [{"kind": "_repr_html_", "sha256": "0" * 64, "size": True}],
+ [{"kind": "unsupported", "sha256": "0" * 64, "size": 0}],
+ ["not-a-mapping"],
+ ],
+)
+def test_load_oracle_rejects_invalid_display_outputs(tmp_path: Path, outputs: list[object]) -> None:
+ oracle = tmp_path / "oracle.json"
+ oracle.write_text(json.dumps({"case": outputs}), encoding="utf-8")
+
+ with pytest.raises(ValueError, match="invalid display-output entry"):
+ notebook_smoke._load_oracle(oracle)
+
+
+def test_execute_notebook_flushes_xy_pyplot_figures(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ notebook = tmp_path / "pyplot.ipynb"
+ marker = tmp_path / "flush.txt"
+ _write_notebook(notebook, ["value = 1\n"])
+
+ class Figure:
+ def _repr_html_(self) -> str:
+ marker.write_text("flushed", encoding="utf-8")
+ return "figure
"
+
+ fake_pyplot = SimpleNamespace(
+ all_figures=lambda: [Figure()],
+ close=lambda target: marker.write_text(
+ marker.read_text(encoding="utf-8") + f":{target}",
+ encoding="utf-8",
+ ),
+ )
+ monkeypatch.setitem(sys.modules, "xy.pyplot", fake_pyplot)
+ case = notebook_smoke.NotebookCase("pyplot", notebook, {})
+
+ result = notebook_smoke._execute_notebook(case, cell_timeout=5)
+
+ assert marker.read_text(encoding="utf-8") == "flushed:all"
+ assert result == notebook_smoke.NotebookResult(code_cells=1, display_outputs=1)
+
+
+def test_execute_notebook_deduplicates_final_expression_pyplot_figure(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ notebook = tmp_path / "dedupe.ipynb"
+ marker = tmp_path / "renders.txt"
+
+ class Figure:
+ def __init__(self) -> None:
+ self.count = 0
+
+ def _repr_html_(self) -> str:
+ self.count += 1
+ marker.write_text(str(self.count), encoding="utf-8")
+ return "figure
"
+
+ figure = Figure()
+ fake_pyplot = SimpleNamespace(
+ figure=figure,
+ all_figures=lambda: [figure],
+ close=lambda target: None,
+ )
+ monkeypatch.setitem(sys.modules, "xy.pyplot", fake_pyplot)
+ _write_notebook(
+ notebook,
+ [
+ """
+import sys
+fig = sys.modules["xy.pyplot"].figure
+fig
+""",
+ ],
+ )
+ case = notebook_smoke.NotebookCase("dedupe", notebook, {})
+
+ result = notebook_smoke._execute_notebook(case, cell_timeout=5)
+
+ assert marker.read_text(encoding="utf-8") == "1"
+ assert result == notebook_smoke.NotebookResult(code_cells=1, display_outputs=1)
+
+
+def test_execute_notebook_preserves_xy_pyplot_figures_between_cells(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ notebook = tmp_path / "cross_cell.ipynb"
+ marker = tmp_path / "renders.txt"
+
+ class Figure:
+ def __init__(self) -> None:
+ self.count = 0
+
+ def _repr_html_(self) -> str:
+ self.count += 1
+ marker.write_text(str(self.count), encoding="utf-8")
+ return "figure
"
+
+ figure = Figure()
+ closed: list[str] = []
+ fake_pyplot = SimpleNamespace(
+ all_figures=lambda: [figure],
+ close=lambda target: closed.append(target),
+ )
+ monkeypatch.setitem(sys.modules, "xy.pyplot", fake_pyplot)
+ _write_notebook(
+ notebook,
+ [
+ 'import sys\nfig = sys.modules["xy.pyplot"].all_figures()[0]\n',
+ "fig\n",
+ ],
+ )
+
+ case = notebook_smoke.NotebookCase("cross_cell", notebook, {})
+
+ result = notebook_smoke._execute_notebook(case, cell_timeout=5)
+
+ assert marker.read_text(encoding="utf-8") == "2"
+ assert closed == ["all"]
+ assert result == notebook_smoke.NotebookResult(code_cells=2, display_outputs=2)
+
+
+def test_execute_notebook_closes_figures_after_each_notebook(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ notebook = tmp_path / "mpl.ipynb"
+ closed: list[str] = []
+ fake_matplotlib = SimpleNamespace(close=lambda target: closed.append(target))
+ monkeypatch.setitem(sys.modules, "matplotlib.pyplot", fake_matplotlib)
+ _write_notebook(notebook, ["value = 1\n", "value += 1\n"])
+
+ case = notebook_smoke.NotebookCase("mpl", notebook, {})
+
+ result = notebook_smoke._execute_notebook(case, cell_timeout=5)
+
+ assert closed == ["all"]
+ assert result == notebook_smoke.NotebookResult(code_cells=2, display_outputs=0)
+
+
+def test_execute_notebook_rejects_incompatible_kernelspec(tmp_path: Path) -> None:
+ notebook = tmp_path / "ruby.ipynb"
+ _write_notebook(notebook, ["value = 1\n"])
+ payload = json.loads(notebook.read_text(encoding="utf-8"))
+ payload["metadata"]["kernelspec"] = {
+ "display_name": "Ruby",
+ "language": "ruby",
+ "name": "ruby",
+ }
+ notebook.write_text(json.dumps(payload), encoding="utf-8")
+
+ case = notebook_smoke.NotebookCase("ruby", notebook, {})
+
+ with pytest.raises(ValueError, match="unsupported kernelspec"):
+ notebook_smoke._execute_notebook(case, cell_timeout=5)
+
+
+def test_execute_notebook_rejects_nonpositive_timeout(tmp_path: Path) -> None:
+ notebook = tmp_path / "timeout.ipynb"
+ _write_notebook(notebook, ["value = 1\n"])
+
+ case = notebook_smoke.NotebookCase("timeout", notebook, {})
+
+ with pytest.raises(ValueError, match="timeout must be positive"):
+ notebook_smoke._execute_notebook(case, cell_timeout=0)
+
+
+def test_smoke_profile_preseeds_reduced_gaia_fixture(tmp_path: Path) -> None:
+ cases = notebook_smoke.smoke_cases(tmp_path)
+
+ gaia = next(case for case in cases if case.name == "real-world-gaia-reduced")
+ csv_path = Path(gaia.env["XY_REAL_WORLD_DATA"]) / f"gaia-dr3-hr-{gaia.env['GAIA_ROWS']}.csv"
+
+ assert csv_path.exists()
+ assert csv_path.read_text(encoding="utf-8").splitlines()[0] == "bp_rp,phot_g_mean_mag,parallax"
diff --git a/tests/test_verify_ci_workflow.py b/tests/test_verify_ci_workflow.py
index b2dd88d9..754ae2ec 100644
--- a/tests/test_verify_ci_workflow.py
+++ b/tests/test_verify_ci_workflow.py
@@ -38,6 +38,112 @@ def test_ci_workflow_accepts_current_gates() -> None:
assert verify_ci_workflow.validate_ci_workflow() == []
+def test_ci_workflow_requires_notebook_smoke_job(tmp_path: Path) -> None:
+ workflow = Path(".github/workflows/ci.yml").read_text(encoding="utf-8")
+ start = workflow.index(" notebook_smoke:\n")
+ end = workflow.index("\n benchmark_vs:\n", start)
+ path = tmp_path / "ci.yml"
+ path.write_text(workflow[:start] + workflow[end + 1 :], encoding="utf-8")
+
+ errors = verify_ci_workflow.validate_ci_workflow(path)
+
+ assert any("notebook_smoke" in error for error in errors)
+
+
+def test_ci_workflow_requires_notebook_smoke_runner(tmp_path: Path) -> None:
+ workflow = Path(".github/workflows/ci.yml").read_text(encoding="utf-8")
+ path = tmp_path / "ci.yml"
+ path.write_text(
+ workflow.replace("scripts/run_notebook_smoke.py --profile pr", "true"),
+ encoding="utf-8",
+ )
+
+ errors = verify_ci_workflow.validate_ci_workflow(path)
+
+ assert any("notebook_smoke" in error and "run_notebook_smoke" in error for error in errors)
+
+
+def test_ci_workflow_requires_notebook_smoke_runner_as_active_command(
+ tmp_path: Path,
+) -> None:
+ workflow = Path(".github/workflows/ci.yml").read_text(encoding="utf-8")
+ path = tmp_path / "ci.yml"
+ path.write_text(
+ workflow.replace(
+ " run: .venv/bin/python scripts/run_notebook_smoke.py --profile pr",
+ " run: true\n # .venv/bin/python scripts/run_notebook_smoke.py --profile pr",
+ ),
+ encoding="utf-8",
+ )
+
+ errors = verify_ci_workflow.validate_ci_workflow(path)
+
+ assert any("notebook smoke runner command" in error for error in errors)
+
+
+def test_ci_workflow_requires_notebook_smoke_timeout(tmp_path: Path) -> None:
+ workflow = Path(".github/workflows/ci.yml").read_text(encoding="utf-8")
+ path = tmp_path / "ci.yml"
+ path.write_text(
+ workflow.replace(" timeout-minutes: 20\n", "", 1),
+ encoding="utf-8",
+ )
+
+ errors = verify_ci_workflow.validate_ci_workflow(path)
+
+ assert any("notebook_smoke" in error and "timeout-minutes" in error for error in errors)
+
+
+def test_ci_workflow_rejects_commented_notebook_smoke_timeout(tmp_path: Path) -> None:
+ workflow = Path(".github/workflows/ci.yml").read_text(encoding="utf-8")
+ path = tmp_path / "ci.yml"
+ path.write_text(
+ workflow.replace(" timeout-minutes: 20\n", " # timeout-minutes: 20\n", 1),
+ encoding="utf-8",
+ )
+
+ errors = verify_ci_workflow.validate_ci_workflow(path)
+
+ assert any("notebook_smoke" in error and "timeout-minutes" in error for error in errors)
+
+
+def test_ci_workflow_rejects_filtered_notebook_smoke_matrix(tmp_path: Path) -> None:
+ workflow = Path(".github/workflows/ci.yml").read_text(encoding="utf-8")
+ path = tmp_path / "ci.yml"
+ path.write_text(
+ workflow.replace(
+ ' python-version: ["3.11", "3.13"]\n',
+ ' python-version: ["3.11", "3.13"]\n'
+ " exclude:\n"
+ ' - python-version: "3.11"\n'
+ ' - python-version: "3.13"\n',
+ 1,
+ ),
+ encoding="utf-8",
+ )
+
+ errors = verify_ci_workflow.validate_ci_workflow(path)
+
+ assert any("notebook_smoke" in error and "include/exclude" in error for error in errors)
+
+
+def test_ci_workflow_rejects_commented_notebook_smoke_matrix(tmp_path: Path) -> None:
+ workflow = Path(".github/workflows/ci.yml").read_text(encoding="utf-8")
+ path = tmp_path / "ci.yml"
+ path.write_text(
+ workflow.replace(
+ ' python-version: ["3.11", "3.13"]\n',
+ ' # python-version: ["3.11", "3.13"]\n',
+ 1,
+ ),
+ encoding="utf-8",
+ )
+
+ errors = verify_ci_workflow.validate_ci_workflow(path)
+
+ assert any("notebook_smoke" in error and "Python version matrix" in error for error in errors)
+
+
def test_ci_workflow_requires_locked_reflex_environment(tmp_path: Path) -> None:
workflow = Path(".github/workflows/ci.yml").read_text(encoding="utf-8")
path = tmp_path / "ci.yml"