From ac8df204134188459845a50e84a9bbf90ede6294 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 30 Jul 2026 12:55:24 -0700 Subject: [PATCH 1/4] Validate the full standard examples corpus in CI pals-project/pals#285 grew the standard's examples/ into a corpus of example machines and unit-test lattices. Validate all of it: every *.pals.yaml is discovered automatically and must be readable with pals.load() (*.subpals.yaml include fragments are exempt), keeping the structural spot-checks for fodo.pals.yaml. The files this implementation cannot read yet are recorded in tests/standard_examples_known_failures.txt, grouped by cause, to be resolved in follow-up PRs; a listed file that starts to load is reported so the list only shrinks. The standard is checked out at a pinned commit (tests/pals_standard_ref.txt) so changes to it cannot break unrelated PRs; a weekly_update workflow, modeled on WarpX's, bumps the pin to the current pals main in a controlled PR of its own. Co-Authored-By: Claude Fable 5 --- .github/workflows/standard_examples.yml | 14 ++- .github/workflows/weekly_update.yml | 71 ++++++++++++ tests/pals_standard_ref.txt | 10 ++ tests/standard_examples_known_failures.txt | 42 +++++++ tests/validate_standard_examples.py | 121 ++++++++++++++++----- 5 files changed, 229 insertions(+), 29 deletions(-) create mode 100644 .github/workflows/weekly_update.yml create mode 100644 tests/pals_standard_ref.txt create mode 100644 tests/standard_examples_known_failures.txt diff --git a/.github/workflows/standard_examples.yml b/.github/workflows/standard_examples.yml index 40b9f90..ae224c1 100644 --- a/.github/workflows/standard_examples.yml +++ b/.github/workflows/standard_examples.yml @@ -22,10 +22,18 @@ jobs: python-version: ["3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v7 + # The standard is validated at a pinned commit (bumped by the + # weekly_update workflow), so changes to it cannot break unrelated PRs. + - name: Read the pinned PALS standard commit + id: pin + run: | + sha=$(grep -vE '^[[:space:]]*(#|$)' tests/pals_standard_ref.txt | head -n1) + echo "sha=${sha}" >> "$GITHUB_OUTPUT" - name: Checkout PALS repository uses: actions/checkout@v7 with: repository: pals-project/pals + ref: ${{ steps.pin.outputs.sha }} path: pals_temp fetch-depth: 1 sparse-checkout: | @@ -40,6 +48,6 @@ jobs: pip install ".[test]" - name: Run standard examples run: | - for file in pals_temp/examples/*.pals.yaml; do - python tests/validate_standard_examples.py --path "${file}" - done + python tests/validate_standard_examples.py \ + --root pals_temp/examples \ + --known-failures tests/standard_examples_known_failures.txt diff --git a/.github/workflows/weekly_update.yml b/.github/workflows/weekly_update.yml new file mode 100644 index 0000000..57f0227 --- /dev/null +++ b/.github/workflows/weekly_update.yml @@ -0,0 +1,71 @@ +name: Weekly update + +# Bumps the pinned pals-project/pals commit (tests/pals_standard_ref.txt) that +# the standard_examples workflow validates against, as a PR. A change to the +# standard that breaks the examples validation therefore shows up in that PR, +# not in unrelated ones. + +on: + schedule: + - cron: "0 9 * * 1" # every Monday at 1:00am PST (9:00am UTC) + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + autoupdate: + name: Autoupdate + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout repository + uses: actions/checkout@v7 + - name: Configure Git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + - name: Checkout branch + run: | + git checkout weekly_update || git checkout -b weekly_update + - name: Update the pinned PALS standard commit + id: bump + run: | + old=$(grep -vE '^[[:space:]]*(#|$)' tests/pals_standard_ref.txt | head -n1) + new=$(git ls-remote https://github.com/pals-project/pals.git refs/heads/main | cut -f1) + sed -i "s/^${old}\$/${new}/" tests/pals_standard_ref.txt + echo "old=${old}" >> "$GITHUB_OUTPUT" + echo "new=${new}" >> "$GITHUB_OUTPUT" + - name: Commit changes + id: commit + run: | + if git diff --quiet; then + echo "The pin already matches pals main; nothing to update." + echo "changed=false" >> "$GITHUB_OUTPUT" + else + git add -u + git commit -m "Update the pinned PALS standard commit" + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + - name: Push changes + if: steps.commit.outputs.changed == 'true' + run: | + git push -f origin weekly_update + - name: Open pull request + if: steps.commit.outputs.changed == 'true' + run: | + if [ -n "$(gh pr list --head weekly_update --state open --json number --jq '.[].number')" ]; then + echo "An open weekly_update PR already exists; the push updated it." + exit 0 + fi + gh pr create \ + --base main \ + --title "PALS standard: weekly update of the pinned commit" \ + --body "Bump the pinned pals-project/pals commit from \ + pals-project/pals@${{ steps.bump.outputs.old }} to \ + pals-project/pals@${{ steps.bump.outputs.new }} \ + (https://github.com/pals-project/pals/compare/${{ steps.bump.outputs.old }}...${{ steps.bump.outputs.new }}). + + Automated via .github/workflows/weekly_update.yml." diff --git a/tests/pals_standard_ref.txt b/tests/pals_standard_ref.txt new file mode 100644 index 0000000..2f92b1d --- /dev/null +++ b/tests/pals_standard_ref.txt @@ -0,0 +1,10 @@ +# The commit of pals-project/pals that the standard_examples workflow checks +# out and validates the examples corpus from. Pinning a fixed commit keeps +# changes to the standard from breaking unrelated pals-python PRs; the +# weekly_update workflow bumps this pin to the current main in a PR of its +# own, so a breaking change in the standard surfaces there, reviewed and +# controlled. +# +# The first non-comment line is the pin: a full commit SHA of +# https://github.com/pals-project/pals +5c39fde043ddad7a961ede01a5ad73d90b098aec diff --git a/tests/standard_examples_known_failures.txt b/tests/standard_examples_known_failures.txt new file mode 100644 index 0000000..8b5e082 --- /dev/null +++ b/tests/standard_examples_known_failures.txt @@ -0,0 +1,42 @@ +# Standard example files pals-python cannot read yet, one path per line +# relative to the standard's examples directory. validate_standard_examples.py +# treats a failure of a listed file as expected and a listed file that starts +# to load as an error, so this list shrinks as support lands. Blank lines and +# `#` comments are ignored. + +# PALSroot models only `version` and `facility`, and `facility` is required: +# the minimal load-family documents (notes, extension labels, or a version +# only) are rejected. +unit_tests/loading/basic/joiner.pals.yaml +unit_tests/loading/diamond/joiner.pals.yaml +unit_tests/loading/diamond/left.pals.yaml +unit_tests/loading/diamond/right.pals.yaml +unit_tests/loading/diamond/shared.pals.yaml +unit_tests/loading/extension_labels/a.pals.yaml +unit_tests/loading/extension_labels/joiner.pals.yaml +unit_tests/loading/implicit_self/a.pals.yaml +unit_tests/loading/implicit_self/joiner.pals.yaml +unit_tests/loading/nested/a.pals.yaml +unit_tests/loading/nested/inner.pals.yaml +unit_tests/loading/nested/joiner.pals.yaml +unit_tests/loading/relative_paths/joiner.pals.yaml +unit_tests/loading/relative_paths/sub/inner.pals.yaml +unit_tests/loading/relative_paths/top.pals.yaml +unit_tests/loading/self_position/a.pals.yaml +unit_tests/loading/self_position/b.pals.yaml +unit_tests/loading/self_position/joiner.pals.yaml +unit_tests/loading/version_match/a.pals.yaml +unit_tests/loading/version_match/joiner.pals.yaml + +# An integer `version` (root_keys writes `version: 1`) is rejected; the model +# wants a string or null. +unit_tests/document/root_keys.pals.yaml + +# `include` entries inside a facility are not resolved. +machine/machine.pals.yaml +unit_tests/loading/include/sub/layout.pals.yaml + +# The sequence form of `variables` and the compact `sets` form are not +# modeled. +unit_tests/expressions/inline_expressions.pals.yaml +unit_tests/sets/sets_compact.pals.yaml diff --git a/tests/validate_standard_examples.py b/tests/validate_standard_examples.py index 6cd46cd..d4a1953 100644 --- a/tests/validate_standard_examples.py +++ b/tests/validate_standard_examples.py @@ -1,37 +1,39 @@ -"""Utility script to validate standard PALS example files. +"""Validate the standard PALS example files from pals-project/pals/examples. -This script is not run by pytest and is intended to be used as a standalone script. -Run it from the repository root like: +Every *.pals.yaml file under --root is discovered and must be readable with +pals.load(). *.subpals.yaml files are exempt: per the standard's notation +section they are sub-level include fragments, spliced into (and read through) +the file that includes them. - python tests/validate_standard_examples.py --path /path/to/example.pals.yaml +Files this implementation cannot read yet are recorded in the known-failures +list (tests/standard_examples_known_failures.txt): a failure of a listed file +is expected, and a listed file that starts to load is reported so the list +shrinks as support lands. -Before running, download the desired standard PALS example files from pals-project/pals/examples. +This script is not run by pytest; the standard_examples workflow runs it +against a checkout of the standard, pinned to the commit named in +tests/pals_standard_ref.txt. Run it locally, from the repository root: + + python tests/validate_standard_examples.py \ + --root /path/to/pals/examples \ + --known-failures tests/standard_examples_known_failures.txt """ import argparse +import pathlib +import sys -from pals import load -from pals.kinds import PlaceholderName -from pals.kinds.BeamLine import BeamLine -from pals.kinds.Drift import Drift -from pals.kinds.Lattice import Lattice -from pals.kinds.Quadrupole import Quadrupole +import pals -def main(): - # Parse command-line arguments - parser = argparse.ArgumentParser() - parser.add_argument( - "--path", - required=True, - help="Path to the example file", - ) - args = parser.parse_args() - example_file = args.path - # Parse and validate YAML data from file - lattice = load(example_file) - # The following assertions are based on the standard PALS example file - # fodo.pals.yaml from pals-project/pals/examples +def check_fodo(lattice): + """Structural spot-checks of the introductory example fodo.pals.yaml.""" + from pals.kinds import PlaceholderName + from pals.kinds.BeamLine import BeamLine + from pals.kinds.Drift import Drift + from pals.kinds.Lattice import Lattice + from pals.kinds.Quadrupole import Quadrupole + assert isinstance(lattice.facility[0], Drift) assert lattice.facility[0].name == "drift1" assert isinstance(lattice.facility[1], Quadrupole) @@ -45,5 +47,72 @@ def main(): assert isinstance(lattice.facility[5], PlaceholderName) +def read_known_failures(path): + """The known-failures list: one root-relative path per line, # comments.""" + known = set() + for line in pathlib.Path(path).read_text().splitlines(): + line = line.strip() + if line and not line.startswith("#"): + known.add(line) + return known + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--root", + required=True, + help="Path to the standard's examples directory", + ) + parser.add_argument( + "--known-failures", + required=True, + help="Path to the known-failures list", + ) + args = parser.parse_args() + + root = pathlib.Path(args.root) + known = read_known_failures(args.known_failures) + + failures = 0 + seen = set() + files = sorted(root.rglob("*.pals.yaml")) + for path in files: + rel = path.relative_to(root).as_posix() + seen.add(rel) + error = None + try: + lattice = pals.load(str(path)) + if rel == "fodo.pals.yaml": + check_fodo(lattice) + except Exception as e: # noqa: BLE001 -- any reader failure counts + error = e + if rel in known: + if error is None: + print(f"XPASS {rel}") + print(" loads now: remove it from the known-failures list") + failures += 1 + else: + print(f"XFAIL {rel}") + elif error is None: + print(f"PASS {rel}") + else: + print(f"FAIL {rel}") + message = f"{type(error).__name__}: {error}" + for detail in message.splitlines()[:3]: + print(f" {detail}") + failures += 1 + + # A listed file that no longer exists means the corpus moved on and the + # list still describes it. + for rel in sorted(known - seen): + print(f"FAIL {rel}") + print(" in the known-failures list, but not found under --root") + failures += 1 + + print(f"{len(files)} files checked, {failures} unexpected results") + return 1 if failures else 0 + + if __name__ == "__main__": - main() + sys.exit(main()) From ddbb83d1d47dae3b688b00f2df430b381795965a Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Thu, 30 Jul 2026 14:13:07 -0700 Subject: [PATCH 2/4] Improve main's inline comments in validate_standard_examples.py --- tests/validate_standard_examples.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/validate_standard_examples.py b/tests/validate_standard_examples.py index d4a1953..5b04776 100644 --- a/tests/validate_standard_examples.py +++ b/tests/validate_standard_examples.py @@ -74,6 +74,7 @@ def main(): root = pathlib.Path(args.root) known = read_known_failures(args.known_failures) + # Track every unexpected result for the final exit status. failures = 0 seen = set() files = sorted(root.rglob("*.pals.yaml")) @@ -81,6 +82,7 @@ def main(): rel = path.relative_to(root).as_posix() seen.add(rel) error = None + # Capture load errors so expected failures can be distinguished. try: lattice = pals.load(str(path)) if rel == "fodo.pals.yaml": @@ -103,8 +105,7 @@ def main(): print(f" {detail}") failures += 1 - # A listed file that no longer exists means the corpus moved on and the - # list still describes it. + # Flag stale entries in the known-failures list. for rel in sorted(known - seen): print(f"FAIL {rel}") print(" in the known-failures list, but not found under --root") From e3e09634c19c27e6f8e76d027e9f0f84b93fd7c0 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 30 Jul 2026 14:40:29 -0700 Subject: [PATCH 3/4] simplify why some files are rejected --- tests/standard_examples_known_failures.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/standard_examples_known_failures.txt b/tests/standard_examples_known_failures.txt index 8b5e082..4365661 100644 --- a/tests/standard_examples_known_failures.txt +++ b/tests/standard_examples_known_failures.txt @@ -5,7 +5,7 @@ # `#` comments are ignored. # PALSroot models only `version` and `facility`, and `facility` is required: -# the minimal load-family documents (notes, extension labels, or a version +# additional attributes (notes, extension labels, or a version # only) are rejected. unit_tests/loading/basic/joiner.pals.yaml unit_tests/loading/diamond/joiner.pals.yaml From 6d28297712a000377c3cb95dc2029b3f1cd4de54 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 30 Jul 2026 14:53:51 -0700 Subject: [PATCH 4/4] Rename git ref file --- .github/workflows/standard_examples.yml | 2 +- .github/workflows/weekly_update.yml | 7 ++++--- ...tandard_ref.txt => standard_examples_git_reference.txt} | 0 tests/validate_standard_examples.py | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) rename tests/{pals_standard_ref.txt => standard_examples_git_reference.txt} (100%) diff --git a/.github/workflows/standard_examples.yml b/.github/workflows/standard_examples.yml index ae224c1..b40be12 100644 --- a/.github/workflows/standard_examples.yml +++ b/.github/workflows/standard_examples.yml @@ -27,7 +27,7 @@ jobs: - name: Read the pinned PALS standard commit id: pin run: | - sha=$(grep -vE '^[[:space:]]*(#|$)' tests/pals_standard_ref.txt | head -n1) + sha=$(grep -vE '^[[:space:]]*(#|$)' tests/standard_examples_git_reference.txt | head -n1) echo "sha=${sha}" >> "$GITHUB_OUTPUT" - name: Checkout PALS repository uses: actions/checkout@v7 diff --git a/.github/workflows/weekly_update.yml b/.github/workflows/weekly_update.yml index 57f0227..b33f593 100644 --- a/.github/workflows/weekly_update.yml +++ b/.github/workflows/weekly_update.yml @@ -1,6 +1,7 @@ name: Weekly update -# Bumps the pinned pals-project/pals commit (tests/pals_standard_ref.txt) that +# Bumps the pinned pals-project/pals commit +# (tests/standard_examples_git_reference.txt) that # the standard_examples workflow validates against, as a PR. A change to the # standard that breaks the examples validation therefore shows up in that PR, # not in unrelated ones. @@ -33,9 +34,9 @@ jobs: - name: Update the pinned PALS standard commit id: bump run: | - old=$(grep -vE '^[[:space:]]*(#|$)' tests/pals_standard_ref.txt | head -n1) + old=$(grep -vE '^[[:space:]]*(#|$)' tests/standard_examples_git_reference.txt | head -n1) new=$(git ls-remote https://github.com/pals-project/pals.git refs/heads/main | cut -f1) - sed -i "s/^${old}\$/${new}/" tests/pals_standard_ref.txt + sed -i "s/^${old}\$/${new}/" tests/standard_examples_git_reference.txt echo "old=${old}" >> "$GITHUB_OUTPUT" echo "new=${new}" >> "$GITHUB_OUTPUT" - name: Commit changes diff --git a/tests/pals_standard_ref.txt b/tests/standard_examples_git_reference.txt similarity index 100% rename from tests/pals_standard_ref.txt rename to tests/standard_examples_git_reference.txt diff --git a/tests/validate_standard_examples.py b/tests/validate_standard_examples.py index 5b04776..1ff0543 100644 --- a/tests/validate_standard_examples.py +++ b/tests/validate_standard_examples.py @@ -12,7 +12,7 @@ This script is not run by pytest; the standard_examples workflow runs it against a checkout of the standard, pinned to the commit named in -tests/pals_standard_ref.txt. Run it locally, from the repository root: +tests/standard_examples_git_reference.txt. Run it locally, from the repository root: python tests/validate_standard_examples.py \ --root /path/to/pals/examples \