Validate the full standard examples corpus in CI

.github/workflows/standard_examples.yml

@@ -22,10 +22,18 @@ jobs:
         python-version: ["3.11", "3.12", "3.13", "3.14"]
     steps:
       - uses: actions/checkout@v7
+      # The standard is validated at a pinned commit (bumped by the
+      # weekly_update workflow), so changes to it cannot break unrelated PRs.
+      - name: Read the pinned PALS standard commit
+        id: pin
+        run: |
+          sha=$(grep -vE '^[[:space:]]*(#|$)' tests/standard_examples_git_reference.txt | head -n1)
+          echo "sha=${sha}" >> "$GITHUB_OUTPUT"
       - name: Checkout PALS repository
         uses: actions/checkout@v7
         with:
           repository: pals-project/pals
+          ref: ${{ steps.pin.outputs.sha }}
           path: pals_temp
           fetch-depth: 1
           sparse-checkout: |
@@ -40,6 +48,6 @@ jobs:
           pip install ".[test]"
       - name: Run standard examples
         run: |
-          for file in pals_temp/examples/*.pals.yaml; do
-            python tests/validate_standard_examples.py --path "${file}"
-          done
+          python tests/validate_standard_examples.py \
+            --root pals_temp/examples \
+            --known-failures tests/standard_examples_known_failures.txt

.github/workflows/weekly_update.yml

@@ -0,0 +1,72 @@
+name: Weekly update
+
+# Bumps the pinned pals-project/pals commit
+# (tests/standard_examples_git_reference.txt) that
+# the standard_examples workflow validates against, as a PR. A change to the
+# standard that breaks the examples validation therefore shows up in that PR,
+# not in unrelated ones.
+
+on:
+  schedule:
+    - cron: "0 9 * * 1"  # every Monday at 1:00am PST (9:00am UTC)
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  autoupdate:
+    name: Autoupdate
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7
+      - name: Configure Git
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+      - name: Checkout branch
+        run: |
+          git checkout weekly_update || git checkout -b weekly_update
+      - name: Update the pinned PALS standard commit
+        id: bump
+        run: |
+          old=$(grep -vE '^[[:space:]]*(#|$)' tests/standard_examples_git_reference.txt | head -n1)
+          new=$(git ls-remote https://github.com/pals-project/pals.git refs/heads/main | cut -f1)
+          sed -i "s/^${old}\$/${new}/" tests/standard_examples_git_reference.txt
+          echo "old=${old}" >> "$GITHUB_OUTPUT"
+          echo "new=${new}" >> "$GITHUB_OUTPUT"
+      - name: Commit changes
+        id: commit
+        run: |
+          if git diff --quiet; then
+            echo "The pin already matches pals main; nothing to update."
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+            git add -u
+            git commit -m "Update the pinned PALS standard commit"
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+      - name: Push changes
+        if: steps.commit.outputs.changed == 'true'
+        run: |
+          git push -f origin weekly_update
+      - name: Open pull request
+        if: steps.commit.outputs.changed == 'true'
+        run: |
+          if [ -n "$(gh pr list --head weekly_update --state open --json number --jq '.[].number')" ]; then
+            echo "An open weekly_update PR already exists; the push updated it."
+            exit 0
+          fi
+          gh pr create \
+            --base main \
+            --title "PALS standard: weekly update of the pinned commit" \
+            --body "Bump the pinned pals-project/pals commit from \
+          pals-project/pals@${{ steps.bump.outputs.old }} to \
+          pals-project/pals@${{ steps.bump.outputs.new }} \
+          (https://github.com/pals-project/pals/compare/${{ steps.bump.outputs.old }}...${{ steps.bump.outputs.new }}).
+
+          Automated via .github/workflows/weekly_update.yml."

tests/standard_examples_git_reference.txt

@@ -0,0 +1,10 @@
+# The commit of pals-project/pals that the standard_examples workflow checks
+# out and validates the examples corpus from. Pinning a fixed commit keeps
+# changes to the standard from breaking unrelated pals-python PRs; the
+# weekly_update workflow bumps this pin to the current main in a PR of its
+# own, so a breaking change in the standard surfaces there, reviewed and
+# controlled.
+#
+# The first non-comment line is the pin: a full commit SHA of
+# https://github.com/pals-project/pals
+5c39fde043ddad7a961ede01a5ad73d90b098aec

tests/standard_examples_known_failures.txt

@@ -0,0 +1,42 @@
+# Standard example files pals-python cannot read yet, one path per line
+# relative to the standard's examples directory. validate_standard_examples.py
+# treats a failure of a listed file as expected and a listed file that starts
+# to load as an error, so this list shrinks as support lands. Blank lines and
+# `#` comments are ignored.
+
+# PALSroot models only `version` and `facility`, and `facility` is required:
+# additional attributes (notes, extension labels, or a version
+# only) are rejected.
+unit_tests/loading/basic/joiner.pals.yaml
+unit_tests/loading/diamond/joiner.pals.yaml
+unit_tests/loading/diamond/left.pals.yaml
+unit_tests/loading/diamond/right.pals.yaml
+unit_tests/loading/diamond/shared.pals.yaml
+unit_tests/loading/extension_labels/a.pals.yaml
+unit_tests/loading/extension_labels/joiner.pals.yaml
+unit_tests/loading/implicit_self/a.pals.yaml
+unit_tests/loading/implicit_self/joiner.pals.yaml
+unit_tests/loading/nested/a.pals.yaml
+unit_tests/loading/nested/inner.pals.yaml
+unit_tests/loading/nested/joiner.pals.yaml
+unit_tests/loading/relative_paths/joiner.pals.yaml
+unit_tests/loading/relative_paths/sub/inner.pals.yaml
+unit_tests/loading/relative_paths/top.pals.yaml
+unit_tests/loading/self_position/a.pals.yaml
+unit_tests/loading/self_position/b.pals.yaml
+unit_tests/loading/self_position/joiner.pals.yaml
+unit_tests/loading/version_match/a.pals.yaml
+unit_tests/loading/version_match/joiner.pals.yaml
+
+# An integer `version` (root_keys writes `version: 1`) is rejected; the model
+# wants a string or null.
+unit_tests/document/root_keys.pals.yaml
+
+# `include` entries inside a facility are not resolved.
+machine/machine.pals.yaml
+unit_tests/loading/include/sub/layout.pals.yaml
+
+# The sequence form of `variables` and the compact `sets` form are not
+# modeled.
+unit_tests/expressions/inline_expressions.pals.yaml
+unit_tests/sets/sets_compact.pals.yaml

tests/validate_standard_examples.py

@@ -1,37 +1,39 @@
-"""Utility script to validate standard PALS example files.
+"""Validate the standard PALS example files from pals-project/pals/examples.
 
-This script is not run by pytest and is intended to be used as a standalone script.
-Run it from the repository root like:
+Every *.pals.yaml file under --root is discovered and must be readable with
+pals.load(). *.subpals.yaml files are exempt: per the standard's notation
+section they are sub-level include fragments, spliced into (and read through)
+the file that includes them.
 
-    python tests/validate_standard_examples.py --path /path/to/example.pals.yaml
+Files this implementation cannot read yet are recorded in the known-failures
+list (tests/standard_examples_known_failures.txt): a failure of a listed file
+is expected, and a listed file that starts to load is reported so the list
+shrinks as support lands.
 
-Before running, download the desired standard PALS example files from pals-project/pals/examples.
+This script is not run by pytest; the standard_examples workflow runs it
+against a checkout of the standard, pinned to the commit named in
+tests/standard_examples_git_reference.txt. Run it locally, from the repository root:
+
+    python tests/validate_standard_examples.py \
+        --root /path/to/pals/examples \
+        --known-failures tests/standard_examples_known_failures.txt
 """
 
 import argparse
+import pathlib
+import sys
 
-from pals import load
-from pals.kinds import PlaceholderName
-from pals.kinds.BeamLine import BeamLine
-from pals.kinds.Drift import Drift
-from pals.kinds.Lattice import Lattice
-from pals.kinds.Quadrupole import Quadrupole
+import pals
 
 
-def main():
-    # Parse command-line arguments
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--path",
-        required=True,
-        help="Path to the example file",
-    )
-    args = parser.parse_args()
-    example_file = args.path
-    # Parse and validate YAML data from file
-    lattice = load(example_file)
-    # The following assertions are based on the standard PALS example file
-    # fodo.pals.yaml from pals-project/pals/examples
+def check_fodo(lattice):
+    """Structural spot-checks of the introductory example fodo.pals.yaml."""
+    from pals.kinds import PlaceholderName
+    from pals.kinds.BeamLine import BeamLine
+    from pals.kinds.Drift import Drift
+    from pals.kinds.Lattice import Lattice
+    from pals.kinds.Quadrupole import Quadrupole
+
     assert isinstance(lattice.facility[0], Drift)
     assert lattice.facility[0].name == "drift1"
     assert isinstance(lattice.facility[1], Quadrupole)
@@ -45,5 +47,73 @@ def main():
     assert isinstance(lattice.facility[5], PlaceholderName)
 
 
+def read_known_failures(path):
+    """The known-failures list: one root-relative path per line, # comments."""
+    known = set()
+    for line in pathlib.Path(path).read_text().splitlines():
+        line = line.strip()
+        if line and not line.startswith("#"):
+            known.add(line)
+    return known
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--root",
+        required=True,
+        help="Path to the standard's examples directory",
+    )
+    parser.add_argument(
+        "--known-failures",
+        required=True,
+        help="Path to the known-failures list",
+    )
+    args = parser.parse_args()
+
+    root = pathlib.Path(args.root)
+    known = read_known_failures(args.known_failures)
+
+    # Track every unexpected result for the final exit status.
+    failures = 0
+    seen = set()
+    files = sorted(root.rglob("*.pals.yaml"))
+    for path in files:
+        rel = path.relative_to(root).as_posix()
+        seen.add(rel)
+        error = None
+        # Capture load errors so expected failures can be distinguished.
+        try:
+            lattice = pals.load(str(path))
+            if rel == "fodo.pals.yaml":
+                check_fodo(lattice)
+        except Exception as e:  # noqa: BLE001 -- any reader failure counts
+            error = e
+        if rel in known:
+            if error is None:
+                print(f"XPASS  {rel}")
+                print("       loads now: remove it from the known-failures list")
+                failures += 1
+            else:
+                print(f"XFAIL  {rel}")
+        elif error is None:
+            print(f"PASS   {rel}")
+        else:
+            print(f"FAIL   {rel}")
+            message = f"{type(error).__name__}: {error}"
+            for detail in message.splitlines()[:3]:
+                print(f"       {detail}")
+            failures += 1
+
+    # Flag stale entries in the known-failures list.
+    for rel in sorted(known - seen):
+        print(f"FAIL   {rel}")
+        print("       in the known-failures list, but not found under --root")
+        failures += 1
+
+    print(f"{len(files)} files checked, {failures} unexpected results")
+    return 1 if failures else 0
+
+
 if __name__ == "__main__":
-    main()
+    sys.exit(main())