diff --git a/.github/workflows/standard_examples.yml b/.github/workflows/standard_examples.yml index 40b9f90..b40be12 100644 --- a/.github/workflows/standard_examples.yml +++ b/.github/workflows/standard_examples.yml @@ -22,10 +22,18 @@ jobs: python-version: ["3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v7 + # The standard is validated at a pinned commit (bumped by the + # weekly_update workflow), so changes to it cannot break unrelated PRs. + - name: Read the pinned PALS standard commit + id: pin + run: | + sha=$(grep -vE '^[[:space:]]*(#|$)' tests/standard_examples_git_reference.txt | head -n1) + echo "sha=${sha}" >> "$GITHUB_OUTPUT" - name: Checkout PALS repository uses: actions/checkout@v7 with: repository: pals-project/pals + ref: ${{ steps.pin.outputs.sha }} path: pals_temp fetch-depth: 1 sparse-checkout: | @@ -40,6 +48,6 @@ jobs: pip install ".[test]" - name: Run standard examples run: | - for file in pals_temp/examples/*.pals.yaml; do - python tests/validate_standard_examples.py --path "${file}" - done + python tests/validate_standard_examples.py \ + --root pals_temp/examples \ + --known-failures tests/standard_examples_known_failures.txt diff --git a/.github/workflows/weekly_update.yml b/.github/workflows/weekly_update.yml new file mode 100644 index 0000000..b33f593 --- /dev/null +++ b/.github/workflows/weekly_update.yml @@ -0,0 +1,72 @@ +name: Weekly update + +# Bumps the pinned pals-project/pals commit +# (tests/standard_examples_git_reference.txt) that +# the standard_examples workflow validates against, as a PR. A change to the +# standard that breaks the examples validation therefore shows up in that PR, +# not in unrelated ones. + +on: + schedule: + - cron: "0 9 * * 1" # every Monday at 1:00am PST (9:00am UTC) + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + autoupdate: + name: Autoupdate + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout repository + uses: actions/checkout@v7 + - name: Configure Git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + - name: Checkout branch + run: | + git checkout weekly_update || git checkout -b weekly_update + - name: Update the pinned PALS standard commit + id: bump + run: | + old=$(grep -vE '^[[:space:]]*(#|$)' tests/standard_examples_git_reference.txt | head -n1) + new=$(git ls-remote https://github.com/pals-project/pals.git refs/heads/main | cut -f1) + sed -i "s/^${old}\$/${new}/" tests/standard_examples_git_reference.txt + echo "old=${old}" >> "$GITHUB_OUTPUT" + echo "new=${new}" >> "$GITHUB_OUTPUT" + - name: Commit changes + id: commit + run: | + if git diff --quiet; then + echo "The pin already matches pals main; nothing to update." + echo "changed=false" >> "$GITHUB_OUTPUT" + else + git add -u + git commit -m "Update the pinned PALS standard commit" + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + - name: Push changes + if: steps.commit.outputs.changed == 'true' + run: | + git push -f origin weekly_update + - name: Open pull request + if: steps.commit.outputs.changed == 'true' + run: | + if [ -n "$(gh pr list --head weekly_update --state open --json number --jq '.[].number')" ]; then + echo "An open weekly_update PR already exists; the push updated it." + exit 0 + fi + gh pr create \ + --base main \ + --title "PALS standard: weekly update of the pinned commit" \ + --body "Bump the pinned pals-project/pals commit from \ + pals-project/pals@${{ steps.bump.outputs.old }} to \ + pals-project/pals@${{ steps.bump.outputs.new }} \ + (https://github.com/pals-project/pals/compare/${{ steps.bump.outputs.old }}...${{ steps.bump.outputs.new }}). + + Automated via .github/workflows/weekly_update.yml." diff --git a/tests/standard_examples_git_reference.txt b/tests/standard_examples_git_reference.txt new file mode 100644 index 0000000..2f92b1d --- /dev/null +++ b/tests/standard_examples_git_reference.txt @@ -0,0 +1,10 @@ +# The commit of pals-project/pals that the standard_examples workflow checks +# out and validates the examples corpus from. Pinning a fixed commit keeps +# changes to the standard from breaking unrelated pals-python PRs; the +# weekly_update workflow bumps this pin to the current main in a PR of its +# own, so a breaking change in the standard surfaces there, reviewed and +# controlled. +# +# The first non-comment line is the pin: a full commit SHA of +# https://github.com/pals-project/pals +5c39fde043ddad7a961ede01a5ad73d90b098aec diff --git a/tests/standard_examples_known_failures.txt b/tests/standard_examples_known_failures.txt new file mode 100644 index 0000000..4365661 --- /dev/null +++ b/tests/standard_examples_known_failures.txt @@ -0,0 +1,42 @@ +# Standard example files pals-python cannot read yet, one path per line +# relative to the standard's examples directory. validate_standard_examples.py +# treats a failure of a listed file as expected and a listed file that starts +# to load as an error, so this list shrinks as support lands. Blank lines and +# `#` comments are ignored. + +# PALSroot models only `version` and `facility`, and `facility` is required: +# additional attributes (notes, extension labels, or a version +# only) are rejected. +unit_tests/loading/basic/joiner.pals.yaml +unit_tests/loading/diamond/joiner.pals.yaml +unit_tests/loading/diamond/left.pals.yaml +unit_tests/loading/diamond/right.pals.yaml +unit_tests/loading/diamond/shared.pals.yaml +unit_tests/loading/extension_labels/a.pals.yaml +unit_tests/loading/extension_labels/joiner.pals.yaml +unit_tests/loading/implicit_self/a.pals.yaml +unit_tests/loading/implicit_self/joiner.pals.yaml +unit_tests/loading/nested/a.pals.yaml +unit_tests/loading/nested/inner.pals.yaml +unit_tests/loading/nested/joiner.pals.yaml +unit_tests/loading/relative_paths/joiner.pals.yaml +unit_tests/loading/relative_paths/sub/inner.pals.yaml +unit_tests/loading/relative_paths/top.pals.yaml +unit_tests/loading/self_position/a.pals.yaml +unit_tests/loading/self_position/b.pals.yaml +unit_tests/loading/self_position/joiner.pals.yaml +unit_tests/loading/version_match/a.pals.yaml +unit_tests/loading/version_match/joiner.pals.yaml + +# An integer `version` (root_keys writes `version: 1`) is rejected; the model +# wants a string or null. +unit_tests/document/root_keys.pals.yaml + +# `include` entries inside a facility are not resolved. +machine/machine.pals.yaml +unit_tests/loading/include/sub/layout.pals.yaml + +# The sequence form of `variables` and the compact `sets` form are not +# modeled. +unit_tests/expressions/inline_expressions.pals.yaml +unit_tests/sets/sets_compact.pals.yaml diff --git a/tests/validate_standard_examples.py b/tests/validate_standard_examples.py index 6cd46cd..1ff0543 100644 --- a/tests/validate_standard_examples.py +++ b/tests/validate_standard_examples.py @@ -1,37 +1,39 @@ -"""Utility script to validate standard PALS example files. +"""Validate the standard PALS example files from pals-project/pals/examples. -This script is not run by pytest and is intended to be used as a standalone script. -Run it from the repository root like: +Every *.pals.yaml file under --root is discovered and must be readable with +pals.load(). *.subpals.yaml files are exempt: per the standard's notation +section they are sub-level include fragments, spliced into (and read through) +the file that includes them. - python tests/validate_standard_examples.py --path /path/to/example.pals.yaml +Files this implementation cannot read yet are recorded in the known-failures +list (tests/standard_examples_known_failures.txt): a failure of a listed file +is expected, and a listed file that starts to load is reported so the list +shrinks as support lands. -Before running, download the desired standard PALS example files from pals-project/pals/examples. +This script is not run by pytest; the standard_examples workflow runs it +against a checkout of the standard, pinned to the commit named in +tests/standard_examples_git_reference.txt. Run it locally, from the repository root: + + python tests/validate_standard_examples.py \ + --root /path/to/pals/examples \ + --known-failures tests/standard_examples_known_failures.txt """ import argparse +import pathlib +import sys -from pals import load -from pals.kinds import PlaceholderName -from pals.kinds.BeamLine import BeamLine -from pals.kinds.Drift import Drift -from pals.kinds.Lattice import Lattice -from pals.kinds.Quadrupole import Quadrupole +import pals -def main(): - # Parse command-line arguments - parser = argparse.ArgumentParser() - parser.add_argument( - "--path", - required=True, - help="Path to the example file", - ) - args = parser.parse_args() - example_file = args.path - # Parse and validate YAML data from file - lattice = load(example_file) - # The following assertions are based on the standard PALS example file - # fodo.pals.yaml from pals-project/pals/examples +def check_fodo(lattice): + """Structural spot-checks of the introductory example fodo.pals.yaml.""" + from pals.kinds import PlaceholderName + from pals.kinds.BeamLine import BeamLine + from pals.kinds.Drift import Drift + from pals.kinds.Lattice import Lattice + from pals.kinds.Quadrupole import Quadrupole + assert isinstance(lattice.facility[0], Drift) assert lattice.facility[0].name == "drift1" assert isinstance(lattice.facility[1], Quadrupole) @@ -45,5 +47,73 @@ def main(): assert isinstance(lattice.facility[5], PlaceholderName) +def read_known_failures(path): + """The known-failures list: one root-relative path per line, # comments.""" + known = set() + for line in pathlib.Path(path).read_text().splitlines(): + line = line.strip() + if line and not line.startswith("#"): + known.add(line) + return known + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--root", + required=True, + help="Path to the standard's examples directory", + ) + parser.add_argument( + "--known-failures", + required=True, + help="Path to the known-failures list", + ) + args = parser.parse_args() + + root = pathlib.Path(args.root) + known = read_known_failures(args.known_failures) + + # Track every unexpected result for the final exit status. + failures = 0 + seen = set() + files = sorted(root.rglob("*.pals.yaml")) + for path in files: + rel = path.relative_to(root).as_posix() + seen.add(rel) + error = None + # Capture load errors so expected failures can be distinguished. + try: + lattice = pals.load(str(path)) + if rel == "fodo.pals.yaml": + check_fodo(lattice) + except Exception as e: # noqa: BLE001 -- any reader failure counts + error = e + if rel in known: + if error is None: + print(f"XPASS {rel}") + print(" loads now: remove it from the known-failures list") + failures += 1 + else: + print(f"XFAIL {rel}") + elif error is None: + print(f"PASS {rel}") + else: + print(f"FAIL {rel}") + message = f"{type(error).__name__}: {error}" + for detail in message.splitlines()[:3]: + print(f" {detail}") + failures += 1 + + # Flag stale entries in the known-failures list. + for rel in sorted(known - seen): + print(f"FAIL {rel}") + print(" in the known-failures list, but not found under --root") + failures += 1 + + print(f"{len(files)} files checked, {failures} unexpected results") + return 1 if failures else 0 + + if __name__ == "__main__": - main() + sys.exit(main())