diff --git a/src/pals/PALS.py b/src/pals/PALS.py index 9354219..e0a97e2 100644 --- a/src/pals/PALS.py +++ b/src/pals/PALS.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from pydantic import model_validator from typing import Self @@ -40,9 +40,12 @@ class ExtensionLabels(BaseModel, extra="forbid"): class PALSroot(BaseModel): """Represent the root PALS structure""" + # Preserve root-level standard metadata that is not modeled explicitly yet. + model_config = ConfigDict(extra="allow") + # The standard documents `version` as a string, but the standard's own - # examples also write bare numbers (e.g. `version: 1`). - version: str | int | None = None + # examples also write bare numbers (e.g. `version: 1` or `version: 1.0`). + version: str | int | float | None = None authors: list[Author] | None = None diff --git a/src/pals/functions.py b/src/pals/functions.py index 531e9e1..91b3649 100644 --- a/src/pals/functions.py +++ b/src/pals/functions.py @@ -3,18 +3,24 @@ import os -def inspect_file_extensions(filename: str): +def inspect_file_extensions(filename: str, sub_level: bool = False): """Attempt to strip two levels of file extensions to determine the schema. filename examples: fodo.pals.yaml, fodo.pals.json, ... + + Sub-level files, spliced into another file by its `include` entries, use + the inner extension .subpals per the standard's File Formats section + (e.g. elements.subpals.yaml); .pals is accepted for them as well. """ file_noext, extension = os.path.splitext(filename) file_noext_noext, extension_inner = os.path.splitext(file_noext) - if extension_inner != ".pals": + allowed_inner = (".pals", ".subpals") if sub_level else (".pals",) + if extension_inner not in allowed_inner: + expected = " or ".join(f"{inner}.yaml" for inner in allowed_inner) raise RuntimeError( f"inspect_file_extensions: No support for file {filename} with extension {extension}. " - f"PALS files must end in .pals.json or .pals.yaml or similar." + f"PALS files must end in {expected} or similar." ) return { @@ -25,11 +31,98 @@ def inspect_file_extensions(filename: str): } -def load_file_to_dict(filename: str) -> dict: +def _load_included_file(include_file, base_dir: str, include_chain: tuple): + """Load the target of one `include` entry, relative to the including file.""" + if not isinstance(include_file, str): + raise TypeError( + f"process_includes: an 'include' value must be a file name string, " + f"but we got {include_file!r}" + ) + filepath = os.path.join(base_dir, include_file) + return load_file_to_dict(filepath, sub_level=True, _include_chain=include_chain) + + +def process_includes(data, base_dir: str, include_chain: tuple = ()): + """Recursively resolve `include` entries in the data structure. + + Per the standard, included file data is included verbatim at the current + level of nesting: an `include` key in a mapping splices the included + mapping's entries into it (entries local to the mapping win), and a list + item holding only an `include` key splices the included sequence into the + list. Include file names are resolved relative to the including file. + + Args: + data: The parsed data structure to resolve + base_dir: Directory of the file the data came from + include_chain: Files on the include path so far, for cycle detection + + Returns: + The data structure with all includes resolved + """ + if isinstance(data, dict): + if "include" in data: + included_data = _load_included_file( + data["include"], base_dir, include_chain + ) + if not isinstance(included_data, dict): + raise TypeError( + f"process_includes: file {data['include']!r} is included at a " + f"mapping level and must hold a mapping, " + f"but we got {type(included_data).__name__}" + ) + local_data = { + key: process_includes(value, base_dir, include_chain) + for key, value in data.items() + if key != "include" + } + # Entries local to the including mapping win over included ones. + return {**included_data, **local_data} + + return { + key: process_includes(value, base_dir, include_chain) + for key, value in data.items() + } + + elif isinstance(data, list): + new_list = [] + for item in data: + # A list item holding only an include splices in the included file + if isinstance(item, dict) and set(item) == {"include"}: + included_data = _load_included_file( + item["include"], base_dir, include_chain + ) + if isinstance(included_data, list): + new_list.extend(included_data) + elif isinstance(included_data, dict): + new_list.append(included_data) + else: + raise TypeError( + f"process_includes: file {item['include']!r} is included at a " + f"sequence level and must hold a sequence or mapping, " + f"but we got {type(included_data).__name__}" + ) + else: + new_list.append(process_includes(item, base_dir, include_chain)) + return new_list + + else: + return data + + +def load_file_to_dict( + filename: str, sub_level: bool = False, _include_chain: tuple = () +) -> dict: + # Guard against include cycles: a file including itself through any chain. + # realpath canonicalizes symlinks so a cycle cannot hide behind one. + filepath = os.path.realpath(filename) + if filepath in _include_chain: + chain = " -> ".join(_include_chain + (filepath,)) + raise RuntimeError(f"load_file_to_dict: circular include: {chain}") + # Attempt to strip two levels of file extensions to determine the schema. # Examples: fodo.pals.yaml, fodo.pals.json, ... file_noext, extension, file_noext_noext, extension_inner = inspect_file_extensions( - filename + filename, sub_level=sub_level ).values() # examples: fodo.pals.yaml, fodo.pals.json @@ -51,6 +144,17 @@ def load_file_to_dict(filename: str) -> dict: f"load_file_to_dict: No support for PALS file {filename} with extension {extension} yet." ) + # Resolve include entries, tracking this file for cycle detection. In a + # full document, include statements must be within the PALS root node; + # information outside of it is outside the standard and is not touched. + base_dir = os.path.dirname(filename) + include_chain = _include_chain + (filepath,) + if isinstance(pals_data, dict) and "PALS" in pals_data: + pals_data = dict(pals_data) + pals_data["PALS"] = process_includes(pals_data["PALS"], base_dir, include_chain) + else: + pals_data = process_includes(pals_data, base_dir, include_chain) + return pals_data diff --git a/src/pals/kinds/Lattice.py b/src/pals/kinds/Lattice.py index 5610e8e..61e7276 100644 --- a/src/pals/kinds/Lattice.py +++ b/src/pals/kinds/Lattice.py @@ -30,8 +30,46 @@ def model_dump(self, *args, **kwargs): @staticmethod def from_file(filename: str) -> Self: - """Load a Lattice from a text file""" + """Load a Lattice from a text file. + + The file can hold either a single Lattice or a full PALS document. + Per the standard's use statement, the lattice instantiated from a full + document is the last one defined, unless a `use` entry selects another. + """ pals_dict = load_file_to_dict(filename) + + if isinstance(pals_dict, dict) and "PALS" in pals_dict: + from pals.PALS import PALSroot + from pals.kinds.PlaceholderName import PlaceholderName + + pals_root = PALSroot(**pals_dict) + facility = pals_root.facility or [] + lattices = [item for item in facility if isinstance(item, Lattice)] + by_name = {lattice.name: lattice for lattice in lattices} + + # A `use` entry overrides the last-lattice default; with several, + # the last one wins. It must name a Lattice the document defines. + use_entries = [ + item + for item in facility + if isinstance(item, PlaceholderName) and item.is_use + ] + if use_entries: + selected = use_entries[-1].name + if selected not in by_name: + raise ValueError( + f"PALS root document {filename!r} selects {selected!r} " + f"with its use entry, but defines no Lattice of that " + f"name; defined Lattices: {sorted(by_name)}" + ) + return by_name[selected] + + if not lattices: + raise ValueError( + f"PALS root document {filename!r} does not define a Lattice" + ) + return lattices[-1] + return Lattice(**pals_dict) def to_file(self, filename: str): diff --git a/src/pals/kinds/PlaceholderName.py b/src/pals/kinds/PlaceholderName.py index c9b7c96..aef0ebf 100644 --- a/src/pals/kinds/PlaceholderName.py +++ b/src/pals/kinds/PlaceholderName.py @@ -41,14 +41,21 @@ class PlaceholderName(BaseModel): "BaseElement | None", Field(default=None, description="Reference to the resolved element object"), ] = None + is_use: bool = Field( + default=False, + description="True when this reference was written as a `use:` entry", + ) @model_serializer(mode="plain") - def _serialize_as_name(self) -> str: - """Serialize this reference as just its name. + def _serialize_as_name(self) -> str | dict[str, str]: + """Serialize this reference as its name, or its `use:` entry form. This makes `model_dump()` return a string (the element name), so nested serialization (e.g. inside BeamLine.line) produces plain strings too. + References written as `use:` entries keep that form. """ + if self.is_use: + return {"use": self.name} return self.name def __init__(self, name: str | None = None, /, **data): diff --git a/src/pals/kinds/mixin/all_element_mixin.py b/src/pals/kinds/mixin/all_element_mixin.py index d67e5c8..3b40795 100644 --- a/src/pals/kinds/mixin/all_element_mixin.py +++ b/src/pals/kinds/mixin/all_element_mixin.py @@ -51,7 +51,7 @@ def unpack_element_items(items: list, container_type: str) -> list: # can resolve it. if not isinstance(fields, dict): if name == "use" and isinstance(fields, str): - new_list.append(PlaceholderName(fields)) + new_list.append(PlaceholderName(fields, is_use=True)) continue raise TypeError( f"Value for element key {name!r} must be a dict (the element's properties), " diff --git a/tests/pals_files/include/circular/a.pals.yaml b/tests/pals_files/include/circular/a.pals.yaml new file mode 100644 index 0000000..2678f53 --- /dev/null +++ b/tests/pals_files/include/circular/a.pals.yaml @@ -0,0 +1,3 @@ +# Includes b.subpals.yaml, which includes this file again: a cycle. +PALS: + include: "b.subpals.yaml" diff --git a/tests/pals_files/include/circular/b.subpals.yaml b/tests/pals_files/include/circular/b.subpals.yaml new file mode 100644 index 0000000..3db2753 --- /dev/null +++ b/tests/pals_files/include/circular/b.subpals.yaml @@ -0,0 +1,2 @@ +# Includes a.pals.yaml, closing the include cycle. +include: "a.pals.yaml" diff --git a/tests/pals_files/include/element/include-Q-params.subpals.yaml b/tests/pals_files/include/element/include-Q-params.subpals.yaml new file mode 100644 index 0000000..da04131 --- /dev/null +++ b/tests/pals_files/include/element/include-Q-params.subpals.yaml @@ -0,0 +1,3 @@ +# Included into q01.pals.yaml, from the standard's include example. +MagneticMultipoleP: + Kn3L: 0.3 diff --git a/tests/pals_files/include/element/q01.pals.yaml b/tests/pals_files/include/element/q01.pals.yaml new file mode 100644 index 0000000..1061e52 --- /dev/null +++ b/tests/pals_files/include/element/q01.pals.yaml @@ -0,0 +1,5 @@ +# The standard's element-level include example: the element pulls a +# parameter group in from include-Q-params.subpals.yaml. +Q01: + kind: Quadrupole + include: "include-Q-params.subpals.yaml" diff --git a/tests/pals_files/include/globals.subpals.yaml b/tests/pals_files/include/globals.subpals.yaml new file mode 100644 index 0000000..8d89b3b --- /dev/null +++ b/tests/pals_files/include/globals.subpals.yaml @@ -0,0 +1,6 @@ +# Included at the root level of main.pals.yaml: version and notes per the +# standard's example, plus an unmodeled key that must be preserved. +version: 1.0 +notes: + - "included note" +my_extension_data: "kept" diff --git a/tests/pals_files/include/main.pals.yaml b/tests/pals_files/include/main.pals.yaml new file mode 100644 index 0000000..7d6db4d --- /dev/null +++ b/tests/pals_files/include/main.pals.yaml @@ -0,0 +1,28 @@ +# A FODO document assembled with includes: a root-level include contributes +# document metadata (globals.subpals.yaml) and a facility-level include +# splices in element definitions (sub/quads.subpals.yaml, which itself +# includes parts/extra.subpals.yaml relative to its own directory). +PALS: + include: "globals.subpals.yaml" + facility: + - drift1: + kind: Drift + length: 0.25 + + - include: "sub/quads.subpals.yaml" + + - fodo_cell: + kind: BeamLine + line: + - drift1 + - quad1 + - drift2 + - quad2 + - drift1 + + - fodo_lattice: + kind: Lattice + branches: + - fodo_cell + + - use: fodo_lattice diff --git a/tests/pals_files/include/mismatch/elements.subpals.yaml b/tests/pals_files/include/mismatch/elements.subpals.yaml new file mode 100644 index 0000000..8ee321f --- /dev/null +++ b/tests/pals_files/include/mismatch/elements.subpals.yaml @@ -0,0 +1,3 @@ +# A sequence, included by q01.pals.yaml at a mapping level: an error. +- a: + kind: Drift diff --git a/tests/pals_files/include/mismatch/q01.pals.yaml b/tests/pals_files/include/mismatch/q01.pals.yaml new file mode 100644 index 0000000..f6fb04d --- /dev/null +++ b/tests/pals_files/include/mismatch/q01.pals.yaml @@ -0,0 +1,4 @@ +# Structural error: includes a file holding a sequence at a mapping level. +Q01: + kind: Quadrupole + include: "elements.subpals.yaml" diff --git a/tests/pals_files/include/nested/leaf.subpals.yaml b/tests/pals_files/include/nested/leaf.subpals.yaml new file mode 100644 index 0000000..06a5388 --- /dev/null +++ b/tests/pals_files/include/nested/leaf.subpals.yaml @@ -0,0 +1,3 @@ +# Included by middle.subpals.yaml; its `shared` entry is overridden. +leaf: val +shared: included diff --git a/tests/pals_files/include/nested/middle.subpals.yaml b/tests/pals_files/include/nested/middle.subpals.yaml new file mode 100644 index 0000000..4f63da2 --- /dev/null +++ b/tests/pals_files/include/nested/middle.subpals.yaml @@ -0,0 +1,5 @@ +# Included by root.pals.yaml; includes leaf.subpals.yaml itself. Its own +# `shared` entry wins over the one from the leaf. +middle: val +shared: local +include: "leaf.subpals.yaml" diff --git a/tests/pals_files/include/nested/root.pals.yaml b/tests/pals_files/include/nested/root.pals.yaml new file mode 100644 index 0000000..86f242a --- /dev/null +++ b/tests/pals_files/include/nested/root.pals.yaml @@ -0,0 +1,5 @@ +# A chain of includes: this file includes middle.subpals.yaml, which +# includes leaf.subpals.yaml. Keys local to an including file win over +# included ones. +root: + include: "middle.subpals.yaml" diff --git a/tests/pals_files/include/outside/globals.subpals.yaml b/tests/pals_files/include/outside/globals.subpals.yaml new file mode 100644 index 0000000..0518df1 --- /dev/null +++ b/tests/pals_files/include/outside/globals.subpals.yaml @@ -0,0 +1,2 @@ +# Included within the PALS node of main.pals.yaml. +version: "1.0" diff --git a/tests/pals_files/include/outside/main.pals.yaml b/tests/pals_files/include/outside/main.pals.yaml new file mode 100644 index 0000000..1a4a788 --- /dev/null +++ b/tests/pals_files/include/outside/main.pals.yaml @@ -0,0 +1,9 @@ +# Information outside the PALS root node is outside the standard and is +# ignored: the top-level include names a file that does not exist and must +# not be resolved. The include inside the PALS node is. +include: "missing.subpals.yaml" +not_pals_data: true +PALS: + include: "globals.subpals.yaml" + notes: + - "the PALS node itself" diff --git a/tests/pals_files/include/parts/extra.subpals.yaml b/tests/pals_files/include/parts/extra.subpals.yaml new file mode 100644 index 0000000..1fa547d --- /dev/null +++ b/tests/pals_files/include/parts/extra.subpals.yaml @@ -0,0 +1,6 @@ +# Spliced into sub/quads.subpals.yaml by its include entry. +- quad2: + kind: Quadrupole + MagneticMultipoleP: + Bn1: -1.0 + length: 1.0 diff --git a/tests/pals_files/include/sub/quads.subpals.yaml b/tests/pals_files/include/sub/quads.subpals.yaml new file mode 100644 index 0000000..78a1576 --- /dev/null +++ b/tests/pals_files/include/sub/quads.subpals.yaml @@ -0,0 +1,13 @@ +# Spliced into the facility of main.pals.yaml; includes a further file +# relative to its own directory (not the directory of the main file). +- quad1: + kind: Quadrupole + MagneticMultipoleP: + Bn1: 1.0 + length: 1.0 + +- drift2: + kind: Drift + length: 0.5 + +- include: "../parts/extra.subpals.yaml" diff --git a/tests/pals_files/lattice_use/bad_use.pals.yaml b/tests/pals_files/lattice_use/bad_use.pals.yaml new file mode 100644 index 0000000..5fd9bc3 --- /dev/null +++ b/tests/pals_files/lattice_use/bad_use.pals.yaml @@ -0,0 +1,18 @@ +# A use entry naming no defined Lattice: since use overrides the default +# lattice selection, this is an error, not a fallback to the last lattice. +PALS: + facility: + - line1: + kind: BeamLine + line: + - m1: + kind: Marker + - lat1: + kind: Lattice + branches: + - line1 + - lat2: + kind: Lattice + branches: + - line1 + - use: "typo" diff --git a/tests/pals_files/lattice_use/no_lattice.pals.yaml b/tests/pals_files/lattice_use/no_lattice.pals.yaml new file mode 100644 index 0000000..bd21b3a --- /dev/null +++ b/tests/pals_files/lattice_use/no_lattice.pals.yaml @@ -0,0 +1,4 @@ +# A document defining no Lattice: Lattice.from_file reports an error. +PALS: + notes: + - "no facility here" diff --git a/tests/pals_files/lattice_use/no_use.pals.yaml b/tests/pals_files/lattice_use/no_use.pals.yaml new file mode 100644 index 0000000..49fc108 --- /dev/null +++ b/tests/pals_files/lattice_use/no_use.pals.yaml @@ -0,0 +1,17 @@ +# Two lattices and no use entry: per the standard's use statement, the +# last lattice defined (lat2) is the one instantiated. +PALS: + facility: + - line1: + kind: BeamLine + line: + - m1: + kind: Marker + - lat1: + kind: Lattice + branches: + - line1 + - lat2: + kind: Lattice + branches: + - line1 diff --git a/tests/pals_files/lattice_use/with_use.pals.yaml b/tests/pals_files/lattice_use/with_use.pals.yaml new file mode 100644 index 0000000..058a364 --- /dev/null +++ b/tests/pals_files/lattice_use/with_use.pals.yaml @@ -0,0 +1,18 @@ +# Two lattices and a use entry: use overrides the last-lattice default, +# selecting lat1. +PALS: + facility: + - line1: + kind: BeamLine + line: + - m1: + kind: Marker + - lat1: + kind: Lattice + branches: + - line1 + - lat2: + kind: Lattice + branches: + - line1 + - use: "lat1" diff --git a/tests/standard_examples_known_failures.txt b/tests/standard_examples_known_failures.txt index 73bf3dd..ddd81f7 100644 --- a/tests/standard_examples_known_failures.txt +++ b/tests/standard_examples_known_failures.txt @@ -4,10 +4,6 @@ # to load as an error, so this list shrinks as support lands. Blank lines and # `#` comments are ignored. -# `include` entries inside a facility are not resolved. -machine/machine.pals.yaml -unit_tests/loading/include/sub/layout.pals.yaml - # The sequence form of `variables` and the compact `sets` form are not # modeled. unit_tests/expressions/inline_expressions.pals.yaml diff --git a/tests/test_include.py b/tests/test_include.py new file mode 100644 index 0000000..bf7f6c3 --- /dev/null +++ b/tests/test_include.py @@ -0,0 +1,125 @@ +"""Tests for resolving `include` entries while loading files. + +The documents used here live under tests/pals_files/include and +tests/pals_files/lattice_use, one directory per scenario. +""" + +import pathlib + +import pytest + +import pals + +PALS_FILES = pathlib.Path(__file__).parent / "pals_files" +INCLUDE = PALS_FILES / "include" +LATTICE_USE = PALS_FILES / "lattice_use" + + +def test_include(): + """Includes splice into the root node and the facility list.""" + main_file = str(INCLUDE / "main.pals.yaml") + + lattice = pals.Lattice.from_file(main_file) + assert lattice.name == "fodo_lattice" + assert lattice.branches[0] == "fodo_cell" + + data = pals.PALSroot.from_file(main_file) + # From the root-level include of globals.subpals.yaml, with the + # unmodeled key preserved. + assert data.version == 1.0 + assert data.notes == ["included note"] + assert data.my_extension_data == "kept" + assert not hasattr(data, "include") + + # sub/quads.subpals.yaml spliced in quad1 and drift2, and its own + # include (resolved relative to sub/) spliced in quad2. + names = [elem.name for elem in data.facility] + assert names == [ + "drift1", + "quad1", + "drift2", + "quad2", + "fodo_cell", + "fodo_lattice", + "fodo_lattice", + ] + assert isinstance(data.facility[3], pals.Quadrupole) + assert data.facility[3].MagneticMultipoleP.Bn1 == -1.0 + + +def test_nested_include(): + """A chain of includes resolves through each file; local keys win.""" + data = pals.functions.load_file_to_dict(str(INCLUDE / "nested/root.pals.yaml")) + + assert data["root"]["middle"] == "val" + assert data["root"]["leaf"] == "val" + # The including file's own entry wins over the included one. + assert data["root"]["shared"] == "local" + assert "include" not in data["root"] + + +def test_include_element_parameters(): + """The standard's element-level include example: a parameter group file.""" + data = pals.functions.load_file_to_dict(str(INCLUDE / "element/q01.pals.yaml")) + + assert data["Q01"] == { + "kind": "Quadrupole", + "MagneticMultipoleP": {"Kn3L": 0.3}, + } + + +def test_include_structure_mismatch(): + """Including a sequence at a mapping level is a structural error.""" + with pytest.raises(TypeError, match="mapping level"): + pals.functions.load_file_to_dict(str(INCLUDE / "mismatch/q01.pals.yaml")) + + +def test_include_outside_pals_node(): + """Include statements must be within the PALS root node; the top-level + sibling include naming a missing file is outside the standard and is + left untouched.""" + data = pals.functions.load_file_to_dict(str(INCLUDE / "outside/main.pals.yaml")) + assert data["include"] == "missing.subpals.yaml" + assert data["PALS"]["version"] == "1.0" + + root = pals.load(str(INCLUDE / "outside/main.pals.yaml")) + assert root.version == "1.0" + assert root.notes == ["the PALS node itself"] + + +def test_circular_include(): + """An include cycle is reported instead of recursing forever.""" + with pytest.raises(RuntimeError, match="circular include"): + pals.functions.load_file_to_dict(str(INCLUDE / "circular/a.pals.yaml")) + + +def test_symlinked_circular_include(tmp_path): + """A cycle routed through a directory symlink is still detected.""" + link = tmp_path / "link" + try: + link.symlink_to(tmp_path, target_is_directory=True) + except OSError: + pytest.skip("platform cannot create symlinks") + root_file = tmp_path / "root.pals.yaml" + root_file.write_text('PALS:\n include: "link/root.pals.yaml"\n') + + with pytest.raises(RuntimeError, match="circular include"): + pals.functions.load_file_to_dict(str(root_file)) + + +def test_lattice_from_full_document_use(): + """Per the use statement, the last lattice is instantiated unless a + `use` entry selects another.""" + lattice = pals.Lattice.from_file(str(LATTICE_USE / "no_use.pals.yaml")) + assert lattice.name == "lat2" + + lattice = pals.Lattice.from_file(str(LATTICE_USE / "with_use.pals.yaml")) + assert lattice.name == "lat1" + + with pytest.raises(ValueError, match="does not define a Lattice"): + pals.Lattice.from_file(str(LATTICE_USE / "no_lattice.pals.yaml")) + + # A use entry that names no defined Lattice errors instead of silently + # falling back to the last lattice. + with pytest.raises(ValueError, match="no Lattice of that name"): + pals.Lattice.from_file(str(LATTICE_USE / "bad_use.pals.yaml"))