Literal Include

src/pals/PALS.py

@@ -1,4 +1,4 @@
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 from pydantic import model_validator
 from typing import Self
@@ -40,9 +40,12 @@ class ExtensionLabels(BaseModel, extra="forbid"):
 class PALSroot(BaseModel):
     """Represent the root PALS structure"""
 
+    # Preserve root-level standard metadata that is not modeled explicitly yet.
+    model_config = ConfigDict(extra="allow")
+
     # The standard documents `version` as a string, but the standard's own
-    # examples also write bare numbers (e.g. `version: 1`).
-    version: str | int | None = None
+    # examples also write bare numbers (e.g. `version: 1` or `version: 1.0`).
+    version: str | int | float | None = None
 
     authors: list[Author] | None = None
 

src/pals/functions.py

@@ -3,18 +3,24 @@
 import os
 
 
-def inspect_file_extensions(filename: str):
+def inspect_file_extensions(filename: str, sub_level: bool = False):
     """Attempt to strip two levels of file extensions to determine the schema.
 
     filename examples: fodo.pals.yaml, fodo.pals.json, ...
+
+    Sub-level files, spliced into another file by its `include` entries, use
+    the inner extension .subpals per the standard's File Formats section
+    (e.g. elements.subpals.yaml); .pals is accepted for them as well.
     """
     file_noext, extension = os.path.splitext(filename)
     file_noext_noext, extension_inner = os.path.splitext(file_noext)
 
-    if extension_inner != ".pals":
+    allowed_inner = (".pals", ".subpals") if sub_level else (".pals",)
+    if extension_inner not in allowed_inner:
+        expected = " or ".join(f"{inner}.yaml" for inner in allowed_inner)
         raise RuntimeError(
             f"inspect_file_extensions: No support for file {filename} with extension {extension}. "
-            f"PALS files must end in .pals.json or .pals.yaml or similar."
+            f"PALS files must end in {expected} or similar."
         )
 
     return {
@@ -25,11 +31,98 @@ def inspect_file_extensions(filename: str):
     }
 
 
-def load_file_to_dict(filename: str) -> dict:
+def _load_included_file(include_file, base_dir: str, include_chain: tuple):
+    """Load the target of one `include` entry, relative to the including file."""
+    if not isinstance(include_file, str):
+        raise TypeError(
+            f"process_includes: an 'include' value must be a file name string, "
+            f"but we got {include_file!r}"
+        )
+    filepath = os.path.join(base_dir, include_file)
+    return load_file_to_dict(filepath, sub_level=True, _include_chain=include_chain)
+
+
+def process_includes(data, base_dir: str, include_chain: tuple = ()):
+    """Recursively resolve `include` entries in the data structure.
+
+    Per the standard, included file data is included verbatim at the current
+    level of nesting: an `include` key in a mapping splices the included
+    mapping's entries into it (entries local to the mapping win), and a list
+    item holding only an `include` key splices the included sequence into the
+    list. Include file names are resolved relative to the including file.
+
+    Args:
+        data: The parsed data structure to resolve
+        base_dir: Directory of the file the data came from
+        include_chain: Files on the include path so far, for cycle detection
+
+    Returns:
+        The data structure with all includes resolved
+    """
+    if isinstance(data, dict):
+        if "include" in data:
+            included_data = _load_included_file(
+                data["include"], base_dir, include_chain
+            )
+            if not isinstance(included_data, dict):
+                raise TypeError(
+                    f"process_includes: file {data['include']!r} is included at a "
+                    f"mapping level and must hold a mapping, "
+                    f"but we got {type(included_data).__name__}"
+                )
+            local_data = {
+                key: process_includes(value, base_dir, include_chain)
+                for key, value in data.items()
+                if key != "include"
+            }
+            # Entries local to the including mapping win over included ones.
+            return {**included_data, **local_data}
+
+        return {
+            key: process_includes(value, base_dir, include_chain)
+            for key, value in data.items()
+        }
+
+    elif isinstance(data, list):
+        new_list = []
+        for item in data:
+            # A list item holding only an include splices in the included file
+            if isinstance(item, dict) and set(item) == {"include"}:
+                included_data = _load_included_file(
+                    item["include"], base_dir, include_chain
+                )
+                if isinstance(included_data, list):
+                    new_list.extend(included_data)
+                elif isinstance(included_data, dict):
+                    new_list.append(included_data)
+                else:
+                    raise TypeError(
+                        f"process_includes: file {item['include']!r} is included at a "
+                        f"sequence level and must hold a sequence or mapping, "
+                        f"but we got {type(included_data).__name__}"
+                    )
+            else:
+                new_list.append(process_includes(item, base_dir, include_chain))
+        return new_list
+
+    else:
+        return data
+
+
+def load_file_to_dict(
+    filename: str, sub_level: bool = False, _include_chain: tuple = ()
+) -> dict:
+    # Guard against include cycles: a file including itself through any chain.
+    # realpath canonicalizes symlinks so a cycle cannot hide behind one.
+    filepath = os.path.realpath(filename)
+    if filepath in _include_chain:
+        chain = " -> ".join(_include_chain + (filepath,))
+        raise RuntimeError(f"load_file_to_dict: circular include: {chain}")
+
     # Attempt to strip two levels of file extensions to determine the schema.
     #   Examples: fodo.pals.yaml, fodo.pals.json, ...
     file_noext, extension, file_noext_noext, extension_inner = inspect_file_extensions(
-        filename
+        filename, sub_level=sub_level
     ).values()
 
     # examples: fodo.pals.yaml, fodo.pals.json
@@ -51,6 +144,17 @@ def load_file_to_dict(filename: str) -> dict:
                 f"load_file_to_dict: No support for PALS file {filename} with extension {extension} yet."
             )
 
+    # Resolve include entries, tracking this file for cycle detection. In a
+    # full document, include statements must be within the PALS root node;
+    # information outside of it is outside the standard and is not touched.
+    base_dir = os.path.dirname(filename)
+    include_chain = _include_chain + (filepath,)
+    if isinstance(pals_data, dict) and "PALS" in pals_data:
+        pals_data = dict(pals_data)
+        pals_data["PALS"] = process_includes(pals_data["PALS"], base_dir, include_chain)
+    else:
+        pals_data = process_includes(pals_data, base_dir, include_chain)
+
     return pals_data
 
 

src/pals/kinds/Lattice.py

@@ -30,8 +30,46 @@ def model_dump(self, *args, **kwargs):
 
     @staticmethod
     def from_file(filename: str) -> Self:
-        """Load a Lattice from a text file"""
+        """Load a Lattice from a text file.
+
+        The file can hold either a single Lattice or a full PALS document.
+        Per the standard's use statement, the lattice instantiated from a full
+        document is the last one defined, unless a `use` entry selects another.
+        """
         pals_dict = load_file_to_dict(filename)
+
+        if isinstance(pals_dict, dict) and "PALS" in pals_dict:
+            from pals.PALS import PALSroot
+            from pals.kinds.PlaceholderName import PlaceholderName
+
+            pals_root = PALSroot(**pals_dict)
+            facility = pals_root.facility or []
+            lattices = [item for item in facility if isinstance(item, Lattice)]
+            by_name = {lattice.name: lattice for lattice in lattices}
+
+            # A `use` entry overrides the last-lattice default; with several,
+            # the last one wins. It must name a Lattice the document defines.
+            use_entries = [
+                item
+                for item in facility
+                if isinstance(item, PlaceholderName) and item.is_use
+            ]
+            if use_entries:
+                selected = use_entries[-1].name
+                if selected not in by_name:
+                    raise ValueError(
+                        f"PALS root document {filename!r} selects {selected!r} "
+                        f"with its use entry, but defines no Lattice of that "
+                        f"name; defined Lattices: {sorted(by_name)}"
+                    )
+                return by_name[selected]
+
+            if not lattices:
+                raise ValueError(
+                    f"PALS root document {filename!r} does not define a Lattice"
+                )
+            return lattices[-1]
+
         return Lattice(**pals_dict)
 
     def to_file(self, filename: str):

src/pals/kinds/PlaceholderName.py

@@ -41,14 +41,21 @@ class PlaceholderName(BaseModel):
         "BaseElement | None",
         Field(default=None, description="Reference to the resolved element object"),
     ] = None
+    is_use: bool = Field(
+        default=False,
+        description="True when this reference was written as a `use:` entry",
+    )
 
     @model_serializer(mode="plain")
-    def _serialize_as_name(self) -> str:
-        """Serialize this reference as just its name.
+    def _serialize_as_name(self) -> str | dict[str, str]:
+        """Serialize this reference as its name, or its `use:` entry form.
 
         This makes `model_dump()` return a string (the element name), so nested
         serialization (e.g. inside BeamLine.line) produces plain strings too.
+        References written as `use:` entries keep that form.
         """
+        if self.is_use:
+            return {"use": self.name}
         return self.name
 
     def __init__(self, name: str | None = None, /, **data):

src/pals/kinds/mixin/all_element_mixin.py

@@ -51,7 +51,7 @@ def unpack_element_items(items: list, container_type: str) -> list:
             # can resolve it.
             if not isinstance(fields, dict):
                 if name == "use" and isinstance(fields, str):
-                    new_list.append(PlaceholderName(fields))
+                    new_list.append(PlaceholderName(fields, is_use=True))
                     continue
                 raise TypeError(
                     f"Value for element key {name!r} must be a dict (the element's properties), "

tests/pals_files/include/circular/a.pals.yaml

@@ -0,0 +1,3 @@
+# Includes b.subpals.yaml, which includes this file again: a cycle.
+PALS:
+  include: "b.subpals.yaml"

tests/pals_files/include/circular/b.subpals.yaml

@@ -0,0 +1,2 @@
+# Includes a.pals.yaml, closing the include cycle.
+include: "a.pals.yaml"

tests/pals_files/include/element/include-Q-params.subpals.yaml

@@ -0,0 +1,3 @@
+# Included into q01.pals.yaml, from the standard's include example.
+MagneticMultipoleP:
+  Kn3L: 0.3

tests/pals_files/include/element/q01.pals.yaml

@@ -0,0 +1,5 @@
+# The standard's element-level include example: the element pulls a
+# parameter group in from include-Q-params.subpals.yaml.
+Q01:
+  kind: Quadrupole
+  include: "include-Q-params.subpals.yaml"

tests/pals_files/include/globals.subpals.yaml

@@ -0,0 +1,6 @@
+# Included at the root level of main.pals.yaml: version and notes per the
+# standard's example, plus an unmodeled key that must be preserved.
+version: 1.0
+notes:
+  - "included note"
+my_extension_data: "kept"

tests/pals_files/include/main.pals.yaml

@@ -0,0 +1,28 @@
+# A FODO document assembled with includes: a root-level include contributes
+# document metadata (globals.subpals.yaml) and a facility-level include
+# splices in element definitions (sub/quads.subpals.yaml, which itself
+# includes parts/extra.subpals.yaml relative to its own directory).
+PALS:
+  include: "globals.subpals.yaml"
+  facility:
+    - drift1:
+        kind: Drift
+        length: 0.25
+
+    - include: "sub/quads.subpals.yaml"
+
+    - fodo_cell:
+        kind: BeamLine
+        line:
+          - drift1
+          - quad1
+          - drift2
+          - quad2
+          - drift1
+
+    - fodo_lattice:
+        kind: Lattice
+        branches:
+          - fodo_cell
+
+    - use: fodo_lattice

tests/pals_files/include/mismatch/elements.subpals.yaml

@@ -0,0 +1,3 @@
+# A sequence, included by q01.pals.yaml at a mapping level: an error.
+- a:
+    kind: Drift

tests/pals_files/include/mismatch/q01.pals.yaml

@@ -0,0 +1,4 @@
+# Structural error: includes a file holding a sequence at a mapping level.
+Q01:
+  kind: Quadrupole
+  include: "elements.subpals.yaml"

tests/pals_files/include/nested/leaf.subpals.yaml

@@ -0,0 +1,3 @@
+# Included by middle.subpals.yaml; its `shared` entry is overridden.
+leaf: val
+shared: included

tests/pals_files/include/nested/middle.subpals.yaml

@@ -0,0 +1,5 @@
+# Included by root.pals.yaml; includes leaf.subpals.yaml itself. Its own
+# `shared` entry wins over the one from the leaf.
+middle: val
+shared: local
+include: "leaf.subpals.yaml"

tests/pals_files/include/nested/root.pals.yaml

@@ -0,0 +1,5 @@
+# A chain of includes: this file includes middle.subpals.yaml, which
+# includes leaf.subpals.yaml. Keys local to an including file win over
+# included ones.
+root:
+  include: "middle.subpals.yaml"

tests/pals_files/include/outside/globals.subpals.yaml

@@ -0,0 +1,2 @@
+# Included within the PALS node of main.pals.yaml.
+version: "1.0"

tests/pals_files/include/outside/main.pals.yaml

@@ -0,0 +1,9 @@
+# Information outside the PALS root node is outside the standard and is
+# ignored: the top-level include names a file that does not exist and must
+# not be resolved. The include inside the PALS node is.
+include: "missing.subpals.yaml"
+not_pals_data: true
+PALS:
+  include: "globals.subpals.yaml"
+  notes:
+    - "the PALS node itself"

tests/pals_files/include/parts/extra.subpals.yaml

@@ -0,0 +1,6 @@
+# Spliced into sub/quads.subpals.yaml by its include entry.
+- quad2:
+    kind: Quadrupole
+    MagneticMultipoleP:
+      Bn1: -1.0
+    length: 1.0

tests/pals_files/include/sub/quads.subpals.yaml

@@ -0,0 +1,13 @@
+# Spliced into the facility of main.pals.yaml; includes a further file
+# relative to its own directory (not the directory of the main file).
+- quad1:
+    kind: Quadrupole
+    MagneticMultipoleP:
+      Bn1: 1.0
+    length: 1.0
+
+- drift2:
+    kind: Drift
+    length: 0.5
+
+- include: "../parts/extra.subpals.yaml"

tests/pals_files/lattice_use/bad_use.pals.yaml

@@ -0,0 +1,18 @@
+# A use entry naming no defined Lattice: since use overrides the default
+# lattice selection, this is an error, not a fallback to the last lattice.
+PALS:
+  facility:
+    - line1:
+        kind: BeamLine
+        line:
+          - m1:
+              kind: Marker
+    - lat1:
+        kind: Lattice
+        branches:
+          - line1
+    - lat2:
+        kind: Lattice
+        branches:
+          - line1
+    - use: "typo"

tests/pals_files/lattice_use/no_lattice.pals.yaml

@@ -0,0 +1,4 @@
+# A document defining no Lattice: Lattice.from_file reports an error.
+PALS:
+  notes:
+    - "no facility here"