Remove load_from/save_to folder_metadata in save/load operations

.github/scripts/serialization/objects.py

@@ -29,10 +29,13 @@
     "json": ".json",
     "pickle": ".pkl",
     "binary": "_binary",
+    "binary_parallel": "_binary_parallel",
     "numpy_folder": "_numpy_folder",
     "zarr": ".zarr",
+    "zarr_parallel": "_parallel.zarr",
 }
 
+DEFAULT_DURATION = 2.0  # seconds, for recordings and sortings
 
 # --- json (recipe) entries: moved class + a second class -------------------------
 
@@ -46,7 +49,12 @@ def _build_noise_generator_recording():
     except ImportError:
         from spikeinterface.core.generate import NoiseGeneratorRecording
 
-    return NoiseGeneratorRecording(num_channels=4, sampling_frequency=30000.0, durations=[1.0, 1.5], seed=0)
+    return NoiseGeneratorRecording(
+        num_channels=4,
+        sampling_frequency=30000.0,
+        durations=[DEFAULT_DURATION, DEFAULT_DURATION + 0.5],
+        seed=0,
+    )
 
 
 def _check_noise_generator_recording(rec):
@@ -59,7 +67,7 @@ def _check_noise_generator_recording(rec):
 def _build_mock_recording():
     from spikeinterface.core import generate_recording
 
-    return generate_recording(num_channels=4, durations=[1.0], sampling_frequency=30000.0, seed=0)
+    return generate_recording(num_channels=4, durations=[DEFAULT_DURATION], sampling_frequency=30000.0, seed=0)
 
 
 def _check_mock_recording(rec):
@@ -77,7 +85,7 @@ def _build_recording_with_properties():
     import numpy as np
     from spikeinterface.core import generate_recording
 
-    rec = generate_recording(num_channels=4, durations=[1.0], sampling_frequency=30000.0, seed=0)
+    rec = generate_recording(num_channels=4, durations=[DEFAULT_DURATION], sampling_frequency=30000.0, seed=0)
     rec.set_property("quality", np.array(["good", "good", "bad", "good"]))
     rec.annotate(experimenter="test")
     return rec
@@ -94,7 +102,7 @@ def _build_recording_with_probe():
     from probeinterface import generate_linear_probe
     from spikeinterface.core import generate_recording
 
-    rec = generate_recording(num_channels=8, durations=[1.0], sampling_frequency=30000.0, seed=0)
+    rec = generate_recording(num_channels=8, durations=[DEFAULT_DURATION], sampling_frequency=30000.0, seed=0)
     probe = generate_linear_probe(num_elec=8)
     probe.set_device_channel_indices(np.arange(8))
     if parse(si_version) <= parse("0.105.0"):
@@ -104,6 +112,20 @@ def _build_recording_with_probe():
     return rec_with_probe
 
 
+def _build_recording_with_timestamps():
+    rec = _build_mock_recording()
+    times = rec.get_times(segment_index=0) + 100
+    rec.set_times(times, segment_index=0)
+    return rec
+
+
+def _check_recording_with_timestamps(rec):
+    import numpy as np
+    expected_times = np.arange(int(DEFAULT_DURATION * 30000)) / 30000.0 + 100
+    times = rec.get_times(segment_index=0)
+    assert np.allclose(times, expected_times)
+
+
 def _check_recording_with_probe(rec):
     import numpy as np
 
@@ -121,7 +143,7 @@ def _build_recording_with_interleaved_probes():
     from probeinterface import ProbeGroup, generate_linear_probe
     from spikeinterface.core import generate_recording
 
-    rec = generate_recording(num_channels=8, durations=[1.0], sampling_frequency=30000.0, seed=0)
+    rec = generate_recording(num_channels=8, durations=[DEFAULT_DURATION], sampling_frequency=30000.0, seed=0)
     probe0 = generate_linear_probe(num_elec=4)
     probe1 = generate_linear_probe(num_elec=4)
     probe1.move([100.0, 0.0])
@@ -164,7 +186,7 @@ def _build_preprocessed_chain():
     from spikeinterface.core import generate_recording
     from spikeinterface.preprocessing import common_reference, scale
 
-    rec = generate_recording(num_channels=4, durations=[1.0], sampling_frequency=30000.0, seed=0)
+    rec = generate_recording(num_channels=4, durations=[DEFAULT_DURATION], sampling_frequency=30000.0, seed=0)
     # Two nested scipy-free preprocessing wrappers (scale then common_reference): this
     # exercises recursive parent reload without pulling scipy into the environments.
     return common_reference(scale(rec, gain=2.0))
@@ -181,7 +203,7 @@ def _check_preprocessed_chain(rec):
 def _build_sorting():
     from spikeinterface.core import generate_sorting
 
-    return generate_sorting(num_units=5, sampling_frequency=30000.0, durations=[1.0])
+    return generate_sorting(num_units=5, sampling_frequency=30000.0, durations=[DEFAULT_DURATION])
 
 
 def _check_sorting(sorting):
@@ -195,7 +217,7 @@ def _build_sorting_with_properties():
     import numpy as np
     from spikeinterface.core import generate_sorting
 
-    sorting = generate_sorting(num_units=4, sampling_frequency=30000.0, durations=[1.0])
+    sorting = generate_sorting(num_units=4, sampling_frequency=30000.0, durations=[DEFAULT_DURATION])
     sorting.set_property("quality", np.array(["good", "good", "bad", "good"]))
     sorting.annotate(experimenter="test")
     return sorting
@@ -224,19 +246,25 @@ def _check_sorting_with_properties(sorting):
         "id": "recording_with_properties",
         "build": _build_recording_with_properties,
         "check": _check_recording_with_properties,
-        "formats": ["binary", "zarr"],
+        "formats": ["binary", "zarr", "binary_parallel", "zarr_parallel"],
+    },
+    {
+        "id": "recording_with_timestamps",
+        "build": _build_recording_with_timestamps,
+        "check": _check_recording_with_timestamps,
+        "formats": ["binary", "zarr", "binary_parallel", "zarr_parallel"],
     },
     {
         "id": "recording_with_probe",
         "build": _build_recording_with_probe,
         "check": _check_recording_with_probe,
-        "formats": ["binary", "zarr"],
+        "formats": ["binary", "zarr", "binary_parallel", "zarr_parallel"],
     },
     {
         "id": "recording_with_interleaved_probes",
         "build": _build_recording_with_interleaved_probes,
         "check": _check_recording_with_interleaved_probes,
-        "formats": ["binary", "zarr"],
+        "formats": ["binary", "zarr", "binary_parallel", "zarr_parallel"],
     },
     {
         "id": "preprocessed_chain",

.github/scripts/serialization/serialize_objects.py

@@ -33,10 +33,14 @@
             obj.dump_to_pickle(dest)
         elif fmt == "binary":
             obj.save(folder=dest, format="binary", overwrite=True)
+        elif fmt == "binary_parallel":
+            obj.save(folder=dest, format="binary", overwrite=True, n_jobs=2)
         elif fmt == "numpy_folder":
             obj.save(folder=dest, format="numpy_folder", overwrite=True)
         elif fmt == "zarr":
             obj.save(folder=dest, format="zarr", overwrite=True)
+        elif fmt == "zarr_parallel":
+            obj.save(folder=dest, format="zarr", overwrite=True, n_jobs=2)
         print(f"  wrote {dest.name} ({fmt})")
 
 print(f"Fixtures written to: {out_dir.resolve()}")

.github/workflows/cross_version_serialization.yml

@@ -6,9 +6,8 @@ name: Cross-version serialization
 #
 # Delivery model: Live generation. Fixtures are regenerated from a real old install each
 # run in an isolated uv environment (nothing committed). The version list is computed
-# from PyPI (latest patch of each minor at or above MIN_VERSION_TO_TEST): on a pull
-# request only the latest released minor is tested; the full matrix runs weekly and on
-# manual dispatch.
+# from PyPI (latest patch of each minor at or above MIN_VERSION_TO_TEST), and the full
+# matrix is tested on every pull request and on manual dispatch.
 
 on:
   pull_request:
@@ -23,8 +22,6 @@ on:
       - 'src/spikeinterface/core/**'
       - '.github/scripts/serialization/**'
       - '.github/workflows/cross_version_serialization.yml'
-  schedule:
-    - cron: '0 4 * * 0'  # weekly, Sunday 04:00 UTC
   workflow_dispatch:
 
 concurrency:
@@ -33,8 +30,8 @@ concurrency:
 
 jobs:
   # Compute which released versions the matrix below tests loading from, exposed as the
-  # job output `list`. Latest patch of each minor at or above the floor on schedule /
-  # dispatch; only the latest minor on a pull request (keeps per-PR runs cheap).
+  # job output `list`: the latest patch of each minor at or above the floor, tested on
+  # every trigger.
   versions:
     name: Select versions
     runs-on: ubuntu-latest
@@ -46,7 +43,6 @@ jobs:
           python-version: '3.11'
       - id: set
         env:
-          GITHUB_EVENT_NAME: ${{ github.event_name }}
           # Support floor: oldest minor to test. Below this, installs fail on the CI
           # Python (numcodecs dependency rot in 0.100/0.101). Raise when 0.102 rots.
           MIN_VERSION_TO_TEST: "0.102"  # We will change this when making breaking changes (hopefully not too often)
@@ -67,10 +63,6 @@ jobs:
           candidates = [v for v in available if not v.is_prerelease and (v.major, v.minor) >= (floor.major, floor.minor)]
           minor_releases = [max(group) for _, group in groupby(candidates, key=lambda v: (v.major, v.minor))]
 
-          # on a pull request, test only the latest minor
-          if os.environ.get('GITHUB_EVENT_NAME') == 'pull_request':
-              minor_releases = minor_releases[-1:]
-
           print(json.dumps([str(v) for v in minor_releases]))
           ")
 

src/spikeinterface/core/base.py

@@ -481,7 +481,6 @@ def to_dict(
         include_annotations: bool = False,
         include_properties: bool = False,
         relative_to: str | Path | None = None,
-        folder_metadata=None,
         recursive: bool = False,
     ) -> dict:
         """
@@ -509,9 +508,6 @@ def to_dict(
             If provided, file and folder paths will be made relative to this path,
             enabling portability in folder formats such as the waveform extractor,
             by default None.
-        folder_metadata : str | Path | None, default: None
-            Path to a folder containing additional metadata files (e.g., probe information in BaseRecording)
-            in numpy `npy` format, by default None.
         recursive : bool, default: False
             If True, recursively apply `to_dict` to dictionaries within the kwargs, by default False.
 
@@ -532,13 +528,11 @@ def to_dict(
                 "relative_paths": <whether paths are relative>,
                 "annotations": <annotations dictionary, if `include_annotations` is True>,
                 "properties": <properties dictionary, if `include_properties` is True>,
-                "folder_metadata": <relative path to folder_metadata, if specified>
             }
 
         Notes
         -----
         - The `relative_to` argument only has an effect if `recursive` is set to True.
-        - The `folder_metadata` argument will be made relative to `relative_to` if both are specified.
         - The `version` field in the resulting dictionary reflects the version of the module
           from which the extractor class originates.
         - The full class attribute above is the full import of the class, e.g.
@@ -557,7 +551,6 @@ def to_dict(
                 include_properties=include_properties,
                 # make_paths_relative() will make the recusrivity later:
                 relative_to=None,
-                folder_metadata=folder_metadata,
                 recursive=recursive,
             )
 
@@ -604,11 +597,6 @@ def to_dict(
                 # warnings.warn("Try to BaseExtractor.to_dict() using relative_to but there is no common folder")
                 dump_dict["relative_paths"] = False
 
-        if folder_metadata is not None:
-            if relative_to is not None:
-                folder_metadata = Path(folder_metadata).resolve().absolute().relative_to(relative_to)
-            dump_dict["folder_metadata"] = str(folder_metadata)
-
         self._extra_metadata_to_dict(dump_dict)
 
         return dump_dict
@@ -635,39 +623,8 @@ def from_dict(dictionary: dict, base_folder: Path | str | None = None) -> "BaseE
             assert base_folder is not None, "When  relative_paths=True, need to provide base_folder"
             dictionary = make_paths_absolute(dictionary, base_folder)
         extractor = _load_extractor_from_dict(dictionary)
-        folder_metadata = dictionary.get("folder_metadata", None)
-        if folder_metadata is not None:
-            folder_metadata = Path(folder_metadata)
-            if dictionary.get("relative_paths", False):
-                folder_metadata = base_folder / folder_metadata
-            extractor.load_metadata_from_folder(folder_metadata)
         return extractor
 
-    def load_metadata_from_folder(self, folder_metadata):
-        # hack to load probe for recording
-        folder_metadata = Path(folder_metadata)
-
-        # load properties
-        prop_folder = folder_metadata / "properties"
-        if prop_folder.is_dir():
-            for prop_file in prop_folder.iterdir():
-                if prop_file.suffix == ".npy":
-                    values = np.load(prop_file, allow_pickle=True)
-                    key = prop_file.stem
-                    self.set_property(key, values)
-
-        self._extra_metadata_from_folder(folder_metadata)
-
-    def save_metadata_to_folder(self, folder_metadata):
-        self._extra_metadata_to_folder(folder_metadata)
-
-        # save properties
-        prop_folder = folder_metadata / "properties"
-        prop_folder.mkdir(parents=True, exist_ok=False)
-        for key in self.get_property_keys():
-            values = self.get_property(key)
-            np.save(prop_folder / (key + ".npy"), values)
-
     def clone(self) -> "BaseExtractor":
         """
         Clones an existing extractor into a new instance.
@@ -726,7 +683,7 @@ def _get_file_path(file_path: str | Path, extensions: Sequence) -> Path:
         )
         return file_path
 
-    def dump(self, file_path: str | Path, relative_to=None, folder_metadata=None) -> None:
+    def dump(self, file_path: str | Path, relative_to=None) -> None:
         """
         Dumps extractor to json or pickle
 
@@ -739,17 +696,16 @@ def dump(self, file_path: str | Path, relative_to=None, folder_metadata=None) ->
             This means that file and folder paths in extractor objects kwargs are changed to be relative rather than absolute.
         """
         if str(file_path).endswith(".json"):
-            self.dump_to_json(file_path, relative_to=relative_to, folder_metadata=folder_metadata)
+            self.dump_to_json(file_path, relative_to=relative_to)
         elif str(file_path).endswith(".pkl") or str(file_path).endswith(".pickle"):
-            self.dump_to_pickle(file_path, relative_to=relative_to, folder_metadata=folder_metadata)
+            self.dump_to_pickle(file_path, relative_to=relative_to)
         else:
             raise ValueError("Dump: file must .json or .pkl")
 
     def dump_to_json(
         self,
         file_path: str | Path | None = None,
         relative_to: str | Path | bool | None = None,
-        folder_metadata: str | Path | None = None,
     ) -> None:
         """
         Dump recording extractor to json file.
@@ -762,8 +718,6 @@ def dump_to_json(
         relative_to: str, Path, True or None
             If not None, files and folders are serialized relative to this path. If True, the relative folder is the parent folder.
             This means that file and folder paths in extractor objects kwargs are changed to be relative rather than absolute.
-        folder_metadata: str, Path, or None
-            Folder with files containing additional information (e.g. probe in BaseRecording) and properties
         """
         assert self.check_serializability("json"), "The extractor is not json serializable"
 
@@ -776,7 +730,6 @@ def dump_to_json(
             include_annotations=True,
             include_properties=False,
             relative_to=relative_to,
-            folder_metadata=folder_metadata,
             recursive=True,
         )
         file_path = self._get_file_path(file_path, [".json"])
@@ -791,7 +744,6 @@ def dump_to_pickle(
         file_path: str | Path | None = None,
         relative_to: str | Path | bool | None = None,
         include_properties: bool = True,
-        folder_metadata: str | Path | None = None,
     ):
         """
         Dump recording extractor to a pickle file.
@@ -806,8 +758,6 @@ def dump_to_pickle(
             This means that file and folder paths in extractor objects kwargs are changed to be relative rather than absolute.
         include_properties: bool
             If True, all properties are dumped
-        folder_metadata: str, Path, or None
-            Folder with files containing additional information (e.g. probe in BaseRecording) and properties.
         """
         assert self.check_serializability("pickle"), "The extractor is not serializable to file with pickle"
 
@@ -823,7 +773,6 @@ def dump_to_pickle(
         dump_dict = self.to_dict(
             include_annotations=True,
             include_properties=include_properties,
-            folder_metadata=folder_metadata,
             relative_to=relative_to,
             recursive=recursive,
         )
@@ -859,14 +808,6 @@ def _save(self, folder, **save_kwargs):
         # this is internally call by cache(...) main function
         raise NotImplementedError
 
-    def _extra_metadata_from_folder(self, folder):
-        # This implemented in BaseRecording for probe
-        pass
-
-    def _extra_metadata_to_folder(self, folder):
-        # This implemented in BaseRecording for probe
-        pass
-
     def _extra_metadata_from_dict(self, dump_dict):
         # This implemented in BaseRecording for probe
         pass
@@ -1011,9 +952,7 @@ def save_to_folder(
             warnings.warn("The extractor is not serializable to file. The provenance will not be saved.")
 
         # save data (done the subclass)
-        self.save_metadata_to_folder(folder)
         cached = self._save(folder=folder, verbose=verbose, **save_kwargs)
-        cached.load_metadata_from_folder(folder)
 
         # copy properties/
         self.copy_metadata(cached)