diff --git a/.make/CHANGES_MAKEFILE.md b/.make/CHANGES_MAKEFILE.md index bdb2c53..31b758c 100644 --- a/.make/CHANGES_MAKEFILE.md +++ b/.make/CHANGES_MAKEFILE.md @@ -6,6 +6,12 @@ ______________________________________________________________________ +## [1.3.1](https://github.com/RolnickLab/lab-advanced-template/tree/makefile-1.3.1) (2026-03-24) + +______________________________________________________________________ + +- Fix issue where the `ENV_COMMAND_TOOL` variable is not what was expected with `conda` environments + ## [1.3.0](https://github.com/RolnickLab/lab-advanced-template/tree/makefile-1.3.0) (2026-02-12) ______________________________________________________________________ diff --git a/.make/Makefile b/.make/Makefile index 41bd9ae..a04257a 100644 --- a/.make/Makefile +++ b/.make/Makefile @@ -10,7 +10,7 @@ # files to include. ######################################################################################## PROJECT_PATH := $(dir $(abspath $(firstword $(MAKEFILE_LIST)))) -MAKEFILE_VERSION := 1.3.0 +MAKEFILE_VERSION := 1.3.1 BUMP_TOOL := bump-my-version BUMP_CONFIG_FILE := $(PROJECT_PATH).bumpversion.toml SHELL := /usr/bin/env bash diff --git a/.make/base.make b/.make/base.make index b92dd71..4f29f55 100644 --- a/.make/base.make +++ b/.make/base.make @@ -18,7 +18,7 @@ PROJECT_PATH := $(dir $(abspath $(firstword $(MAKEFILE_LIST)))) MAKEFILE_NAME := $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST)) SHELL := /usr/bin/env bash BUMP_TOOL := bump-my-version -MAKEFILE_VERSION := 1.3.0 +MAKEFILE_VERSION := 1.3.1 DOCKER_COMPOSE ?= docker compose AUTO_INSTALL ?= diff --git a/.make/poetry.make b/.make/poetry.make index 71f6fe2..77f12ba 100644 --- a/.make/poetry.make +++ b/.make/poetry.make @@ -14,14 +14,17 @@ ifeq ($(DEFAULT_INSTALL_ENV),venv) POETRY_COMMAND_WITH_PROJECT_ENV := source $(VENV_ACTIVATE) && $(POETRY_COMMAND_WITH_PROJECT_ENV) else ifeq ($(DEFAULT_INSTALL_ENV),poetry) POETRY_COMMAND_WITH_PROJECT_ENV := $(POETRY_COMMAND_WITH_PROJECT_ENV) -else ifeq ($(DEFAULT_INSTALL_ENV),conda) -POETRY_COMMAND_WITH_PROJECT_ENV := $(CONDA_ENV_TOOL) run -n $(CONDA_ENVIRONMENT) $(POETRY_COMMAND_WITH_PROJECT_ENV) endif # Do not rename these unless you also rename across all other make files in .make/ ENV_COMMAND_TOOL := $(POETRY_COMMAND_WITH_PROJECT_ENV) run ENV_INSTALL_TOOL := $(POETRY_COMMAND_WITH_PROJECT_ENV) install +ifeq ($(DEFAULT_INSTALL_ENV),conda) +ENV_COMMAND_TOOL := $(CONDA_ENV_TOOL) run -n $(CONDA_ENVIRONMENT) +ENV_INSTALL_TOOL := $(ENV_COMMAND_TOOL) $(POETRY_COMMAND_WITH_PROJECT_ENV) install +endif + ## -- Poetry targets ------------------------------------------------------------------------------------------------ ## diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5d4254c..6874901 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -exclude: "^docs/|/migrations/" +exclude: ^docs/|/migrations/|Makefile* default_stages: [commit] repos: @@ -17,8 +17,18 @@ repos: - id: check-added-large-files args: ["--maxkb=5000"] + - repo: https://github.com/PyCQA/autoflake + rev: v2.3.1 + hooks: + - id: autoflake + + - repo: https://github.com/hhatto/autopep8 + rev: v2.3.2 + hooks: + - id: autopep8 + - repo: https://github.com/psf/black - rev: 23.12.1 + rev: 24.4.2 hooks: - id: black diff --git a/climateset/download/__init__.py b/climateset/download/__init__.py index e69de29..b5bc4e2 100644 --- a/climateset/download/__init__.py +++ b/climateset/download/__init__.py @@ -0,0 +1 @@ +from .downloader import download_from_config_file # noqa F401 diff --git a/climateset/download/abstract_downloader.py b/climateset/download/abstract_downloader.py new file mode 100644 index 0000000..ebf69cd --- /dev/null +++ b/climateset/download/abstract_downloader.py @@ -0,0 +1,7 @@ +from abc import ABC, abstractmethod + + +class AbstractDownloader(ABC): + @abstractmethod + def download(self): + pass diff --git a/climateset/download/client.py b/climateset/download/client.py new file mode 100644 index 0000000..e1190b8 --- /dev/null +++ b/climateset/download/client.py @@ -0,0 +1,164 @@ +from typing import Any, List, Optional + +from pyesgf.search import SearchConnection +from pyesgf.search.context import DatasetSearchContext + +from climateset.download.constants import NODE_LINK_URLS +from climateset.download.constraints import BaseSearchConstraints +from climateset.utils import create_logger + +LOGGER = create_logger(__name__) + + +class SearchClient: + """ + Client for performing searches against ESGF nodes with failover support. + + Acts as a factory for SearchSession objects. + """ + + def __init__(self, node_urls: List[str] | None = None, distrib: bool = True): + self.node_urls = node_urls if node_urls is not None else NODE_LINK_URLS + self.distrib = distrib + self.logger = LOGGER + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + pass + + def new_session(self) -> "SearchSession": + """Start a new search session.""" + return SearchSession(self.node_urls, self.distrib, self.logger) + + +class SearchSession: + """ + Stateful session for building iterative search queries. + + Handles node failover by replaying applied constraints. + """ + + def __init__(self, node_urls: List[str], distrib: bool, logger): + self.node_urls = node_urls + self.distrib = distrib + self.logger = logger + + # History of constraints applied to this session + self._constraints_history: List[BaseSearchConstraints] = [] + + # State relative to correct active connection + self._current_node_index = 0 + self._connection: Optional[SearchConnection] = None + self._context: Optional[DatasetSearchContext] = None + + # Initialize connection logic + self._ensure_connection() + + def _ensure_connection(self): + """ + Ensures a valid connection/context exists. + + If not, attempts to connect to available nodes. Once connected, replays history. + """ + if self._context is not None: + return + + while self._current_node_index < len(self.node_urls): + url = self.node_urls[self._current_node_index] + try: + self.logger.info(f"Connecting to ESGF node: {url}") + self._connection = SearchConnection(url=url, distrib=self.distrib) + + # Create fresh context + ctx = self._connection.new_context() + + # Replay constraints + for constraints in self._constraints_history: + params = constraints.to_esgf_params() + if params: + ctx = ctx.constrain(**params) + + self._context = ctx + return + except Exception as e: # pylint: disable=broad-exception-caught + self.logger.warning(f"Failed to connect to {url}: {e}") + self._current_node_index += 1 + self._connection = None + self._context = None + + raise ConnectionError(f"Could not connect to any ESGF node. Tried: {self.node_urls}") + + def _rotate_node(self): + """Force rotation to the next node (e.g. after a search failure).""" + self.logger.info("Rotating to next ESGF node...") + self._current_node_index += 1 + self._connection = None + self._context = None + self._ensure_connection() + + def constrain(self, constraints: BaseSearchConstraints) -> "SearchSession": + """Apply a new set of constraints to the session.""" + self._constraints_history.append(constraints) + + # If we have an active context, apply immediately. + # If not (e.g. all nodes down), _ensure_connection will handle it next time. + if self._context: + params = constraints.to_esgf_params() + if params: + try: + self._context = self._context.constrain(**params) + except Exception as e: # pylint: disable=broad-exception-caught + self.logger.warning(f"Error applying constraints on current node: {e}") + self._rotate_node() + else: + # Try to establish connection if we were disconnected + try: + self._ensure_connection() + except ConnectionError: + pass # Delay error until actual search/facet request + + return self + + def get_available_facets(self, facet_name: str) -> List[str]: + """ + Get available counts/values for a specific facet. + + Retries on other nodes if current fails. + """ + max_attempts = len(self.node_urls) + attempts = 0 + + while attempts < max_attempts: + try: + self._ensure_connection() + if facet_name in self._context.facet_counts: + return list(self._context.facet_counts[facet_name].keys()) + return [] + except Exception as e: # pylint: disable=broad-exception-caught + self.logger.warning(f"Error fetching facets from {self.node_urls[self._current_node_index]}: {e}") + self._rotate_node() + attempts += 1 + + return [] + + def search(self) -> List[Any]: + """ + Execute the search using applied constraints. + + Retries on other nodes if current fails. + """ + max_attempts = len(self.node_urls) + attempts = 0 + + while attempts < max_attempts: + try: + self._ensure_connection() + return self._context.search() + except Exception as e: # pylint: disable=broad-exception-caught + self.logger.warning(f"Search failed on {self.node_urls[self._current_node_index]}: {e}") + self._rotate_node() + attempts += 1 + + raise ConnectionError("Search failed on all available nodes.") diff --git a/climateset/download/cmip6_downloader.py b/climateset/download/cmip6_downloader.py new file mode 100644 index 0000000..0873c17 --- /dev/null +++ b/climateset/download/cmip6_downloader.py @@ -0,0 +1,169 @@ +from pathlib import Path + +from esgpull import Esgpull + +from climateset.download.abstract_downloader import AbstractDownloader +from climateset.download.constants.esgf import CMIP6 +from climateset.download.downloader_config import ( + CMIP6DownloaderConfig, + create_cmip6_downloader_config_from_file, +) +from climateset.download.esgpull_utils import ( + esgpull_search_and_download_esgf_model_single_var, + isolated_esgpull_context, +) +from climateset.download.utils import search_and_download_esgf_model_single_var +from climateset.utils import create_logger + +LOGGER = create_logger(__name__) + + +class CMIP6Downloader(AbstractDownloader): + def __init__(self, config: CMIP6DownloaderConfig): + self.logger = LOGGER + self.config = config + + def download(self): + """ + Function handling the download of all variables that are associated with a model's output. + + Searches for all files associated with the respected variables and experiment that the downloader + was initialized with. + + A search connection is established and the search is iteratively constraint to meet all specifications. + Data is downloaded and stored in a separate file for each year. The default format is netCDF4. + + Resulting hierarchy: + + `CMIPx/model_id/ensemble_member/experiment/variable/nominal_resolution/frequency/year.nc` + + If the constraints cannot be met, per default behaviour for the downloader to select first other + available value + """ + for model in self.config.models: + self.logger.info(f"Downloading data for model: [{model}]") + for variable in self.config.variables: + self.logger.info(f"Downloading data for variable: [{variable}]") + for experiment in self.config.experiments: + self.logger.info(f"Downloading data for experiment: [{experiment}]") + self.download_from_model_single_var( + model=model, project=self.config.project, variable=variable, experiment=experiment + ) + + def download_from_model_single_var( + self, + model: str, + variable: str, + experiment: str, + project: str = CMIP6, + default_frequency: str = "mon", + preferred_version: str = "latest", + default_grid_label: str = "gn", + ): + """ + Function handling the download of a single variable-experiment pair that is associated with a model's output + (CMIP data). + + Args: + model (str): The model ID + variable: variable ID + experiment: experiment ID + project: umbrella project id e.g. CMIPx + default_frequency: default frequency to download + preferred_version: data upload version, if 'latest', the newest version will get selected always + default_grid_label: default gridding method in which the data is provided + """ + results_list = search_and_download_esgf_model_single_var( + model=model, + variable=variable, + experiment=experiment, + project=project, + default_frequency=default_frequency, + default_grid_label=default_grid_label, + preferred_version=preferred_version, + ensemble_members=self.config.ensemble_members, + max_ensemble_members=self.config.max_ensemble_members, + base_path=self.config.data_dir, + ) + self.logger.info(f"Download results: {results_list}") + + +class CMIP6DownloaderV2(AbstractDownloader): + def __init__(self, config: CMIP6DownloaderConfig, distrib: bool = True): + self.logger = LOGGER + self.distrib = distrib + self.config = config + + def download(self): + """ + Function handling the download of all variables that are associated with a model's output. + + Searches for all files associated with the respected variables and experiment that the downloader + was initialized with. + + A search connection is established and the search is iteratively constraint to meet all specifications. + Data is downloaded and stored in a separate file for each year. The default format is netCDF4. + + Resulting hierarchy: + + `CMIPx/model_id/ensemble_member/experiment/variable/nominal_resolution/frequency/year.nc` + + If the constraints cannot be met, per default behaviour for the downloader to select first other + available value + """ + with isolated_esgpull_context(self.config.data_dir) as esg: + for model in self.config.models: + self.logger.info(f"Downloading data for model: [{model}]") + for variable in self.config.variables: + self.logger.info(f"Downloading data for variable: [{variable}]") + for experiment in self.config.experiments: + self.logger.info(f"Downloading data for experiment: [{experiment}]") + self.download_from_model_single_var( + esg=esg, model=model, project=self.config.project, variable=variable, experiment=experiment + ) + + def download_from_model_single_var( + self, + esg: Esgpull, + model: str, + variable: str, + experiment: str, + project: str = CMIP6, + default_frequency: str = "mon", + preferred_version: str = "latest", + default_grid_label: str = "gn", + ): + """ + Function handling the download of a single variable-experiment pair that is associated with a model's output + (CMIP data). + + Args: + model (str): The model ID + variable: variable ID + experiment: experiment ID + project: umbrella project id e.g. CMIPx + default_frequency: default frequency to download + preferred_version: data upload version, if 'latest', the newest version will get selected always + default_grid_label: default gridding method in which the data is provided + """ + results_list = esgpull_search_and_download_esgf_model_single_var( + esg=esg, + model=model, + variable=variable, + experiment=experiment, + project=project, + default_frequency=default_frequency, + default_grid_label=default_grid_label, + preferred_version=preferred_version, + ensemble_members=self.config.ensemble_members, + max_ensemble_members=self.config.max_ensemble_members, + data_dir=Path(self.config.data_dir), + distrib=self.distrib, + ) + self.logger.info(f"Download results: {results_list}") + + +def cmip6_download_from_config(config): + config_object = create_cmip6_downloader_config_from_file(config) + downloader = CMIP6DownloaderV2(config=config_object) + downloader.download() diff --git a/climateset/download/constants/__init__.py b/climateset/download/constants/__init__.py index e69de29..c91abec 100644 --- a/climateset/download/constants/__init__.py +++ b/climateset/download/constants/__init__.py @@ -0,0 +1,10 @@ +NODE_LINK_URLS = [ + "https://esgf-node.llnl.gov/esg-search", + "https://esgf.ceda.ac.uk/esg-search", + "https://esgf-data.dkrz.de/esg-search", + "https://esgf-node.ipsl.upmc.fr/esg-search", + "https://esg-dn1.nsc.liu.se/esg-search", + "https://esgf.nci.org.au/esg-search", + "https://esgf.nccs.nasa.gov/esg-search", + "https://esgdata.gfdl.noaa.gov/esg-search", +] diff --git a/climateset/download/constants/cmip6.py b/climateset/download/constants/cmip6.py new file mode 100644 index 0000000..656a29f --- /dev/null +++ b/climateset/download/constants/cmip6.py @@ -0,0 +1,34 @@ +# TODO remove raw variables from here +# pylint: disable=C0103 +from dataclasses import dataclass +from typing import Final + +from climateset.utils import get_yaml_config + + +@dataclass(frozen=True) +class Cmip6Constants: + """ + Dataclass to represent CMIP6 constants that are used by the download module. + + Attributes: + NODE_LINK : Where the data can be accessed + MODEL_SOURCES : Identifiers for supported climate models + VAR_SOURCE_LOOKUP : model and raw variables + SUPPORTED_EXPERIMENTS : experiments of climate models (runs) that are supported + """ + + NODE_LINK: Final[str] + MODEL_SOURCES: Final[tuple[str, ...]] + VAR_SOURCE_LOOKUP: Final[tuple[str, ...]] + SUPPORTED_EXPERIMENTS: Final[tuple[str, ...]] + + +_data = get_yaml_config("downloader/constants/cmip6.yaml") + +CMIP6_CONSTANTS = Cmip6Constants( + NODE_LINK=_data["node_link"], + MODEL_SOURCES=tuple(_data["model_sources"]), + SUPPORTED_EXPERIMENTS=tuple(_data["supported_experiments"]), + VAR_SOURCE_LOOKUP=tuple(_data["var_source_lookup"]), +) diff --git a/climateset/download/constants/cmip6plus.py b/climateset/download/constants/cmip6plus.py new file mode 100644 index 0000000..d725342 --- /dev/null +++ b/climateset/download/constants/cmip6plus.py @@ -0,0 +1,35 @@ +# pylint: disable=C0103 +from dataclasses import dataclass +from typing import Final + +from climateset.utils import get_yaml_config + +# TODO remove raw variables from here + + +@dataclass(frozen=True) +class Cmip6plusConstants: + """ + Dataclass to represent CMIP6PLUS constants that are used by the download module. + + Attributes: + NODE_LINK : Where the data can be accessed + MODEL_SOURCES : Identifiers for supported climate models + VAR_SOURCE_LOOKUP : model and raw variables + SUPPORTED_EXPERIMENTS : experiments of climate models (runs) that are supported + """ + + NODE_LINK: Final[str] + MODEL_SOURCES: Final[tuple[str, ...]] + VAR_SOURCE_LOOKUP: Final[tuple[str, ...]] + SUPPORTED_EXPERIMENTS: Final[tuple[str, ...]] + + +_data = get_yaml_config("downloader/constants/cmip6plus.yaml") + +CMIP6PLUS_CONSTANTS = Cmip6plusConstants( + NODE_LINK=_data["node_link"], + MODEL_SOURCES=tuple(_data["model_sources"]), + SUPPORTED_EXPERIMENTS=tuple(_data["supported_experiments"]), + VAR_SOURCE_LOOKUP=tuple(_data["var_source_lookup"]), +) diff --git a/climateset/download/constants/data_constants.py b/climateset/download/constants/data_constants.py deleted file mode 100644 index ff89eb1..0000000 --- a/climateset/download/constants/data_constants.py +++ /dev/null @@ -1,18 +0,0 @@ -EMISSIONS_ENDINGS = ["_em_openburning", "_em_anthro", "_em_AIR_anthro"] - -META_ENDINGS_PRC = [ - "_percentage_AGRI", - "_percentage_BORF", - "_percentage_DEFO", - "_percentage_PEAT", - "_percentage_SAVA", - "_percentage_TEMF", -] -META_ENDINGS_SHAR = ["_openburning_share"] - -LON_LAT_TO_GRID_SIZE = { - (720, 360): "25_km", - (360, 720): "25_km", - (96, 144): "250_km", - (144, 96): "250_km", -} diff --git a/climateset/download/constants/esgf.py b/climateset/download/constants/esgf.py new file mode 100644 index 0000000..00ebc92 --- /dev/null +++ b/climateset/download/constants/esgf.py @@ -0,0 +1,23 @@ +from .cmip6 import CMIP6_CONSTANTS +from .cmip6plus import CMIP6PLUS_CONSTANTS +from .input4mips import INPUT4MIPS_CONSTANTS + +CMIP6 = "CMIP6" +CMIP6PLUS = "CMIP6Plus" +INPUT4MIPS = "input4MIPs" + +ESGF_PROJECTS = frozenset([CMIP6, CMIP6PLUS, INPUT4MIPS]) + +# constant classes for esgf projects implemented here +# add your own esgf project for downloading to download/constants/ and add the constant class to the dict and lists here +ESGF_PROJECTS_CONSTANTS = { + CMIP6: CMIP6_CONSTANTS, + CMIP6PLUS: CMIP6PLUS_CONSTANTS, + INPUT4MIPS: INPUT4MIPS_CONSTANTS, +} + +# datasets that provide inputs to climate models +ESGF_RAW_INPUT_LIST = [INPUT4MIPS] + +# datasets that provide outputs from climate models +ESGF_MODEL_OUTPUT_LIST = [CMIP6, CMIP6PLUS] diff --git a/climateset/download/constants/esgf_server.py b/climateset/download/constants/esgf_server.py deleted file mode 100644 index cf5bd17..0000000 --- a/climateset/download/constants/esgf_server.py +++ /dev/null @@ -1,1359 +0,0 @@ -# Supported Model sources -NODE_LINK = "http://esgf-node.llnl.gov/esg-search/" -MODEL_SOURCES = { - "ACCESS-CM2": { - "node_link": NODE_LINK, - "center": "CSIRO-ARCCSS", - }, - "ACCESS-ESM1-5": { - "node_link": NODE_LINK, - "center": "CSIRO", - }, - "AWI-CM-1-1-MR": { - "node_link": NODE_LINK, - "center": "AWI", - }, - "BCC-CSM2-MR": { - "node_link": NODE_LINK, - "center": "BCC", - }, - "CAMS-CSM1-0": { - "node_link": NODE_LINK, - "center": "CAMS", - }, - "CAS-ESM2-0": { - "node_link": NODE_LINK, - "center": "CAS", - }, - "CESM2": {"node_link": NODE_LINK, "center": "NCAR"}, - "CESM2-WACCM": { - "node_link": NODE_LINK, - "center": "NCAR", - }, - "CMCC-CM2-SR5": { - "node_link": NODE_LINK, - "center": "NCAR", - }, - "CMCC-ESM2": { - "node_link": NODE_LINK, - "center": "CMCC", - }, - "CNRM-CM6-1": { - "node_link": NODE_LINK, - "center": "CNRM-CERFACS", - }, - "CNRM-CM6-1-HR": { - "node_link": NODE_LINK, - "center": "CNRM-CERFACS", - }, - "CNRM-ESM2-1": { - "node_link": NODE_LINK, - "center": "CNRM-CERFACS", - }, - "EC-Earth3": { - "node_link": NODE_LINK, - "center": "EC-Earth-Consortium", - }, - "EC-Earth3-Veg": { - "node_link": NODE_LINK, - "center": "EC-Earth-Consortium", - }, - "EC-Earth3-Veg-LR": { - "node_link": NODE_LINK, - "center": "EC-Earth-Consortium", - }, - "FGOALS-f3-L": { - "node_link": NODE_LINK, - "center": "CAS", - }, - "FGOALS-g3": { - "node_link": NODE_LINK, - "center": "CAS", - }, - "GFDL-ESM4": { - "node_link": NODE_LINK, - "center": "NOAA-GFDL", - }, - "GISS-E2-1-G": { - "node_link": NODE_LINK, - "center": "NASA-GISS", - }, - "GISS-E2-1-H": { - "node_link": NODE_LINK, - "center": "NASA-GISS", - }, - "GISS-E2-2-G": { - "node_link": NODE_LINK, - "center": "NASA-GISS", - }, - "IITM-ESM": { - "node_link": NODE_LINK, - "center": "CCCR-IITM", - }, - "INM-CM4-8": { - "node_link": NODE_LINK, - "center": "INM", - }, - "INM-CM5-0": { - "node_link": NODE_LINK, - "center": "INM", - }, - "IPSL-CM6A-LR": { - "node_link": NODE_LINK, - "center": "IPSL", - }, - "KACE-1-0-G": { - "node_link": NODE_LINK, - "center": "NIMS-KMA ", - }, - "MCM-UA-1-0": { - "node_link": NODE_LINK, - "center": "UA", - }, - "MIROC6": {"node_link": NODE_LINK, "center": "MIROC"}, - # there are several centers for the MPI models - consider choosing another one if needed - "MPI-ESM1-2-HR": { - "node_link": NODE_LINK, - "center": "MPI-M", - }, - "MPI-ESM1-2-LR": { - "node_link": NODE_LINK, - "center": "MPI-M", - }, - "MRI-ESM2-0": { - "node_link": NODE_LINK, - "center": "MRI", - }, - "NorESM2-LM": { - "node_link": "https://esgf-data.dkrz.de/esg-search", - "center": "NCC", - }, - "NorESM2-MM": { - "node_link": "https://esgf-data.dkrz.de/esg-search", - "center": "NCC", - }, - "TaiESM1": { - "node_link": NODE_LINK, - "center": "AS-RCEC", - }, - # there are several centers for the UKESM models - consider choosing another one if needed - "UKESM1-0-LL": { - "node_link": NODE_LINK, - "center": "MOHC", - }, - # "NorESM2-LM": {"node_link": "https://esgf-data.dkrz.de/esg-search", "center": "NCC"}, - # "CanESM5" : {"node_link": NODE_LINK, "center": "CCCma"} -} - -VAR_SOURCE_LOOKUP = { - "model": [ - "ztp", - "zsatcalc", - "zsatarag", - "zostoga", - "zossq", - "zos", - "zoocos", - "zooc", - "zo2min", - "zhalfo", - "zg500", - "zg1000", - "zg100", - "zg10", - "zg", - "zfullo", - "wtd", - "wo", - "wmo", - "wfonocorr", - "wfo", - "wetss", - "wetso4", - "wetso2", - "wetlandFrac", - "wetlandCH4", - "wetbc", - "wap500", - "wap", - "vsf", - "volo", - "volcello", - "vo", - "vmo", - "vegHeight", - "va", - "uo", - "umo", - "ua", - "tslsi", - "tsl", - "ts", - "tran", - "tossq", - "tosga", - "tos", - "tob", - "thkcello", - "thetaot700", - "thetaot300", - "thetaot2000", - "thetaot", - "thetaoga", - "thetao", - "tgs", - "tcs", - "tauvo", - "tauv", - "tauuo", - "tauu", - "tasmin", - "tasmax", - "tas", - "talkos", - "talknat", - "talk", - "ta850", - "ta700", - "ta500", - "ta", - "t20d", - "spco2", - "sossq", - "sosga", - "sos", - "sootsn", - "somint", - "soga", - "sob", - "so2", - "so", - "snw", - "sndmasswindrif", - "sndmasssnf", - "sndmasssi", - "sndmassmelt", - "snd", - "snc", - "sivols", - "sivoln", - "sivol", - "siv", - "siu", - "sitimefrac", - "sithick", - "sitemptop", - "sitempsnic", - "sitempbot", - "sistryubot", - "sistrydtop", - "sistrxubot", - "sistrxdtop", - "sispeed", - "sisnthick", - "sisnmass", - "sisnhc", - "sisnconc", - "sirdgthick", - "sirdgconc", - "sipr", - "sios", - "simpconc", - "simass", - "siitdthick", - "siitdsnthick", - "siitdsnconc", - "siitdconc", - "sihc", - "siforcetilty", - "siforcetiltx", - "siforceintstry", - "siforceintstrx", - "siforcecorioly", - "siforcecoriolx", - "siflswutop", - "siflswdtop", - "siflswdbot", - "siflsensupbot", - "siflsenstop", - "sifllwutop", - "sifllwdtop", - "sifllatstop", - "siflfwdrain", - "siflfwbot", - "siflcondtop", - "siflcondbot", - "sifb", - "siextents", - "siextentn", - "sidmasstrany", - "sidmasstranx", - "sidmassth", - "sidmasssi", - "sidmassmelttop", - "sidmassmeltbot", - "sidmasslat", - "sidmassgrowthwat", - "sidmassgrowthbot", - "sidmassevapsubl", - "sidmassdyn", - "sidivvel", - "sidconcth", - "sidconcdyn", - "siconc", - "sicompstren", - "siarean", - "siage", - "si", - "sftof", - "sftlf", - "sftgif", - "sfdsi", - "sfcWind", - "sf6", - "rtmt", - "rsutcsaf", - "rsutcs", - "rsutaf", - "rsut", - "rsuscs", - "rsus", - "rsntds", - "rsdt", - "rsdsdiff", - "rsdscs", - "rsds", - "rlutcsaf", - "rlutcs", - "rlutaf", - "rlut", - "rlus", - "rldscs", - "rlds", - "rh", - "reffclwtop", - "ra", - "rMaint", - "rGrowth", - "qgwr", - "pso", - "psl", - "ps", - "prw", - "prveg", - "prsn", - "prra", - "prc", - "pr", - "ppos", - "pp", - "popos", - "pop", - "ponos", - "pon", - "po4os", - "po4", - "phynos", - "phyn", - "phyfeos", - "phyfe", - "phyc", - "phos", - "phnat", - "phalf", - "ph", - "pfull", - "pctisccp", - "pbo", - "orog", - "opottempmint", - "oh", - "od870aer", - "od550ss", - "od550so4", - "od550oa", - "od550lt1aer", - "od550dust", - "od550csaer", - "od550bc", - "od550aerh2o", - "od550aer", - "od440aer", - "obvfsq", - "o3", - "o2satos", - "o2sat", - "o2os", - "o2min", - "o2", - "nppWood", - "nppRoot", - "nppLeaf", - "npp", - "no3os", - "no3", - "nep", - "nbp", - "nVeg", - "nStem", - "nSoil", - "nRoot", - "nMineralNO3", - "nMineralNH4", - "nMineral", - "nLitter", - "nLeaf", - "nLand", - "n2oglobal", - "msftmzmpa", - "msftmz", - "msftmrhompa", - "msftmrho", - "msftbarot", - "mrtws", - "mrsos", - "mrsol", - "mrso", - "mrsll", - "mrsfl", - "mrros", - "mrrob", - "mrro", - "mrlso", - "mrfso", - "mmrss", - "mmrsoa", - "mmrso4", - "mmrpm2p5", - "mmrpm1", - "mmroa", - "mmrdust", - "mmrbc", - "mmraerh2o", - "mlotstsq", - "mlotstmin", - "mlotstmax", - "mlotst", - "mfo", - "masso", - "masscello", - "lwsnl", - "lwp", - "loadss", - "loaddust", - "lai", - "isop", - "intpp", - "intpoc", - "intpn2", - "intdoc", - "intdic", - "huss", - "hus", - "hurs", - "hur", - "hfy", - "hfx", - "hfss", - "hfls", - "hfds", - "hfbasinpmdiff", - "hfbasinpmadv", - "hfbasinpadv", - "hfbasin", - "gpp", - "fsitherm", - "froc", - "frn", - "friver", - "fric", - "frfe", - "ficeberg", - "fgo2", - "fgdms", - "fgco2nat", - "fgco2", - "fVegLitterSenescence", - "fVegLitterMortality", - "fVegLitter", - "fNup", - "fNnetmin", - "fNloss", - "fNleach", - "fNgasNonFire", - "fNgasFire", - "fNgas", - "fNfert", - "fNdep", - "fNProduct", - "fNOx", - "fN2O", - "fLuc", - "fLitterFire", - "fHarvestToProduct", - "fHarvest", - "fFireNat", - "fFire", - "fDeforestToProduct", - "fBNF", - "evspsblveg", - "evspsblsoi", - "evspsbl", - "evs", - "esn", - "es", - "epsi100", - "epp100", - "epn100", - "epfe100", - "epcalc100", - "epc100", - "emivoc", - "emiss", - "emiso4", - "emiso2", - "emioa", - "emiisop", - "emidust", - "emidms", - "emibvoc", - "emibc", - "ec", - "dryso4", - "dryso2", - "drybc", - "dpco2", - "dmsos", - "dms", - "dmlt", - "dissocos", - "dissoc", - "dissicos", - "dissicnat", - "dissic", - "dfeos", - "dfe", - "detocos", - "detoc", - "deptho", - "cod", - "co3satcalcos", - "co3satcalc", - "co3sataragos", - "co3satarag", - "co3os", - "co3nat", - "co3", - "co2mass", - "co2", - "clwvi", - "clwmodis", - "clw", - "cltmodis", - "cltisccp", - "cltcalipso", - "clt", - "clmcalipso", - "cllcalipso", - "clivi", - "climodis", - "cli", - "clhcalipso", - "cl", - "chlos", - "chl", - "chepsoa", - "ch4global", - "cfc12global", - "cfc12", - "cfc11global", - "cfc11", - "cdnc", - "cct", - "ccn", - "ccb", - "calcos", - "calc", - "cWood", - "cVeg", - "cStem", - "cSoilSlow", - "cSoilMedium", - "cSoilFast", - "cSoilAbove1m", - "cSoil", - "cRoot", - "cMisc", - "cLitter", - "cLeaf", - "cLand", - "cCwd", - "bsios", - "bsi", - "bldep", - "bfeos", - "bfe", - "basin", - "ares", - "areacello", - "areacella", - "albisccp", - "airmass", - "agessc", - "abs550aer", - ], - "raw": [ - "years", - "year_weight", - "year_fr", - "wlenbinsize", - "wlen_bnds", - "wlen", - "wfo", - "wetnoy", - "wetnhx", - "water_vapor", - "vos", - "volume_density", - "vo", - "vmro3", - "vas", - "urban_to_secdn", - "urban_to_secdf", - "urban_to_range", - "urban_to_pastr", - "urban_to_c4per", - "urban_to_c4ann", - "urban_to_c3per", - "urban_to_c3nfx", - "urban_to_c3ann", - "urban", - "uos", - "uo", - "uas", - "tsi", - "ts", - "total_solar_irradiance", - "tosbcs", - "tos", - "thetao", - "theta", - "temp_level", - "temp_layer", - "tauv", - "tauu", - "tas", - "surface_temperature", - "surface_emissivity", - "surface_albedo", - "sst", - "ssn", - "ssi", - "ssa550", - "sos", - "solar_zenith_angle", - "so2f2_SH", - "so2f2_NH", - "so2f2_GM", - "so", - "sithick", - "sig_lon_W", - "sig_lon_E", - "sig_lat_W", - "sig_lat_E", - "siconcbcs", - "siconca", - "siconc", - "sftof", - "sftflf", - "sf6_SH", - "sf6_NH", - "sf6_GM", - "secyf_harv", - "secyf_bioh", - "secnf_harv", - "secnf_bioh", - "secmf_harv", - "secmf_bioh", - "secmb", - "secma", - "secdn_to_urban", - "secdn_to_secdf", - "secdn_to_range", - "secdn_to_pastr", - "secdn_to_c4per", - "secdn_to_c4ann", - "secdn_to_c3per", - "secdn_to_c3nfx", - "secdn_to_c3ann", - "secdn", - "secdf_to_urban", - "secdf_to_secdn", - "secdf_to_range", - "secdf_to_pastr", - "secdf_to_c4per", - "secdf_to_c4ann", - "secdf_to_c3per", - "secdf_to_c3nfx", - "secdf_to_c3ann", - "secdf", - "scph", - "scnum", - "sad_of_big_particles", - "sad", - "rsds", - "rndwd", - "rmean", - "rlds", - "range_to_urban", - "range_to_secdn", - "range_to_secdf", - "range_to_pastr", - "range_to_c4per", - "range_to_c4ann", - "range_to_c3per", - "range_to_c3nfx", - "range_to_c3ann", - "range", - "ptbio", - "psl", - "prsn", - "prra", - "profile_weight", - "primn_to_urban", - "primn_to_secdf", - "primn_to_range", - "primn_to_pastr", - "primn_to_c4per", - "primn_to_c4ann", - "primn_to_c3per", - "primn_to_c3nfx", - "primn_to_c3ann", - "primn_harv", - "primn_bioh", - "primn", - "primf_to_urban", - "primf_to_secdn", - "primf_to_range", - "primf_to_pastr", - "primf_to_c4per", - "primf_to_c4ann", - "primf_to_c3per", - "primf_to_c3nfx", - "primf_to_c3ann", - "primf_harv", - "primf_bioh", - "primf", - "pressure", - "pres_level", - "pres_layer", - "pr", - "plume_number", - "plume_lon", - "plume_lat", - "plume_feature", - "percentage_TEMF", - "percentage_SAVA", - "percentage_PEAT", - "percentage_DEFO", - "percentage_BORF", - "percentage_AGRI", - "pastr_to_urban", - "pastr_to_secdn", - "pastr_to_secdf", - "pastr_to_range", - "pastr_to_c4per", - "pastr_to_c4ann", - "pastr_to_c3per", - "pastr_to_c3nfx", - "pastr_to_c3ann", - "pastr", - "ozone", - "oxygen_GM", - "nitrous_oxide_SH", - "nitrous_oxide_NH", - "nitrous_oxide_GM", - "nitrogen_GM", - "nf3_SH", - "nf3_NH", - "nf3_GM", - "mrro", - "month", - "mole_fraction_of_so2f2_in_air", - "mole_fraction_of_sf6_in_air", - "mole_fraction_of_nitrous_oxide_in_air", - "mole_fraction_of_nf3_in_air", - "mole_fraction_of_methyl_chloride_in_air", - "mole_fraction_of_methyl_bromide_in_air", - "mole_fraction_of_methane_in_air", - "mole_fraction_of_hfc4310mee_in_air", - "mole_fraction_of_hfc365mfc_in_air", - "mole_fraction_of_hfc32_in_air", - "mole_fraction_of_hfc245fa_in_air", - "mole_fraction_of_hfc23_in_air", - "mole_fraction_of_hfc236fa_in_air", - "mole_fraction_of_hfc227ea_in_air", - "mole_fraction_of_hfc152a_in_air", - "mole_fraction_of_hfc143a_in_air", - "mole_fraction_of_hfc134aeq_in_air", - "mole_fraction_of_hfc134a_in_air", - "mole_fraction_of_hfc125_in_air", - "mole_fraction_of_hcfc22_in_air", - "mole_fraction_of_hcfc142b_in_air", - "mole_fraction_of_hcfc141b_in_air", - "mole_fraction_of_halon2402_in_air", - "mole_fraction_of_halon1301_in_air", - "mole_fraction_of_halon1211_in_air", - "mole_fraction_of_co2eq_in_air", - "mole_fraction_of_chcl3_in_air", - "mole_fraction_of_ch3ccl3_in_air", - "mole_fraction_of_ch2cl2_in_air", - "mole_fraction_of_cfc12eq_in_air", - "mole_fraction_of_cfc12_in_air", - "mole_fraction_of_cfc11eq_in_air", - "mole_fraction_of_cfc11_in_air", - "mole_fraction_of_cfc115_in_air", - "mole_fraction_of_cfc114_in_air", - "mole_fraction_of_cfc113_in_air", - "mole_fraction_of_cf4_in_air", - "mole_fraction_of_carbon_tetrachloride_in_air", - "mole_fraction_of_carbon_dioxide_in_air", - "mole_fraction_of_c_c4f8_in_air", - "mole_fraction_of_c8f18_in_air", - "mole_fraction_of_c7f16_in_air", - "mole_fraction_of_c6f14_in_air", - "mole_fraction_of_c5f12_in_air", - "mole_fraction_of_c4f10_in_air", - "mole_fraction_of_c3f8_in_air", - "mole_fraction_of_c2f6_in_air", - "methyl_chloride_SH", - "methyl_chloride_NH", - "methyl_chloride_GM", - "methyl_bromide_SH", - "methyl_bromide_NH", - "methyl_bromide_GM", - "methane_SH", - "methane_NH", - "methane_GM", - "mask4resto_ipv_Nextrop", - "mask4resto_ipv", - "mask4resto_amv_trop", - "mask4resto_amv_extrop", - "mask4resto_amv", - "lon_bounds", - "licalvf", - "lat_bounds", - "kp", - "is_biomass", - "irrig_c4per", - "irrig_c4ann", - "irrig_c3per", - "irrig_c3nfx", - "irrig_c3ann", - "ipv_index", - "iprp", - "iprm", - "iprg", - "icwtr", - "huss", - "hfds", - "hfc4310mee_SH", - "hfc4310mee_NH", - "hfc4310mee_GM", - "hfc365mfc_SH", - "hfc365mfc_NH", - "hfc365mfc_GM", - "hfc32_SH", - "hfc32_NH", - "hfc32_GM", - "hfc245fa_SH", - "hfc245fa_NH", - "hfc245fa_GM", - "hfc23_SH", - "hfc23_NH", - "hfc23_GM", - "hfc236fa_SH", - "hfc236fa_NH", - "hfc236fa_GM", - "hfc227ea_SH", - "hfc227ea_NH", - "hfc227ea_GM", - "hfc152a_SH", - "hfc152a_NH", - "hfc152a_GM", - "hfc143a_SH", - "hfc143a_NH", - "hfc143a_GM", - "hfc134aeq_SH", - "hfc134aeq_NH", - "hfc134aeq_GM", - "hfc134a_SH", - "hfc134a_NH", - "hfc134a_GM", - "hfc125_SH", - "hfc125_NH", - "hfc125_GM", - "hcfc22_SH", - "hcfc22_NH", - "hcfc22_GM", - "hcfc142b_SH", - "hcfc142b_NH", - "hcfc142b_GM", - "hcfc141b_SH", - "hcfc141b_NH", - "hcfc141b_GM", - "halon2402_SH", - "halon2402_NH", - "halon2402_GM", - "halon1301_SH", - "halon1301_NH", - "halon1301_GM", - "halon1211_SH", - "halon1211_NH", - "halon1211_GM", - "gzdis", - "gridcellarea", - "gpbio", - "gldis", - "glat_bnds", - "glat", - "fulwd", - "ftr_weight", - "fstnf", - "friver", - "flood", - "fill_flag", - "fharv_c4per", - "fharv_c3per", - "fertl_c4per", - "fertl_c4ann", - "fertl_c3per", - "fertl_c3nfx", - "fertl_c3ann", - "f107", - "expt_label", - "evspsbl", - "drynoy", - "drynhx", - "delta13co2_in_air", - "datasource", - "crpbf_total", - "crpbf_c4per", - "crpbf_c4ann", - "crpbf_c3per", - "crpbf_c3nfx", - "crpbf_c3ann", - "combf", - "co2eq_SH", - "co2eq_NH", - "co2eq_GM", - "chcl3_SH", - "chcl3_NH", - "chcl3_GM", - "ch3ccl3_SH", - "ch3ccl3_NH", - "ch3ccl3_GM", - "ch2cl2_SH", - "ch2cl2_NH", - "ch2cl2_GM", - "cfc12eq_SH", - "cfc12eq_NH", - "cfc12eq_GM", - "cfc12_SH", - "cfc12_NH", - "cfc12_GM", - "cfc11eq_SH", - "cfc11eq_NH", - "cfc11eq_GM", - "cfc11_SH", - "cfc11_NH", - "cfc11_GM", - "cfc115_SH", - "cfc115_NH", - "cfc115_GM", - "cfc114_SH", - "cfc114_NH", - "cfc114_GM", - "cfc113_SH", - "cfc113_NH", - "cfc113_GM", - "cf4_SH", - "cf4_NH", - "cf4_GM", - "ccode", - "carea", - "carbon_tetrachloride_SH", - "carbon_tetrachloride_NH", - "carbon_tetrachloride_GM", - "carbon_monoxide_GM", - "carbon_dioxide_SH", - "carbon_dioxide_NH", - "carbon_dioxide_GM", - "calyear", - "calmonth", - "calday", - "c_c4f8_SH", - "c_c4f8_NH", - "c_c4f8_GM", - "c8f18_SH", - "c8f18_NH", - "c8f18_GM", - "c7f16_SH", - "c7f16_NH", - "c7f16_GM", - "c6f14_SH", - "c6f14_NH", - "c6f14_GM", - "c5f12_SH", - "c5f12_NH", - "c5f12_GM", - "c4per_to_urban", - "c4per_to_secdn", - "c4per_to_secdf", - "c4per_to_range", - "c4per_to_pastr", - "c4per_to_c4ann", - "c4per_to_c3per", - "c4per_to_c3nfx", - "c4per_to_c3ann", - "c4per", - "c4f10_SH", - "c4f10_NH", - "c4f10_GM", - "c4ann_to_urban", - "c4ann_to_secdn", - "c4ann_to_secdf", - "c4ann_to_range", - "c4ann_to_pastr", - "c4ann_to_c4per", - "c4ann_to_c3per", - "c4ann_to_c3nfx", - "c4ann_to_c3ann", - "c4ann", - "c3per_to_urban", - "c3per_to_secdn", - "c3per_to_secdf", - "c3per_to_range", - "c3per_to_pastr", - "c3per_to_c4per", - "c3per_to_c4ann", - "c3per_to_c3nfx", - "c3per_to_c3ann", - "c3per", - "c3nfx_to_urban", - "c3nfx_to_secdn", - "c3nfx_to_secdf", - "c3nfx_to_range", - "c3nfx_to_pastr", - "c3nfx_to_c4per", - "c3nfx_to_c4ann", - "c3nfx_to_c3per", - "c3nfx_to_c3ann", - "c3nfx", - "c3f8_SH", - "c3f8_NH", - "c3f8_GM", - "c3ann_to_urban", - "c3ann_to_secdn", - "c3ann_to_secdf", - "c3ann_to_range", - "c3ann_to_pastr", - "c3ann_to_c4per", - "c3ann_to_c4ann", - "c3ann_to_c3per", - "c3ann_to_c3nfx", - "c3ann", - "c2f6_SH", - "c2f6_NH", - "c2f6_GM", - "bounds_time", - "bounds_sector", - "bounds_latitude", - "bounds_altitude", - "beta_b", - "beta_a", - "asy550", - "asl", - "areacello", - "areacellg", - "areacella", - "ap", - "aod_spmx", - "aod_fmbg", - "ann_cycle", - "angstrom", - "amv_index", - "altitude", - "added_tree_cover", - "acabf", - "WST", - "VOC_openburning_share", - "VOC_em_openburning", - "VOC_em_anthro", - "VOC_em_AIR_anthro", - "VOC25_other_voc_em_speciated_VOC_anthro", - "VOC25_other_voc_em_speciated_VOC", - "VOC25-other_voc_em_speciated_VOC", - "VOC24_acids_em_speciated_VOC_anthro", - "VOC24_acids_em_speciated_VOC", - "VOC24-acids_em_speciated_VOC", - "VOC23_ketones_em_speciated_VOC_anthro", - "VOC23_ketones_em_speciated_VOC", - "VOC23-ketones_em_speciated_VOC", - "VOC22_other_alka_em_speciated_VOC_anthro", - "VOC22_other_alka_em_speciated_VOC", - "VOC22-other_alka_em_speciated_VOC", - "VOC21_methanal_em_speciated_VOC_anthro", - "VOC21_methanal_em_speciated_VOC", - "VOC21-methanal_em_speciated_VOC", - "VOC20_chlorinate_em_speciated_VOC_anthro", - "VOC20_chlorinate_em_speciated_VOC", - "VOC20-chlorinate_em_speciated_VOC", - "VOC19_ethers_em_speciated_VOC_anthro", - "VOC19_ethers_em_speciated_VOC", - "VOC19-ethers_em_speciated_VOC", - "VOC18_esters_em_speciated_VOC_anthro", - "VOC18_esters_em_speciated_VOC", - "VOC18-esters_em_speciated_VOC", - "VOC17_other_arom_em_speciated_VOC_anthro", - "VOC17_other_arom_em_speciated_VOC", - "VOC17-other_arom_em_speciated_VOC", - "VOC16_trimethylb_em_speciated_VOC_anthro", - "VOC16_trimethylb_em_speciated_VOC", - "VOC16-trimethylb_em_speciated_VOC", - "VOC15_xylene_em_speciated_VOC_anthro", - "VOC15_xylene_em_speciated_VOC", - "VOC15-xylene_em_speciated_VOC", - "VOC14_toluene_em_speciated_VOC_anthro", - "VOC14_toluene_em_speciated_VOC", - "VOC14-toluene_em_speciated_VOC", - "VOC13_benzene_em_speciated_VOC_anthro", - "VOC13_benzene_em_speciated_VOC", - "VOC13-benzene_em_speciated_VOC", - "VOC12_other_alke_em_speciated_VOC_anthro", - "VOC12_other_alke_em_speciated_VOC", - "VOC12-other_alke_em_speciated_VOC", - "VOC09_ethyne_em_speciated_VOC_anthro", - "VOC09_ethyne_em_speciated_VOC", - "VOC09-ethyne_em_speciated_VOC", - "VOC08_propene_em_speciated_VOC_anthro", - "VOC08_propene_em_speciated_VOC", - "VOC08-propene_em_speciated_VOC", - "VOC07_ethene_em_speciated_VOC_anthro", - "VOC07_ethene_em_speciated_VOC", - "VOC07-ethene_em_speciated_VOC", - "VOC06_hexanes_pl_em_speciated_VOC_anthro", - "VOC06_hexanes_pl_em_speciated_VOC", - "VOC06-hexanes_pl_em_speciated_VOC", - "VOC05_pentanes_em_speciated_VOC_anthro", - "VOC05_pentanes_em_speciated_VOC", - "VOC05-pentanes_em_speciated_VOC", - "VOC04_butanes_em_speciated_VOC_anthro", - "VOC04_butanes_em_speciated_VOC", - "VOC04-butanes_em_speciated_VOC", - "VOC03_propane_em_speciated_VOC_anthro", - "VOC03_propane_em_speciated_VOC", - "VOC03-propane_em_speciated_VOC", - "VOC02_ethane_em_speciated_VOC_anthro", - "VOC02_ethane_em_speciated_VOC", - "VOC02-ethane_em_speciated_VOC", - "VOC01_alcohols_em_speciated_VOC_anthro", - "VOC01_alcohols_em_speciated_VOC", - "VOC01-alcohols_em_speciated_VOC", - "Toluene_lump", - "TRA", - "SO2_openburning_share", - "SO2_em_openburning", - "SO2_em_anthro", - "SO2_em_SOLID_BIOFUEL_anthro", - "SO2_em_AIR_anthro", - "SO2", - "SLV", - "SHP", - "RSLossRem", - "RCO", - "OC_openburning_share", - "OC_em_openburning", - "OC_em_anthro", - "OC_em_SOLID_BIOFUEL_anthro", - "OC_em_AIR_anthro", - "OC", - "NOx_openburning_share", - "NOx_em_openburning", - "NOx_em_anthro", - "NOx_em_SOLID_BIOFUEL_anthro", - "NOx_em_AIR_anthro", - "NOx", - "NMVOC_openburning_share", - "NMVOC_em_openburning", - "NMVOC_em_anthro", - "NMVOC_em_SOLID_BIOFUEL_anthro", - "NMVOC_em_AIR_anthro", - "NMVOC_Toluene_lump_speciated_VOC_openburning_share", - "NMVOC_Toluene_lump_em_speciated_VOC_openburning", - "NMVOC_MEK_speciated_VOC_openburning_share", - "NMVOC_MEK_em_speciated_VOC_openburning", - "NMVOC_Higher_Alkenes_speciated_VOC_openburning_share", - "NMVOC_Higher_Alkenes_em_speciated_VOC_openburning", - "NMVOC_Higher_Alkanes_speciated_VOC_openburning_share", - "NMVOC_Higher_Alkanes_em_speciated_VOC_openburning", - "NMVOC_HOCH2CHO_speciated_VOC_openburning_share", - "NMVOC_HOCH2CHO_em_speciated_VOC_openburning", - "NMVOC_HCOOH_speciated_VOC_openburning_share", - "NMVOC_HCOOH_em_speciated_VOC_openburning", - "NMVOC_HCN_speciated_VOC_openburning_share", - "NMVOC_HCN_em_speciated_VOC_openburning", - "NMVOC_CH3OH_speciated_VOC_openburning_share", - "NMVOC_CH3OH_em_speciated_VOC_openburning", - "NMVOC_CH3COOH_speciated_VOC_openburning_share", - "NMVOC_CH3COOH_em_speciated_VOC_openburning", - "NMVOC_CH3COCHO_speciated_VOC_openburning_share", - "NMVOC_CH3COCHO_em_speciated_VOC_openburning", - "NMVOC_CH2O_speciated_VOC_openburning_share", - "NMVOC_CH2O_em_speciated_VOC_openburning", - "NMVOC_C8H10_speciated_VOC_openburning_share", - "NMVOC_C8H10_em_speciated_VOC_openburning", - "NMVOC_C7H8_speciated_VOC_openburning_share", - "NMVOC_C7H8_em_speciated_VOC_openburning", - "NMVOC_C6H6_speciated_VOC_openburning_share", - "NMVOC_C6H6_em_speciated_VOC_openburning", - "NMVOC_C5H8_speciated_VOC_openburning_share", - "NMVOC_C5H8_em_speciated_VOC_openburning", - "NMVOC_C3H8_speciated_VOC_openburning_share", - "NMVOC_C3H8_em_speciated_VOC_openburning", - "NMVOC_C3H6_speciated_VOC_openburning_share", - "NMVOC_C3H6_em_speciated_VOC_openburning", - "NMVOC_C3H6O_speciated_VOC_openburning_share", - "NMVOC_C3H6O_em_speciated_VOC_openburning", - "NMVOC_C2H6_speciated_VOC_openburning_share", - "NMVOC_C2H6_em_speciated_VOC_openburning", - "NMVOC_C2H6S_speciated_VOC_openburning_share", - "NMVOC_C2H6S_em_speciated_VOC_openburning", - "NMVOC_C2H5OH_speciated_VOC_openburning_share", - "NMVOC_C2H5OH_em_speciated_VOC_openburning", - "NMVOC_C2H4_speciated_VOC_openburning_share", - "NMVOC_C2H4_em_speciated_VOC_openburning", - "NMVOC_C2H4O_speciated_VOC_openburning_share", - "NMVOC_C2H4O_em_speciated_VOC_openburning", - "NMVOC_C2H2_speciated_VOC_openburning_share", - "NMVOC_C2H2_em_speciated_VOC_openburning", - "NMVOC_C10H16_speciated_VOC_openburning_share", - "NMVOC_C10H16_em_speciated_VOC_openburning", - "NMVOC", - "NH3_openburning_share", - "NH3_em_openburning", - "NH3_em_anthro", - "NH3_em_SOLID_BIOFUEL_anthro", - "NH3_em_AIR_anthro", - "NH3", - "N2O", - "MEK", - "IND", - "Higher_Alkenes", - "Higher_Alkanes", - "HOCH2CHO", - "HCOOH", - "HCN", - "H2_openburning_share", - "H2_em_openburning", - "H2SO4_mass", - "H2", - "ENE", - "Delta14co2_in_air", - "CO_openburning_share", - "CO_em_openburning", - "CO_em_anthro", - "CO_em_SOLID_BIOFUEL_anthro", - "CO_em_AIR_anthro", - "CO2_em_anthro", - "CO2_em_AIR_anthro", - "CO2", - "CO", - "CH4_openburning_share", - "CH4_em_openburning", - "CH4_em_anthro", - "CH4_em_SOLID_BIOFUEL_anthro", - "CH4_em_AIR_anthro", - "CH4", - "CH3OH", - "CH3COOH", - "CH3COCHO", - "CH2O", - "C8H10", - "C7H8", - "C6H6", - "C5H8", - "C3H8", - "C3H6O", - "C3H6", - "C2H6S", - "C2H6", - "C2H5OH", - "C2H4O", - "C2H4", - "C2H2", - "C10H16", - "BC_openburning_share", - "BC_em_openburning", - "BC_em_anthro", - "BC_em_SOLID_BIOFUEL_anthro", - "BC_em_AIR_anthro", - "BC", - "AIR", - "AGR", - ], -} - -SUPPORTED_EXPERIMENTS = [ - "ssp585", - "ssp370-lowNTCF", - "ssp370", - "ssp245", - "ssp126", - "piControl", - "piClim-spAer-anthro", - "piClim-spAer-aer", - "piClim-lu", - "piClim-histnat", - "piClim-histghg", - "piClim-histall", - "piClim-histaer", - "piClim-ghg", - "piClim-control", - "piClim-anthro", - "piClim-aer", - "piClim-N2O", - "piClim-CH4", - "piClim-4xCO2", - "piClim-2xss", - "piClim-2xdust", - "piClim-2xVOC", - "piClim-2xDMS", - "pdSST-piArcSIC", - "pdSST-pdSIC", - "pdSST-futArcSIC", - "midHolocene", - "lig127k", - "historical", - "histSST-piNTCF", - "histSST-piAer", - "histSST", - "hist-spAer-all", - "hist-piNTCF", - "hist-piAer", - "hist-nat", - "hist-aer", - "hist-GHG", - "amip", -] -# filepath to var to res Mapping -VAR_RES_MAPPING_PATH = "/home/charlie/Documents/MILA/causalpaca/data/data_description/mappings/variableid2tableid.csv" - - -GRIDDING_HIERACHY = ["gn"] - -# skip subhr because only diagnostics for specific places -REMOVE_RESOLUTONS = ["suhbr"] # resolution endings to remove e.g. kick CFsubhr if this contains 'subhr' - - -RES_TO_CHUNKSIZE = {"year": 1, "mon": 12, "6hr": 1460, "3hr": 2920, "day": 364} diff --git a/climateset/download/constants/esm_constants.py b/climateset/download/constants/esm_constants.py deleted file mode 100644 index 780fbbe..0000000 --- a/climateset/download/constants/esm_constants.py +++ /dev/null @@ -1,82 +0,0 @@ -### DOWNLOADER PARAMS ########################################################## - -# these resolutions are stored in RESOLUTION -TEMP_RES = 0 -VERT_RES = 0 -LON_RES = 0 -LAT_RES = 0 - -# resolution of the end-data-product -RESOLUTION = (TEMP_RES, VERT_RES, LON_RES, LAT_RES) - -# list of years that are considered for the data -YEARS = [0] - - -# distinction not necessary for the mother as we are first just providing data not designing the loader yet, and a -# lookup table to check where to downloda what from anyway -# # variables used as input for the climate model -# IN_VARS = [] -# -# # predicted / target variables of the climate model -# OUT_VARS = [] -# # suggestion charlie -# VARS = ["nan"] -# # Julia: Birth has three steps: downloading, preprocessing, creating the different resolutions -# # and we already need to distinct between in_vars and out_vars for that - - -CO2 = ["CO2", "CO2_em_anthro", "CO2_em_openburning", "CO2_em_AIR_anthro"] -BC = ["BC", "BC_em_anthro", "BC_em_openburning", "BC_em_AIR_anthro"] -CH4 = ["CH4", "CH4_em_anthro", "CH4_em_openburning", "CH4_em_AIR_anthro"] -SO2 = ["SO2", "SO2_em_anthro", "SO2_em_openburning", "SO2_em_AIR_anthro"] - -IN_VARS = CO2 + BC + CH4 + SO2 -OUT_VARS = ["pr", "tas"] - -VARS = IN_VARS + OUT_VARS - -# scenarios -SCENARIOS = ["historical", "ssp126", "ssp245", "ssp370", "ssp585"] -ADDITIONAL_SCENARIOS = ["hist-aer", "hist-GHG", "piControl", "ssp370-lowNTCF"] - -# model -MODELS = ["nan"] - -# number of ensemble members to be considered -NUM_ENSEMBLE = 1 - -# which type of grid -GRID = "grid" - -### RAW PROCESSER PARAMS ####################################################### -# you will see after downloading - -### RESOLUTION PROCESSER PARAMS ################################################ - -# THIS must be moved somewhere else, because it's not static -# tuple of "means" of preprocesser for each variable, e.g. -# [("CO2", "mean"), ["CH4", "median"] -CHOSEN_AGGREGATIONS = [ - "MeanAggregation", - "MinAggregation", - "MaxAggregation", - "InstAggregation", -] -# TODO communicate to other persons which data structure etc you use here -CHOSEN_INTERPOLATIONS = {"nan"} -# TODO create a fixed list for all vars: which aggregation and interpolation - -### ALL PARAMS IN DICT ######################################################### -CORE_PARAMS = { - "models": MODELS, - "scenarios": SCENARIOS, - "years": YEARS, - "in_vars": IN_VARS, - "out_vars": OUT_VARS, - "vars": VARS, - "resolutions": RESOLUTION, - "grid": GRID, - "aggregations": CHOSEN_AGGREGATIONS, - "interpolations": CHOSEN_INTERPOLATIONS, -} diff --git a/climateset/download/constants/input4mips.py b/climateset/download/constants/input4mips.py new file mode 100644 index 0000000..7c6643a --- /dev/null +++ b/climateset/download/constants/input4mips.py @@ -0,0 +1,45 @@ +# TODO add VAR_SOURCE_LOOKUP with raw variables +# TODO add supported experiments +# TODO do we really need emission endings, meta_endings_prc, meta_endings_shar?? how is this used so far? +# pylint: disable=C0103 +from dataclasses import dataclass +from typing import Final + +from climateset.utils import get_yaml_config + + +@dataclass(frozen=True) +class Input4MIPSConstants: + """ + Data class to represent Input4MIPS constants that are used by the download module. + + Attributes: + NODE_LINK : Node link is used to run an ESGF search + EMISSION_ENDINGS : File endings for emission variables + META_ENDINGS_PRC : File endings for PRC meta variables + META_ENDINGS_SHARE : File endings for SHARE meta variables + VAR_SOURCE_LOOKUP : Model and raw variables + """ + + NODE_LINK: Final[str] + EMISSIONS_ENDINGS: Final[tuple[str, ...]] + META_ENDINGS_PRC: Final[tuple[str, ...]] + META_ENDINGS_SHAR: Final[tuple[str, ...]] + MIP_ERA: Final[str] + TARGET_MIP: Final[str] + SUPPORTED_EXPERIMENTS: Final[tuple[str, ...]] + VAR_SOURCE_LOOKUP: Final[tuple[str, ...]] + + +_data = get_yaml_config("downloader/constants/imput4MIPs.yaml") + +INPUT4MIPS_CONSTANTS = Input4MIPSConstants( + NODE_LINK=_data["node_link"], + EMISSIONS_ENDINGS=tuple(_data["emissions_endings"]), + META_ENDINGS_PRC=tuple(_data["meta_endings_prc"]), + META_ENDINGS_SHAR=tuple(_data["meta_endings_shar"]), + MIP_ERA=_data["mip_era"], + TARGET_MIP=_data["target_mip"], + SUPPORTED_EXPERIMENTS=tuple(_data["supported_experiments"]), + VAR_SOURCE_LOOKUP=tuple(_data["var_source_lookup"]), +) diff --git a/climateset/download/constants/omip.py b/climateset/download/constants/omip.py new file mode 100644 index 0000000..abcdb10 --- /dev/null +++ b/climateset/download/constants/omip.py @@ -0,0 +1,22 @@ +class OmipConstants: + """ + Attributes: + NODE_LINK (str): Where the data can be accessed + MODEL_SOURCES (List): Identifiers for supported climate models + VAR_SOURCE_LOOKUP (Dict>): model and raw variables + SUPPORTED_EXPERIMENTS (list): experiments of climate models (runs) that are supported + """ + + NODE_LINK = "http://esgf-data2.llnl.gov" + + MODEL_SOURCES = [ + "NorESM2-LM", + ] + + VAR_SOURCE_LOOKUP = [ + "omldamax", + ] + + SUPPORTED_EXPERIMENTS = [ + "omip1", + ] diff --git a/climateset/download/constraints.py b/climateset/download/constraints.py new file mode 100644 index 0000000..8b57340 --- /dev/null +++ b/climateset/download/constraints.py @@ -0,0 +1,78 @@ +from dataclasses import asdict, dataclass +from typing import Any + +# Define a type alias for fields that can support esgpull multi-value lists +StrOrList = str | list[str] | None + + +@dataclass(frozen=True) +class BaseSearchConstraints: + """ + Immutable base constraints for ESGF searches. + + Attributes: + project (str | list[str] | None): The project name (e.g., "CMIP6"). + variable (str | list[str] | None): The variable name (e.g., "tas"). + frequency (str | list[str] | None): The frequency of the data (e.g., "mon"). + grid_label (str | list[str] | None): The grid label (e.g., "gn"). + nominal_resolution (str | list[str] | None): The nominal resolution (e.g., "100 km"). + version (str | list[str] | None): The version of the dataset (e.g., "20190101"). + """ + + project: StrOrList = None + variable: StrOrList = None + frequency: StrOrList = None + grid_label: StrOrList = None + nominal_resolution: StrOrList = None + version: StrOrList = None + + def to_esgf_params(self) -> dict[str, Any]: + """ + Convert to ESGF search parameters (kwargs), filtering out None values. + + Returns: + dict[str, Any]: A dictionary identifying parameters suitable for esgf-pyclient. + """ + return {k: v for k, v in asdict(self).items() if v is not None} + + def to_esgpull_query(self) -> dict[str, Any]: + """ + Convert constraints to parameters compatible with esgpull.models.Query selection. This explicitly handles multi- + value lists, exclusion (!), and wildcard (*) capabilities provided by esgpull. + + Returns: + dict[str, Any]: A dictionary suitable to be unpacked into an esgpull Query selection. + """ + return {k: v for k, v in asdict(self).items() if v is not None} + + +@dataclass(frozen=True) +class Input4MIPsConstraints(BaseSearchConstraints): + """ + Constraints specific to Input4MIPs searches. + + Attributes: + institution_id (str | list[str] | None): The institution ID. + variable_id (str | list[str] | None): The variable ID. + target_mip (str | list[str] | None): The target MIP. + """ + + institution_id: StrOrList = None + variable_id: StrOrList = None + target_mip: StrOrList = None + + +@dataclass(frozen=True) +class CMIP6Constraints(BaseSearchConstraints): + """ + Constraints specific to CMIP6 searches. + + Attributes: + experiment_id (str | list[str] | None): The experiment ID. + source_id (str | list[str] | None): The source model ID. + variant_label (str | list[str] | None): The variant label (ensemble member). + """ + + experiment_id: StrOrList = None + source_id: StrOrList = None + variant_label: StrOrList = None diff --git a/climateset/download/downloader.py b/climateset/download/downloader.py index 8ddd16f..7ac8d25 100644 --- a/climateset/download/downloader.py +++ b/climateset/download/downloader.py @@ -1,485 +1,38 @@ import logging import pathlib -from typing import Union -import pandas as pd -from pyesgf.search import SearchConnection - -from climateset import RAW_DATA -from climateset.download.constants.data_constants import ( - EMISSIONS_ENDINGS, - META_ENDINGS_PRC, - META_ENDINGS_SHAR, -) -from climateset.download.constants.esgf_server import ( - MODEL_SOURCES, - SUPPORTED_EXPERIMENTS, - VAR_SOURCE_LOOKUP, -) -from climateset.download.utils import ( - _handle_base_search_constraints, - download_metadata_variable, - download_model_variable, - download_raw_input_variable, - get_max_ensemble_member_number, - get_select_model_scenarios, - get_upload_version, -) -from climateset.utils import create_logger, get_keys_from_value, get_yaml_config +from climateset.download.cmip6_downloader import cmip6_download_from_config +from climateset.download.constants.esgf import CMIP6, INPUT4MIPS +from climateset.download.downloader_config import AVAILABLE_CONFIGS +from climateset.download.input4mips_downloader import input4mips_download_from_config +from climateset.download.utils import match_key_in_list +from climateset.utils import create_logger, get_yaml_config LOGGER = create_logger(__name__) -class Downloader: - """ - Class handling the downloading of the data. - - It communicates with the esgf nodes to search and download the specified data. - """ - - # TODO Fix complexity issue - def __init__( # noqa: C901 - self, - model: Union[str, None] = "NorESM2-LM", # default as in ClimateBench - experiments: list[str] = None, # sub-selection of ClimateBench default - variables: list[str] = None, - data_dir: str = RAW_DATA, - max_ensemble_members: int = 10, # if -1 take all - ensemble_members: list[str] = None, # preferred ensemble members used, if None not considered - overwrite: bool = False, # flag if files should be overwritten - download_biomassburning: bool = True, # get biomassburning data for input4mips - download_metafiles: bool = True, # get input4mips meta files - use_plain_emission_vars: bool = True, # specifies if plain variables are given and rest is inferred - logger: logging.Logger = LOGGER, - ): - """ - Init method for the Downloader. - - Args: - model: Model ID from which output should be downloaded. A list of all supported model ids can - be found in parameters.constants.MODEL_SOURCES. Model data only. - experiments: List of simulations from which data should be downloaded. Model data only. - experiments: List of variables for which data should be downloaded. Both model and raw data. - data_dir: Relative or absolute path to the directory where data should be stored. Will be created - if not yet existent. - meta_dir: Relative or absolute path to the directory where the metadata should be sored. Will be - created if not yet existent. - overwrite: Flag if files should be overwritten, if they already exist. - download_biomassburning: Flag if biomassburning data for input4mips variables should be downloaded. - download_metafiles: Flag if metafiles for input4mips variables should be downloaded. - """ - # Args init - self.logger = logger - self.model: str = model - self.model_node_link: str = "" - self.model_source_center: str = "" - if experiments is None: - experiments = [ - "historical", - "ssp370", - "hist-GHG", - "piControl", - "ssp434", - "ssp126", - ] - # TODO: have a list of supported experiments before trying to look for them on the node - # to reduce computation cost - self.experiments: list[str] = experiments - self.raw_vars: list[str] = [] - self.model_vars: list[str] = [] - self.biomass_vars: list[str] = [] - self.meta_vars_percentage: list[str] = [] - self.meta_vars_share: list[str] = [] - self.data_dir: Union[str, pathlib.Path] = data_dir - self.ensemble_members: list[str] = ensemble_members - self.max_ensemble_members: int = max_ensemble_members - self.overwrite: bool = overwrite - self.download_metafiles: bool = download_metafiles - self.download_biomass_burning: bool = download_biomassburning - self.use_plain_emission_vars: bool = use_plain_emission_vars - - # Args processing - selected_scenarios = get_select_model_scenarios() - self._hande_max_possible_member_number( - df_model_source=selected_scenarios, max_ensemble_members=max_ensemble_members - ) - self._handle_variables( - variables=variables, - ) - self._handle_model_params() - - # - # Internal helper functions for class init - # - def _hande_max_possible_member_number(self, df_model_source: pd.DataFrame, max_ensemble_members: int): - max_possible_member_number = get_max_ensemble_member_number( - df_model_source=df_model_source, experiments=self.experiments, model=self.model - ) - if max_ensemble_members == -1: - self.logger.info("Trying to take all ensemble members available.") - self.max_ensemble_members = max_possible_member_number - # verify that we have enough members for wanted experiments - # else choose the smallest available for all - if max_ensemble_members > max_possible_member_number: - self.logger.info("Not enough members available. Choosing smallest maximum.") - self.max_ensemble_members = max_possible_member_number - self.logger.info(f"Downloading data for {self.max_ensemble_members} members.") - - def _handle_variables(self, variables: list[str]): - self._generate_variables(variables=variables) - self._generate_plain_emission_vars() - self.logger.info(f"Raw variables to download: {self.raw_vars}") - self.logger.info(f"Model predicted vars to download: {self.model_vars}") - if self.download_biomass_burning: - self.logger.info(f"Download biomass burning vars: {self.biomass_vars}") - if self.download_metafiles: - self.logger.info(f"Downloading meta vars:\n\t{self.meta_vars_percentage}\n\t{self.meta_vars_share}") - - def _handle_model_params(self): - try: - self.model_node_link = MODEL_SOURCES[self.model]["node_link"] - self.model_source_center = MODEL_SOURCES[self.model]["center"] - except KeyError: - self.model = next(iter(MODEL_SOURCES)) - if self.model is not None: - self.logger.info(f"WARNING: Model {self.model} unknown. Using default instead.") - self.logger.info(f"Using : {self.model}") - self.model_node_link = MODEL_SOURCES[self.model]["node_link"] - self.model_source_center = MODEL_SOURCES[self.model]["center"] - - def _generate_plain_emission_vars(self): - if self.use_plain_emission_vars: - # plain vars are biomass vars - self.biomass_vars = self.raw_vars - self.meta_vars_percentage = [ - biomass_var + ending - for biomass_var in self.biomass_vars - if biomass_var != "CO2" - for ending in META_ENDINGS_PRC - ] - self.meta_vars_share = [ - biomass_var + ending - for biomass_var in self.biomass_vars - if biomass_var != "CO2" - for ending in META_ENDINGS_SHAR - ] - - self.raw_vars = [ - variable + emission_ending for variable in self.raw_vars for emission_ending in EMISSIONS_ENDINGS - ] - # be careful with CO2 - if "CO2_em_openburning" in self.raw_vars: - self.raw_vars.remove("CO2_em_openburning") - else: - # get plain input4mips vars = biomass vars for historical - self.biomass_vars = list({v.split("_")[0] for v in self.raw_vars}) - # remove biomass vars from normal raw vars list - for b in self.biomass_vars: - try: - self.raw_vars.remove(b) - except Exception as error: - self.logger.warning(f"Caught the following exception but continuing : {error}") - - self.meta_vars_percentage = [ - biomass_var + ending - for biomass_var in self.biomass_vars - if biomass_var != "CO2" - for ending in META_ENDINGS_PRC - ] - self.meta_vars_share = [ - biomass_var + ending - for biomass_var in self.biomass_vars - if biomass_var != "CO2" - for ending in META_ENDINGS_SHAR - ] - - def _generate_variables(self, variables: list[str]): - if variables is None: - variables = ["tas", "pr", "SO2_em_anthro", "BC_em_anthro"] - variables = [v.replace(" ", "_").replace("-", "_") for v in variables] - self.logger.info(f"Cleaned variables : {variables}") - for v in variables: - t = get_keys_from_value(d=VAR_SOURCE_LOOKUP, val=v, logger=self.logger) - if t == "model": - self.model_vars.append(v) - elif t == "raw": - self.raw_vars.append(v) - - else: - self.logger.info(f"WARNING: unknown source type for var {v}. Not supported. Skipping.") - - # - # Class functions - # - def download_from_model_single_var( # noqa: C901 - self, - variable: str, - experiment: str, - project: str = "CMIP6", - default_frequency: str = "mon", - preferred_version: str = "latest", - default_grid_label: str = "gn", - ): - """ - Function handling the download of a single variable-experiment pair that is associated with a model's output - (CMIP data). - - Args: - variable: variable ID - experiment: experiment ID - project: umbrella project id e.g. CMIPx - default_frequency: default frequency to download - preferred_version: data upload version, if 'latest', the newest version will get selected always - default_grid_label: default gridding method in which the data is provided - """ - conn = SearchConnection(url=self.model_node_link, distrib=False) - - facets = ( - "project,experiment_id,source_id,variable,frequency,variant_label,variable, nominal_resolution, " - "version, grid_label, experiment_id" - ) - - self.logger.info("Using download_from_model_single_var() function") - - ctx = conn.new_context( - project=project, - experiment_id=experiment, - source_id=self.model, - variable=variable, - facets=facets, - ) - - ctx = _handle_base_search_constraints(ctx, default_frequency, default_grid_label) - - variants = list(ctx.facet_counts["variant_label"]) - - self.logger.info(f"Available variants : {variants}\n") - self.logger.info(f"Length : {len(variants)}") - - # TODO refactor logic of if/else - if not self.ensemble_members: - if self.max_ensemble_members > len(variants): - self.logger.info("Less ensemble members available than maximum number desired. Including all variants.") - ensemble_member_final_list = variants - else: - self.logger.info( - f"{len(variants)} ensemble members available than desired (max {self.max_ensemble_members}. " - f"Choosing only the first {self.max_ensemble_members}.)." - ) - ensemble_member_final_list = variants[: self.max_ensemble_members] - else: - self.logger.info(f"Desired list of ensemble members given: {self.ensemble_members}") - ensemble_member_final_list = list(set(variants) & set(self.ensemble_members)) - if len(ensemble_member_final_list) == 0: - self.logger.info("WARNING: no overlap between available and desired ensemble members!") - self.logger.info("Skipping.") - return None - - for ensemble_member in ensemble_member_final_list: - self.logger.info(f"Ensembles member: {ensemble_member}") - ctx_ensemble = ctx.constrain(variant_label=ensemble_member) - - version = get_upload_version(context=ctx, preferred_version=preferred_version) - if version: - ctx_ensemble = ctx_ensemble.constrain(version=version) - - results = ctx_ensemble.search() - - self.logger.info(f"Result len {len(results)}") - - download_model_variable( - model_id=self.model, search_results=results, variable=variable, base_path=self.data_dir - ) - - def download_raw_input_single_var( # noqa: C901 - self, - variable: str, - project: str = "input4mips", - institution_id: str = "PNNL-JGCRI", - default_frequency: str = "mon", - preferred_version: str = "latest", - default_grid_label: str = "gn", - ): - """ - Function handling the download of all input4mips data associated with a single variable. - - Args: - variable: variable ID - project: umbrella project, here "input4mips" - institution_id: id of the institution that provides the data - default_frequency: default frequency to download - preferred_version: data upload version, if 'latest', the newest version will get selected always - default_grid_label: default gridding method in which the data is provided - """ - self.logger.info("Using download_raw_input_single_var() function") - - facets = "project,frequency,variable,nominal_resolution,version,target_mip,grid_label" - conn = SearchConnection(url=self.model_node_link, distrib=False) - - ctx = conn.new_context( - project=project, - variable=variable, - institution_id=institution_id, - facets=facets, - ) - - ctx = _handle_base_search_constraints(ctx, default_frequency, default_grid_label) - - mips_targets = list(ctx.facet_counts["target_mip"]) - self.logger.info(f"Available target mips: {mips_targets}") - - for target in mips_targets: - ctx_target = ctx.constrain(target_mip=target) - version = get_upload_version(context=ctx_target, preferred_version=preferred_version) - if version: - ctx_target = ctx_target.constrain(version=version) - - results = ctx_target.search() - self.logger.info(f"Result len {len(results)}") - if len(results) > 0: - download_raw_input_variable( - institution_id=institution_id, search_results=results, variable=variable, base_path=self.data_dir - ) - - def download_meta_historic_biomassburning_single_var( - self, - variable: str, - institution_id: str, - project: str = "input4mips", - default_grid_label: str = "gn", - default_frequency: str = "mon", - preferred_version: str = "latest", - ): - """ - Function handling the download of all metadata associated with a single input4mips variable. - - Args: - variable: variable ID - project: umbrella project - institution_id: id of the institution that provides the data - default_grid_label: default gridding method in which the data is provided - default_frequency: default frequency to download - preferred_version: data upload version, if 'latest', the newest version will get selected always - """ - variable_id = variable.replace("_", "-") - variable_search = f"percentage_{variable_id.replace('-', '_').split('_')[-1]}" - self.logger.info(variable, variable_id, institution_id) - conn = SearchConnection(url=self.model_node_link, distrib=False) - facets = "nominal_resolution,version" - ctx = conn.new_context( - project=project, - variable=variable_search, - variable_id=variable_id, - institution_id=institution_id, - target_mip="CMIP", - facets=facets, - ) - - ctx = _handle_base_search_constraints(ctx, default_frequency, default_grid_label) - - version = get_upload_version(context=ctx, preferred_version=preferred_version) - if version: - ctx = ctx.constrain(version=version) - - results = ctx.search() - self.logger.info(f"Result len {len(results)}") - - result_list = [r.file_context().search() for r in results] - self.logger.info(f"List of results :\n{result_list}") - - download_metadata_variable( - institution_id=institution_id, search_results=results, variable=variable, base_path=self.data_dir - ) - - def download_from_model(self): - """ - Function handling the download of all variables that are associated with a model's output. - - Searches for all files associated with the respected variables and experiment that the downloader - was initialized with. - - A search connection is established and the search is iteratively constraint to meet all specifications. - Data is downloaded and stored in a separate file for each year. The default format is netCDF4. - - Resulting hierarchy: - - `CMIPx/model_id/ensemble_member/experiment/variable/nominal_resolution/frequency/year.nc` - - If the constraints cannot be met, per default behaviour for the downloader to select first other - available value - """ - - for variable in self.model_vars: - self.logger.info(f"Downloading data for variable: {variable}") - for experiment in self.experiments: - if experiment in SUPPORTED_EXPERIMENTS: - self.logger.info(f"Downloading data for experiment: {experiment}") - self.download_from_model_single_var(variable=variable, experiment=experiment) - else: - self.logger.info( - f"Chosen experiment {experiment} not supported. All supported experiments: " - f"{SUPPORTED_EXPERIMENTS}. Skipping." - ) - - def download_raw_input(self): - """ - Function handling the download of all variables that are associated with a model's input (input4mips). - - Searches for all files associated with the respected variables that the downloader was initialized with. - A search connection is established and the search is iteratively constraint to meet all specifications. - Data is downloaded and stored in a separate file for each year. The default format is netCDF4. - - Resulting hierarchy: - - `input4mips/experiment/variable/nominal_resolution/frequency/year.nc` - - If the constraints cannot be met, the default behaviour for the downloader is to select first other - available value. - """ - for variable in self.raw_vars: - if variable.endswith("openburning"): - institution_id = "IAMC" - else: - institution_id = "PNNL-JGCRI" - self.logger.info(f"Downloading data for variable: {variable}") - self.download_raw_input_single_var(variable=variable, institution_id=institution_id) - - if self.download_biomass_burning & ("historical" in self.experiments): - for variable in self.biomass_vars: - self.logger.info(f"Downloading biomassburing data for variable: {variable}") - self.download_raw_input_single_var(variable=variable, institution_id="VUA") - - if self.download_metafiles: - for variable in self.meta_vars_percentage: - # percentage are historic and have no scenarios - self.logger.info(f"Downloading meta percentage data for variable: {variable}") - self.download_meta_historic_biomassburning_single_var(variable=variable, institution_id="VUA") - for variable in self.meta_vars_share: - self.logger.info(f"Downloading meta openburning share data for variable: {variable}") - self.download_raw_input_single_var(variable=variable, institution_id="IAMC") - - -def download_from_config_file(config: str, logger: logging.Logger = LOGGER): +def download_from_config_file(config_file: str | pathlib.Path, logger: logging.Logger = LOGGER): """ This function downloads variables automatically from input config file Args: - config: Can be a dictionary containing configurations or a path to a configuration yaml file + config_file: Path to a configuration yaml file logger: Logging instance """ - if not isinstance(config, dict): - if isinstance(config, str): - config = pathlib.Path(config) - config = get_yaml_config(config) - try: - models = config["models"] - except KeyError as e: - logger.warning(f"Caught the following exception but continuing : {e}") - logger.info("No climate models specified. Assuming only input4mips data should be downloaded.") - models = [None] - downloader_kwargs = config["downloader_kwargs"] - logger.info(f"Downloader kwargs : {downloader_kwargs}") - for m in models: - downloader = Downloader(model=m, **downloader_kwargs, logger=logger) - downloader.download_raw_input() - if m is not None: - downloader.download_from_model() + if isinstance(config_file, str): + config_file = pathlib.Path(config_file) + config_dict = get_yaml_config(config_file) + + downloader_factory = { + INPUT4MIPS: input4mips_download_from_config, + CMIP6: cmip6_download_from_config, + } + + for config_key in config_dict: + verified_key = match_key_in_list(input_key=config_key, key_list=AVAILABLE_CONFIGS) + if verified_key: + downloader_factory[verified_key](config=config_file) + else: + logger.error( + f"Input project [{config_key}] from [{config_file}] was not found in available projects. " + "Removing it from download list" + ) diff --git a/climateset/download/downloader_config.py b/climateset/download/downloader_config.py new file mode 100644 index 0000000..5b36a97 --- /dev/null +++ b/climateset/download/downloader_config.py @@ -0,0 +1,252 @@ +import copy +import inspect +import logging +from pathlib import Path + +import yaml + +from climateset import CONFIGS, RAW_DATA +from climateset.download.constants.esgf import ( + CMIP6, + ESGF_PROJECTS, + ESGF_PROJECTS_CONSTANTS, + INPUT4MIPS, +) +from climateset.download.utils import handle_yaml_config_path, match_key_in_list +from climateset.utils import create_logger, get_yaml_config + +LOGGER = create_logger(__name__) + +AVAILABLE_CONFIGS = frozenset([CMIP6, INPUT4MIPS]) + + +class BaseDownloaderConfig: + def __init__( + self, + project: str, + data_dir: str | Path = RAW_DATA, + experiments: list[str] = None, + variables: list[str] = None, + overwrite: bool = False, + logger: logging.Logger = LOGGER, + ): + self.logger = logger + + self.project = "" + uppercase_project = project.upper() + for p in ESGF_PROJECTS: + if p.upper() == uppercase_project: + self.project = p + + if self.project not in ESGF_PROJECTS: + self.logger.error(f"Project {self.project} has not been implemented in the Downloader yet.") + raise ValueError( + f"Project {self.project} is not recognized. Consider adding a constant class in download/constants and " + f"the esgf.py file." + ) + + if isinstance(data_dir, str): + data_dir = Path(data_dir) + self.data_dir = data_dir + + self.experiments = experiments + self.variables = variables + self.overwrite = overwrite + + # init shared constants + self.proj_constants = ESGF_PROJECTS_CONSTANTS[self.project] + self.node_link = self.proj_constants.NODE_LINK + self.avail_variables = self.proj_constants.VAR_SOURCE_LOOKUP + self.avail_experiments = self.proj_constants.SUPPORTED_EXPERIMENTS + self.config_is_valid = True + + self._validate_item_list( + item_list=self.variables, available_items=self.avail_variables, name_of_item="variable" + ) + self._validate_item_list( + item_list=self.experiments, available_items=self.avail_experiments, name_of_item="experiment" + ) + + def _validate_item_list(self, item_list: list[str], available_items: list[str], name_of_item: str) -> None: + """ + This small function checks that the given items (variables, models, experiments, etc.) are valid for their given + project (Input4MIPs, CMIP6, etc.). + + Also remove unvalid items from the list of items as to not. + + Args: + item_list: List of items to check (like self.variables, self.experiments, etc.) + available_items: List of available items against which to check (like self.avail_variables, etc.) + name_of_item: Name of item to check. Write lowercase and singular: ie. variable, experiment, etc. + + Returns: + None + """ + invalid_items = [e for e in item_list if e not in available_items] + for e in invalid_items: + self.logger.error(f"{name_of_item.capitalize()} [{e}] not supported.") + item_list.remove(e) + if invalid_items: + self.logger.error(f"Some or all submitted {name_of_item}s were not found - Please verify") + self.logger.error(f"Available {name_of_item}s: {available_items}") + self.logger.warning(f"List of valid submitted {name_of_item}s: {available_items}") + self.config_is_valid = False + + def generate_config_dict(self): + init_params = inspect.signature(self.__init__).parameters + init_args = set(init_params.keys()) - {"self"} + config_dict = {self.project: {}} + for key, value in self.__dict__.items(): + if key in init_args and key not in ["project", "logger"] and not callable(value): + config_dict[self.project][key] = value + return config_dict + + def generate_config_file(self, config_file_name: str, config_path: str | Path = CONFIGS) -> None: + config_full_path = handle_yaml_config_path(config_file_name, config_path) + data = self.generate_config_dict() + with open(config_full_path, "w", encoding="utf-8") as config_file: + yaml.dump(data, config_file, indent=2) + + def add_to_config_file(self, config_file_name: str, config_path: str | Path = CONFIGS) -> None: + config_full_path = handle_yaml_config_path(config_file_name, config_path) + existing_config = {} + if config_full_path.exists(): + existing_config = get_yaml_config(config_full_path) + new_config = self.generate_config_dict() + existing_config.update(new_config) + with open(config_full_path, "w", encoding="utf-8") as config_file: + yaml.dump(existing_config, config_file, indent=2) + + +class Input4mipsDownloaderConfig(BaseDownloaderConfig): + def __init__( + self, + project: str, + data_dir: str = RAW_DATA, + experiments: list[str] = None, + variables: list[str] = None, + download_biomassburning: bool = True, # get biomassburning data for input4mips + download_metafiles: bool = True, # get input4mips meta files + use_plain_emission_vars: bool = True, + overwrite: bool = False, + logger: logging.Logger = LOGGER, + ): + super().__init__(project, data_dir, experiments, variables, overwrite, logger) + + self.download_metafiles: bool = download_metafiles # TODO infer automatically from vars + self.download_biomass_burning: bool = download_biomassburning # TODO infer automatically from vars + self.use_plain_emission_vars: bool = use_plain_emission_vars + self.emissions_endings = self.proj_constants.EMISSIONS_ENDINGS + self.meta_endings_prc = self.proj_constants.META_ENDINGS_PRC + self.meta_endings_share = self.proj_constants.META_ENDINGS_SHAR + self.mip_area = self.proj_constants.MIP_ERA + self.target_mip = self.proj_constants.TARGET_MIP + + # Attributes that are going to be retrieved / set within this class for + # (all) + # (climate model inputs) + self.biomass_vars: list[str] = [] + self.meta_vars_percentage: list[str] = [] + self.meta_vars_share: list[str] = [] + + self._handle_emission_variables() + + def _handle_emission_variables(self): + self._generate_raw_emission_vars() + self._generate_plain_emission_vars() + self.logger.info(f"Emission variables to download: {self.variables}") + if self.download_biomass_burning: + self.logger.info(f"Biomass burning vars to download: {self.biomass_vars}") + if self.download_metafiles: + self.logger.info( + f"Meta emission vars to download:\n\t{self.meta_vars_percentage}\n\t{self.meta_vars_share}" + ) + + def _generate_raw_emission_vars(self): + variables = copy.deepcopy(self.variables) + if variables is None: + raise ValueError("No variables have been given to the downloader. Variables must be given for downloader.") + self.variables = [v.replace(" ", "_").replace("-", "_") for v in variables] + self.logger.info(f"Cleaned variables : {self.variables}") + + def _generate_plain_emission_vars(self): + if self.use_plain_emission_vars: + # plain vars are biomass vars + self.biomass_vars = self.variables + self.variables = [ + variable + emission_ending for variable in self.variables for emission_ending in self.emissions_endings + ] + # be careful with CO2 + if "CO2_em_openburning" in self.variables: + self.variables.remove("CO2_em_openburning") + else: + # get plain input4mips vars = biomass vars for historical + self.biomass_vars = list({v.split("_")[0] for v in self.variables}) + # remove biomass vars from normal vars list + for b in self.biomass_vars: + try: + self.variables.remove(b) + except ValueError as error: + self.logger.warning(f"Caught the following exception but continuing : {error}") + + self.meta_vars_percentage = [ + biomass_var + ending + for biomass_var in self.biomass_vars + if biomass_var != "CO2" + for ending in self.meta_endings_prc + ] + self.meta_vars_share = [ + biomass_var + ending + for biomass_var in self.biomass_vars + if biomass_var != "CO2" + for ending in self.meta_endings_share + ] + + +class CMIP6DownloaderConfig(BaseDownloaderConfig): + def __init__( + self, + project: str, + data_dir: str = RAW_DATA, + models: list[str] = None, + experiments: list[str] = None, + ensemble_members: list[str] = None, # preferred ensemble members used, if None not considered + max_ensemble_members: int = 10, # if -1 take all + variables: list[str] = None, + overwrite: bool = False, + logger: logging.Logger = LOGGER, + ): + super().__init__(project, data_dir, experiments, variables, overwrite, logger) + + if not models: + models = ["NorESM2-LM"] + if isinstance(models, str): + models = [models] + self.models: list[str] = models + self.avail_models = self.proj_constants.MODEL_SOURCES + self.ensemble_members: list[str] = ensemble_members + self.max_ensemble_members: int = max_ensemble_members + + self._validate_item_list(item_list=self.models, available_items=self.avail_models, name_of_item="model") + + +def _get_config_from_file(config_file, config_id, config_class, logger=LOGGER): + configs = get_yaml_config(config_file) + config_key = config_id + if config_key not in configs: + config_key = match_key_in_list(config_key, list(configs.keys())) + if not config_key: + logger.error(f"Config key [{config_id}] not found in config file [{config_file}]") + class_configs = configs[config_key] + config_object = config_class(project=config_id, **class_configs) + return config_object + + +def create_input4mips_downloader_config_from_file(config_file) -> Input4mipsDownloaderConfig: + config_object = _get_config_from_file(config_file, INPUT4MIPS, Input4mipsDownloaderConfig) + return config_object + + +def create_cmip6_downloader_config_from_file(config_file) -> CMIP6DownloaderConfig: + config_object = _get_config_from_file(config_file, CMIP6, CMIP6DownloaderConfig) + return config_object diff --git a/climateset/download/esgpull_utils.py b/climateset/download/esgpull_utils.py new file mode 100644 index 0000000..93947d6 --- /dev/null +++ b/climateset/download/esgpull_utils.py @@ -0,0 +1,278 @@ +import asyncio +import contextlib +import logging +import shutil +import uuid +from pathlib import Path +from typing import Generator + +from esgpull import Esgpull, Query +from esgpull.models import Options, Selection + +from climateset.download.constraints import CMIP6Constraints, Input4MIPsConstraints +from climateset.download.utils import _handle_ensemble_members +from climateset.utils import create_logger + +# Configure esgpull Selection to accept additional custom facets +Selection.configure("target_mip", "version", replace=False) + +LOGGER = create_logger(__name__) + + +@contextlib.contextmanager +def isolated_esgpull_context(raw_data_path: Path | str) -> Generator[Esgpull, None, None]: + """ + Context manager that creates a unique, isolated execution environment for esgpull to avoid file lock collisions and + pollution of the user's $HOME directory. + + Args: + raw_data_path: The base path for RAW_DATA where .esgpull_jobs will be created. + + Yields: + Esgpull: An isolated instance of Esgpull. + """ + if isinstance(raw_data_path, str): + raw_data_path = Path(raw_data_path) + + # Create a unique, isolated directory for this esgpull instance + # using a UUID to prevent collisions between parallel jobs. + unique_id = uuid.uuid4().hex + esgpull_jobs_dir = raw_data_path / ".esgpull_jobs" + isolated_path = esgpull_jobs_dir / unique_id + + # Ensure the parent directory exists + esgpull_jobs_dir.mkdir(parents=True, exist_ok=True) + + esg = None + try: + esg = Esgpull(path=isolated_path, install=True) + yield esg + finally: + # Tear down and safely purge the isolation folder and its SQLite DB + if isolated_path.exists(): + shutil.rmtree(isolated_path, ignore_errors=True) + + +def esgpull_search_and_download_esgf_raw_single_var( + esg: Esgpull, + variable: str, + institution_id: str, + project: str, + default_grid_label: str, + default_frequency: str, + preferred_version: str, + data_dir: Path, + distrib: bool = False, + logger: logging.Logger = LOGGER, +): + initial_constraints = Input4MIPsConstraints( + project=project, institution_id=institution_id, variable=variable + ).to_esgpull_query() + + query = Query(selection=initial_constraints) + query.options.distrib = distrib + + _apply_facet_fallback(esg, query, "grid_label", default_grid_label, logger) + _apply_facet_fallback(esg, query, "frequency", default_frequency, logger) + + # Esgpull handles multi-values natively. Fetch targets if any. + hints = esg.context.hints(query, file=False, facets=["target_mip"]) + if hints and "target_mip" in hints[0] and hints[0]["target_mip"]: + target_mips = list(hints[0]["target_mip"].keys()) + logger.info(f"Available target mips: {target_mips}") + query.selection["target_mip"] = target_mips + + _apply_version_fallback(esg, query, preferred_version, logger) + + files = esg.context.search(query, file=True) + logger.info(f"Result len: {len(files)}") + + dest_dir = Path(data_dir) / f"{project}/raw_input_vars/{institution_id}/{variable}" + return _download_and_move_files(esg, files, dest_dir, logger) + + +def esgpull_search_and_download_esgf_biomass_single_var( + esg: Esgpull, + variable: str, + variable_id: str, + institution_id: str, + project: str, + default_grid_label: str, + default_frequency: str, + preferred_version: str, + data_dir: Path, + distrib: bool = False, + logger: logging.Logger = LOGGER, +): + initial_constraints = Input4MIPsConstraints( + project=project, + institution_id=institution_id, + variable=variable, + variable_id=variable_id, + ).to_esgpull_query() + + query = Query(selection=initial_constraints) + query.options.distrib = distrib + + _apply_facet_fallback(esg, query, "grid_label", default_grid_label, logger) + _apply_facet_fallback(esg, query, "frequency", default_frequency, logger) + _apply_version_fallback(esg, query, preferred_version, logger) + + files = esg.context.search(query, file=True) + logger.info(f"Result len: {len(files)}") + + dest_dir = Path(data_dir) / f"{project}/meta_vars/{institution_id}/{variable}" + logger.info(f"Destination folder: [{dest_dir}]") + return _download_and_move_files(esg, files, dest_dir, logger) + + +def esgpull_search_and_download_esgf_model_single_var( + esg: Esgpull, + model: str, + variable: str, + experiment: str, + project: str, + default_grid_label: str, + default_frequency: str, + preferred_version: str, + max_ensemble_members: int, + ensemble_members: list[str], + data_dir: Path, + distrib: bool = False, + logger: logging.Logger = LOGGER, +): + cmip_constraints = CMIP6Constraints( + project=project, experiment_id=experiment, source_id=model, variable=variable + ).to_esgpull_query() + + query = Query(selection=cmip_constraints) + query.options.distrib = distrib + + _apply_facet_fallback(esg, query, "frequency", default_frequency, logger) + _apply_facet_fallback(esg, query, "grid_label", default_grid_label, logger) + + ensemble_member_final_list = _apply_variants_filter(esg, query, max_ensemble_members, ensemble_members, logger) + if not ensemble_member_final_list: + logger.info("No items were found for this request.") + return None + + # Esgpull supports multi-value list queries seamlessly + query.selection["variant_label"] = ensemble_member_final_list + _apply_version_fallback(esg, query, preferred_version, logger) + + files = esg.context.search(query, file=True) + logger.info(f"Result len {len(files)}") + + dest_dir = Path(data_dir) / f"{project}/{model}/{variable}" + return _download_and_move_files(esg, files, dest_dir, logger) + + +def _download_and_move_files(esg, files, dest_dir: Path, logger: logging.Logger = LOGGER): + """Downloads tracked files natively via esgpull (asyncio) and moves them from the isolated cache to the final + requested target directory.""" + if not files: + logger.info("No files to download.") + return [] + + unique_files = {} + for f in files: + if f.file_id not in unique_files: + unique_files[f.file_id] = f + files_to_add = list(unique_files.values()) + + logger.info(f"Adding {len(files_to_add)} files to esgpull DB...") + # Add tracked files to the isolated internal DB queue + esg.db.add(*files_to_add) + + async def _run_download(): + logger.info("Starting esg.download async...") + return await esg.download(files_to_add, show_progress=False) + + logger.info("Executing asyncio.run(_run_download())...") + # Execute async native download + downloaded, errors = asyncio.run(_run_download()) + logger.info(f"Download complete. Downloaded: {len(downloaded)}, Errors: {len(errors)}") + + if errors: + for err in errors: + logger.error(f"Download error: {err}") + + # Transfer from cache to strictly formatted project tree + if isinstance(dest_dir, str): + dest_dir = Path(dest_dir) + + dest_dir.mkdir(parents=True, exist_ok=True) + + moved_files = [] + data_cache_dir = esg.config.paths.data + if data_cache_dir.exists(): + for nc_file in data_cache_dir.rglob("*.nc"): + dest_file = dest_dir / nc_file.name + logger.info(f"Moving {nc_file.name} to {dest_dir}") + shutil.move(str(nc_file), str(dest_file)) + moved_files.append(dest_file) + + return moved_files + + +def _apply_facet_fallback( + esg, query: Query, facet_name: str, preferred_value: str | None, logger: logging.Logger = LOGGER +): + """Query the available facets and fall back if preferred_value is not found.""" + hints = esg.context.hints(query, file=False, facets=[facet_name]) + if hints and facet_name in hints[0] and hints[0][facet_name]: + available_facets = list(hints[0][facet_name].keys()) + logger.info(f"Available {facet_name}: {available_facets}") + + if preferred_value and preferred_value in available_facets: + logger.info(f"Choosing {facet_name}: {preferred_value}") + query.selection[facet_name] = [preferred_value] + else: + if preferred_value: + logger.warning(f"Preferred {facet_name} '{preferred_value}' not available.") + chosen = available_facets[0] + logger.info(f"Choosing {facet_name} {chosen} instead.") + query.selection[facet_name] = [chosen] + else: + logger.warning(f"No {facet_name} found.") + + +def _apply_version_fallback(esg, query: Query, preferred_version: str | None, logger: logging.Logger = LOGGER): + if preferred_version == "latest" or preferred_version is None: + # Use latest=True in options. Since Options is an Enum-backed mapped model, + # we can recreate it preserving the existing distrib option + is_distrib = query.options.distrib.name == "true" + query.options = Options(distrib=is_distrib, latest=True) + logger.info("Choosing latest version.") + else: + hints = esg.context.hints(query, file=False, facets=["version"]) + if hints and "version" in hints[0] and hints[0]["version"]: + available_versions = list(hints[0]["version"].keys()) + if preferred_version in available_versions: + query.selection["version"] = [preferred_version] + else: + logger.warning( + f"Preferred version {preferred_version} does not exist. " + f"Resuming with latest {available_versions[0]}" + ) + query.selection["version"] = [available_versions[0]] + + +def _apply_variants_filter( + esg, + query: Query, + max_ensemble_members: int, + ensemble_members: list[str], + logger: logging.Logger = LOGGER, +) -> list[str]: + hints = esg.context.hints(query, file=False, facets=["variant_label"]) + if not hints or "variant_label" not in hints[0] or not hints[0]["variant_label"]: + return [] + + variants = list(hints[0]["variant_label"].keys()) + logger.info(f"Available variants : {variants}\nLength : {len(variants)}") + + ensemble_members_list = _handle_ensemble_members( + variants=variants, ensemble_members=ensemble_members, max_ensemble_members=max_ensemble_members, logger=logger + ) + return ensemble_members_list diff --git a/climateset/download/input4mips_downloader.py b/climateset/download/input4mips_downloader.py new file mode 100644 index 0000000..36e08f3 --- /dev/null +++ b/climateset/download/input4mips_downloader.py @@ -0,0 +1,242 @@ +from pathlib import Path + +from esgpull import Esgpull + +from climateset.download.abstract_downloader import AbstractDownloader +from climateset.download.constants.esgf import INPUT4MIPS +from climateset.download.downloader_config import ( + Input4mipsDownloaderConfig, + create_input4mips_downloader_config_from_file, +) +from climateset.download.esgpull_utils import ( + esgpull_search_and_download_esgf_biomass_single_var, + esgpull_search_and_download_esgf_raw_single_var, + isolated_esgpull_context, +) +from climateset.download.utils import ( + search_and_download_esgf_biomass_single_var, + search_and_download_esgf_raw_single_var, +) +from climateset.utils import create_logger + +LOGGER = create_logger(__name__) + + +class Input4MipsDownloader(AbstractDownloader): + def __init__(self, config: Input4mipsDownloaderConfig): + self.config: Input4mipsDownloaderConfig = config + self.logger = LOGGER + + def download(self): + for variable in self.config.variables: + if variable.endswith("openburning"): + institution_id = "IAMC" + else: + institution_id = "PNNL-JGCRI" + self.logger.info(f"Downloading data for variable: {variable}") + self.download_raw_input_single_var(variable=variable, institution_id=institution_id) + + if self.config.download_biomass_burning and ("historical" in self.config.experiments): + for variable in self.config.biomass_vars: + self.logger.info(f"Downloading biomassburing data for variable: {variable}") + self.download_raw_input_single_var(variable=variable, institution_id="VUA") + + if self.config.download_metafiles: + for variable in self.config.meta_vars_percentage: + # percentage are historic and have no scenarios + self.logger.info(f"Downloading meta percentage data for variable: {variable}") + self.download_meta_historic_biomassburning_single_var(variable=variable, institution_id="VUA") + for variable in self.config.meta_vars_share: + self.logger.info(f"Downloading meta openburning share data for variable: {variable}") + self.download_raw_input_single_var(variable=variable, institution_id="IAMC") + + def download_raw_input_single_var( + self, + variable: str, + project: str = INPUT4MIPS, + institution_id: str = "PNNL-JGCRI", + default_frequency: str = "mon", + preferred_version: str = "latest", + default_grid_label: str = "gn", + ): + """ + Function handling the download of all input4mips data associated with a single variable. + + Args: + variable: variable ID + project: umbrella project, here "input4MIPs" + institution_id: id of the institution that provides the data + default_frequency: default frequency to download + preferred_version: data upload version, if 'latest', the newest version will get selected always + default_grid_label: default gridding method in which the data is provided + """ + self.logger.info("Using download_raw_input_single_var() function") + + # Search context is sensitive to order and sequence, which is why + # it's done in different steps instead of putting everything in `new_context` + results_list = search_and_download_esgf_raw_single_var( + variable=variable, + project=project, + institution_id=institution_id, + default_grid_label=default_grid_label, + default_frequency=default_frequency, + preferred_version=preferred_version, + data_dir=self.config.data_dir, + ) + self.logger.info(f"Download results: {results_list}") + + def download_meta_historic_biomassburning_single_var( + self, + variable: str, + institution_id: str, + project: str = INPUT4MIPS, + default_grid_label: str = "gn", + default_frequency: str = "mon", + preferred_version: str = "latest", + ): + """ + Function handling the download of all metadata associated with a single input4mips variable. + + Args: + variable: variable ID + project: umbrella project + institution_id: id of the institution that provides the data + default_grid_label: default gridding method in which the data is provided + default_frequency: default frequency to download + preferred_version: data upload version, if 'latest', the newest version will get selected always + """ + variable_id = variable.replace("_", "-") + variable_search = f"percentage_{variable_id.replace('-', '_').split('_')[-1]}" + self.logger.info(f"{variable}, {variable_id}, {institution_id}") + + # Search context is sensitive to order and sequence, which is why + # it's done in different steps instead of putting everything in `new_context` + results = search_and_download_esgf_biomass_single_var( + variable=variable_search, + variable_id=variable_id, + project=project, + institution_id=institution_id, + default_grid_label=default_grid_label, + default_frequency=default_frequency, + preferred_version=preferred_version, + base_path=self.config.data_dir, + ) + self.logger.info(f"Download results: {results}") + + +class Input4MipsDownloaderV2(AbstractDownloader): + def __init__(self, config: Input4mipsDownloaderConfig, distrib: bool = True): + self.config: Input4mipsDownloaderConfig = config + self.distrib = distrib + self.logger = LOGGER + + def download(self): + with isolated_esgpull_context(self.config.data_dir) as esg: + for variable in self.config.variables: + if variable.endswith("openburning"): + institution_id = "IAMC" + else: + institution_id = "PNNL-JGCRI" + self.logger.info(f"Downloading data for variable: {variable}") + self.download_raw_input_single_var(esg=esg, variable=variable, institution_id=institution_id) + + if self.config.download_biomass_burning and ("historical" in self.config.experiments): + for variable in self.config.biomass_vars: + self.logger.info(f"Downloading biomassburing data for variable: {variable}") + self.download_raw_input_single_var(esg=esg, variable=variable, institution_id="VUA") + + if self.config.download_metafiles: + for variable in self.config.meta_vars_percentage: + # percentage are historic and have no scenarios + self.logger.info(f"Downloading meta percentage data for variable: {variable}") + self.download_meta_historic_biomassburning_single_var( + esg=esg, variable=variable, institution_id="VUA" + ) + for variable in self.config.meta_vars_share: + self.logger.info(f"Downloading meta openburning share data for variable: {variable}") + self.download_raw_input_single_var(esg=esg, variable=variable, institution_id="IAMC") + + def download_raw_input_single_var( + self, + esg: Esgpull, + variable: str, + project: str = INPUT4MIPS, + institution_id: str = "PNNL-JGCRI", + default_frequency: str = "mon", + preferred_version: str = "latest", + default_grid_label: str = "gn", + ): + """ + Function handling the download of all input4mips data associated with a single variable. + + Args: + variable: variable ID + project: umbrella project, here "input4MIPs" + institution_id: id of the institution that provides the data + default_frequency: default frequency to download + preferred_version: data upload version, if 'latest', the newest version will get selected always + default_grid_label: default gridding method in which the data is provided + """ + self.logger.info("Using download_raw_input_single_var() function") + + # Search context is sensitive to order and sequence, which is why + # it's done in different steps instead of putting everything in `new_context` + results_list = esgpull_search_and_download_esgf_raw_single_var( + esg=esg, + variable=variable, + project=project, + institution_id=institution_id, + default_grid_label=default_grid_label, + default_frequency=default_frequency, + preferred_version=preferred_version, + data_dir=Path(self.config.data_dir), + distrib=self.distrib, + ) + self.logger.info(f"Download results: {results_list}") + + def download_meta_historic_biomassburning_single_var( + self, + esg: Esgpull, + variable: str, + institution_id: str, + project: str = INPUT4MIPS, + default_grid_label: str = "gn", + default_frequency: str = "mon", + preferred_version: str = "latest", + ): + """ + Function handling the download of all metadata associated with a single input4mips variable. + + Args: + variable: variable ID + project: umbrella project + institution_id: id of the institution that provides the data + default_grid_label: default gridding method in which the data is provided + default_frequency: default frequency to download + preferred_version: data upload version, if 'latest', the newest version will get selected always + """ + variable_id = variable.replace("_", "-") + variable_search = f"percentage_{variable_id.replace('-', '_').split('_')[-1]}" + self.logger.info(f"{variable}, {variable_id}, {institution_id}") + + # Search context is sensitive to order and sequence, which is why + # it's done in different steps instead of putting everything in `new_context` + results = esgpull_search_and_download_esgf_biomass_single_var( + esg=esg, + variable=variable_search, + variable_id=variable_id, + project=project, + institution_id=institution_id, + default_grid_label=default_grid_label, + default_frequency=default_frequency, + preferred_version=preferred_version, + data_dir=Path(self.config.data_dir), + distrib=self.distrib, + ) + self.logger.info(f"Download results: {results}") + + +def input4mips_download_from_config(config): + config_object = create_input4mips_downloader_config_from_file(config) + downloader = Input4MipsDownloaderV2(config=config_object) + downloader.download() diff --git a/climateset/download/utils.py b/climateset/download/utils.py index 4adc8e7..7998d78 100644 --- a/climateset/download/utils.py +++ b/climateset/download/utils.py @@ -1,14 +1,18 @@ import logging -import pathlib import re import subprocess import time -from typing import Union +from pathlib import Path -import pandas as pd import xarray as xr -from climateset import APP_ROOT, RAW_DATA +from climateset import RAW_DATA +from climateset.download.client import SearchClient, SearchSession +from climateset.download.constraints import ( + BaseSearchConstraints, + CMIP6Constraints, + Input4MIPsConstraints, +) from climateset.utils import create_logger LOGGER = create_logger(__name__) @@ -31,7 +35,8 @@ def extract_target_mip_exp_name(filename: str, target_mip: str, logger: logging. if "covid" in filename: experiment = f"{experiment}_covid" elif target_mip == "CMIP": - if int(year_end) > 2015: + cutoff_year_for_historical = 2015 + if int(year_end) > cutoff_year_for_historical: logger.info(f"TARGET MIP : {filename}") experiment = f"ssp{filename.split('ssp')[-1][:3]}" else: @@ -49,32 +54,35 @@ def extract_target_mip_exp_name(filename: str, target_mip: str, logger: logging. return experiment -def get_nominal_resolution(context, logger: logging.Logger = LOGGER): +def get_nominal_resolution(session, logger: logging.Logger = LOGGER): """ + Get nominal resolution from the search session's available facets. Args: - context: - logger: + session: climateset.download.client.SearchSession + logger: Logger instance Returns: - + str: Selected nominal resolution """ nominal_resolution = "" - nominal_resolution_list = [] - if "nominal_resolution" in context.facet_counts: - nominal_resolution_list = list(context.facet_counts["nominal_resolution"]) + nominal_resolution_list = session.get_available_facets("nominal_resolution") + + if nominal_resolution_list: logger.info(f"Available nominal resolution : {nominal_resolution_list}") - if not nominal_resolution_list: + else: logger.warning("No nominal resolution") return nominal_resolution + if len(nominal_resolution_list) > 1: logger.warning("Multiple nominal resolutions exist, will try to get smallest resolution.") + nominal_resolution = nominal_resolution_list[0] logger.info(f"Choosing nominal resolution : {nominal_resolution}") return nominal_resolution -def infer_nominal_resolution(ds: xr.Dataset, nominal_resolution: str, logger: logging.Logger = LOGGER) -> str: +def infer_nominal_resolution(ds: xr.Dataset, nominal_resolution: str, logger: logging.Logger = LOGGER) -> str | int: """ This method checks if there really is not nominal resolution by trying to compute it from the longitude increment. @@ -94,7 +102,7 @@ def infer_nominal_resolution(ds: xr.Dataset, nominal_resolution: str, logger: lo degree = abs(ds.lon[0].item() - ds.lon[1].item()) nom_res = int(degree * 100) logger.info(f"Inferring nominal resolution: {nom_res}") - except Exception as error: + except Exception as error: # pylint: disable=W0718 logger.warning(f"Caught the following exception but continuing : {error}") return nom_res @@ -158,40 +166,48 @@ def _download_process(temp_download_path, search_results, logger: logging.Logger _download_result(result=result, download_path=temp_download_path, logger=logger) -def download_raw_input_variable( - institution_id, search_results, variable, base_path: Union[str, pathlib.Path] = RAW_DATA -): +def download_raw_input_variable(project, institution_id, search_results, variable, base_path: str | Path = RAW_DATA): if isinstance(base_path, str): - base_path = pathlib.Path(base_path) - temp_download_path = base_path / f"raw_input_vars/{institution_id}/{variable}" + base_path = Path(base_path) + temp_download_path = base_path / f"{project}/raw_input_vars/{institution_id}/{variable}" _download_process(temp_download_path, search_results) -def download_model_variable(model_id, search_results, variable, base_path: Union[str, pathlib.Path] = RAW_DATA): +def download_model_variable(project, model_id, search_results, variable, base_path: str | Path = RAW_DATA): if isinstance(base_path, str): - base_path = pathlib.Path(base_path) - temp_download_path = base_path / f"model_vars/{model_id}/{variable}" + base_path = Path(base_path) + temp_download_path = base_path / f"{project}/{model_id}/{variable}" _download_process(temp_download_path, search_results) -def download_metadata_variable( - institution_id, search_results, variable, base_path: Union[str, pathlib.Path] = RAW_DATA -): +def download_metadata_variable(project, institution_id, search_results, variable, base_path: str | Path = RAW_DATA): if isinstance(base_path, str): - base_path = pathlib.Path(base_path) - temp_download_path = base_path / f"meta_vars/{institution_id}/{variable}" + base_path = Path(base_path) + temp_download_path = base_path / f"{project}/meta_vars/{institution_id}/{variable}" _download_process(temp_download_path, search_results) -def get_grid_label(context, default_grid_label, logger=LOGGER): +def get_grid_label(session, default_grid_label, logger=LOGGER): + """ + Get grid label from the search session. + + Args: + session: climateset.download.client.SearchSession + default_grid_label: Default grid label to use if available + logger: Logger instance + + Returns: + str: Selected grid label + """ grid_label = "" - grid_label_list = [] - if "grid_label" in context.facet_counts: - grid_label_list = list(context.facet_counts["grid_label"]) + grid_label_list = session.get_available_facets("grid_label") + + if grid_label_list: logger.info(f"Available grid labels : {grid_label_list}") - if not grid_label_list: + else: logger.warning("No grid labels found") return grid_label + if default_grid_label and default_grid_label in grid_label_list: logger.info(f"Choosing grid : {default_grid_label}") grid_label = default_grid_label @@ -202,33 +218,25 @@ def get_grid_label(context, default_grid_label, logger=LOGGER): return grid_label -def get_max_ensemble_member_number(df_model_source: pd.DataFrame, experiments: list[str], model: str, logger=LOGGER): - if model is not None: - if model not in df_model_source["source_id"].tolist(): - logger.info(f"Model {model} not supported.") - raise AttributeError - model_id = df_model_source.index[df_model_source["source_id"] == model].values - # get ensemble members per scenario - max_ensemble_members_list = df_model_source["num_ensemble_members"][model_id].values.tolist()[0].split(" ") - scenarios = df_model_source["scenarios"][model_id].values.tolist()[0].split(" ") - max_ensemble_members_lookup = {} - for s, m in zip(scenarios, max_ensemble_members_list): - max_ensemble_members_lookup[s] = int(m) - filtered_experiments = (e for e in experiments if e != "historical") - max_possible_member_number = min( - max_ensemble_members_lookup[e] for e in filtered_experiments - ) # TODO fix historical - return max_possible_member_number - - -def get_upload_version(context, preferred_version, logger=LOGGER): +def get_upload_version(session, preferred_version, logger=LOGGER): + """ + Get upload version from the search session. + + Args: + session: climateset.download.client.SearchSession + preferred_version: Preferred version ('latest' or specific) + logger: Logger instance + + Returns: + str: Selected version + """ version = "" - versions = [] - if "version" in context.facet_counts: - versions = list(context.facet_counts["version"]) + versions = session.get_available_facets("version") + if not versions: logger.warning("No versions are available. Skipping.") return version + logger.info(f"Available versions : {versions}") if preferred_version: if preferred_version == "latest": @@ -236,7 +244,10 @@ def get_upload_version(context, preferred_version, logger=LOGGER): logger.info(f"Choosing latest version: {version}") else: try: - version = versions[preferred_version] + if preferred_version in versions: + version = preferred_version + else: + raise KeyError except KeyError: logger.warning(f"Preferred version {preferred_version} does not exist.") version = versions[0] @@ -244,15 +255,27 @@ def get_upload_version(context, preferred_version, logger=LOGGER): return version -def get_frequency(context, default_frequency, logger=LOGGER): +def get_frequency(session, default_frequency, logger=LOGGER): + """ + Get frequency from the search session. + + Args: + session: climateset.download.client.SearchSession + default_frequency: Default frequency to use + logger: Logger instance + + Returns: + str: Selected frequency + """ frequency = "" - frequency_list = [] - if "frequency" in context.facet_counts: - frequency_list = list(context.facet_counts["frequency"]) + frequency_list = session.get_available_facets("frequency") + + if frequency_list: logger.info(f"Available frequencies : {frequency_list}") - if not frequency_list: + else: logger.warning("No frequencies are available. Skipping") return frequency + if default_frequency and default_frequency in frequency_list: frequency = default_frequency logger.info(f"Choosing default frequency : {frequency}") @@ -262,32 +285,311 @@ def get_frequency(context, default_frequency, logger=LOGGER): return frequency -def _handle_base_search_constraints(ctx, default_frequency, default_grid_label): - grid_label = get_grid_label(context=ctx, default_grid_label=default_grid_label) - if grid_label: - ctx = ctx.constrain(grid_label=grid_label) - nominal_resolution = get_nominal_resolution(context=ctx) - if nominal_resolution: - ctx = ctx.constrain(nominal_resolution=nominal_resolution) - frequency = get_frequency(context=ctx, default_frequency=default_frequency) - if frequency: - ctx = ctx.constrain(frequency=frequency) - return ctx - +def handle_yaml_config_path(config_file_name, config_path): + if isinstance(config_path, str): + config_path = Path(config_path) + if not config_file_name.endswith(".yaml"): + config_file_name = f"{config_file_name}.yaml" + config_full_path = config_path / config_file_name + return config_full_path + + +def match_key_in_list(input_key: str, key_list: list[str]) -> str | None: + for key in key_list: + if input_key.lower() == key.lower(): + return key + if input_key.upper() == key.upper(): + return key + return None + + +def search_and_download_esgf_raw_single_var( + variable: str, + institution_id: str, + project: str, + default_grid_label: str, + default_frequency: str, + preferred_version: str, + data_dir, + logger=LOGGER, +): + """ + Search and download raw input variables using SearchClient. -def get_select_model_scenarios(path_to_file: Union[str, pathlib.Path] = None) -> pd.DataFrame: + Follows iterative constraint strategy. """ - This function returns a dataframe based on input Json file. + # Use distrib=False to match original behavior and avoid potential distributed search issues/warnings + with SearchClient(distrib=False) as client: + try: + session = client.new_session() - Args: - path_to_file: Path to Json file + initial_constraints = Input4MIPsConstraints( + project=project, institution_id=institution_id, variable=variable + ) + session.constrain(initial_constraints) + + # 1. Grid Label + grid_label = get_grid_label(session, default_grid_label, logger) + if grid_label: + session.constrain(BaseSearchConstraints(grid_label=grid_label)) + + # 2. Nominal Resolution + nominal_resolution = get_nominal_resolution(session, logger) + if nominal_resolution: + session.constrain(BaseSearchConstraints(nominal_resolution=nominal_resolution)) + + # 3. Frequency + frequency = get_frequency(session, default_frequency, logger) + if frequency: + session.constrain(BaseSearchConstraints(frequency=frequency)) + + # 4. Target MIP (Iterative) + mips_targets = session.get_available_facets("target_mip") + logger.info(f"Available target mips: {mips_targets}") + + results_list = [] + + base_constraints = [ + initial_constraints, + BaseSearchConstraints(grid_label=grid_label) if grid_label else BaseSearchConstraints(), + ( + BaseSearchConstraints(nominal_resolution=nominal_resolution) + if nominal_resolution + else BaseSearchConstraints() + ), + BaseSearchConstraints(frequency=frequency) if frequency else BaseSearchConstraints(), + ] + + for target in mips_targets: + logger.debug(f"Checking target mip: {target}") + sub_session = client.new_session() + # Replay base constraints + for c in base_constraints: + sub_session.constrain(c) + + sub_session.constrain(Input4MIPsConstraints(target_mip=target)) + + version = get_upload_version(sub_session, preferred_version, logger) + if version: + sub_session.constrain(BaseSearchConstraints(version=version)) + + results = sub_session.search() + logger.info(f"Result len for target {target}: {len(results)}") + if results: + results_list.append(results) + + if results_list: + for r in results_list: + download_raw_input_variable( + project=project, + institution_id=institution_id, + search_results=r, + variable=variable, + base_path=data_dir, + ) + return results_list + logger.error("Could not find anything for configured nodes") + + except Exception as e: + logger.error(f"Error during search/download: {e}") + raise e + + raise RuntimeError("Could not find anything for all urls") + + +def search_and_download_esgf_biomass_single_var( + variable: str, + variable_id: str, + institution_id: str, + project: str, + default_grid_label: str, + default_frequency: str, + preferred_version: str, + base_path: Path, + logger=LOGGER, +): + with SearchClient(distrib=False) as client: + try: + session = client.new_session() - Returns: - Dataframe - """ - if not path_to_file: - path_to_file = APP_ROOT / "download/constants/selected_scenariosMIPs.json" - if isinstance(path_to_file, str): - path_to_file = pathlib.Path(path_to_file) - selected_scenarios = pd.read_json(path_to_file, orient="records") - return selected_scenarios + initial_constraints = Input4MIPsConstraints( + project=project, institution_id=institution_id, variable=variable, variable_id=variable_id + ) + session.constrain(initial_constraints) + + # 1. Grid Label + grid_label = get_grid_label(session, default_grid_label, logger) + if grid_label: + session.constrain(BaseSearchConstraints(grid_label=grid_label)) + + # 2. Frequency + frequency = get_frequency(session, default_frequency, logger) + if frequency: + session.constrain(BaseSearchConstraints(frequency=frequency)) + + # 3. Version + version = get_upload_version(session, preferred_version, logger) + if version: + session.constrain(BaseSearchConstraints(version=version)) + + results = session.search() + logger.info(f"Result len {len(results)}") + + if results: + try: + logger.info(results[0].file_context()) + except Exception: # pylint: disable=broad-exception-caught + pass + + download_metadata_variable( + project=project, + institution_id=institution_id, + search_results=results, + variable=variable, + base_path=base_path, + ) + return results + logger.error("Could not find anything for configured nodes") + + except Exception as e: # pylint: disable=broad-exception-caught + logger.error(f"Error: {e}") + + raise RuntimeError("Could not find anything for all urls") + + +def _handle_ensemble_members( + variants: list, ensemble_members: list[str], max_ensemble_members: int, logger: logging.Logger +) -> list[str]: + if not ensemble_members: + if max_ensemble_members > len(variants): + logger.info("Less ensemble members available than maximum number desired. Including all variants.") + return variants + logger.info( + f"{len(variants)} ensemble members available, desired (max {max_ensemble_members}). " + f"Choosing only the first {max_ensemble_members}." + ) + return variants[:max_ensemble_members] + + logger.info(f"Desired list of ensemble members given: {ensemble_members}") + ensemble_member_final_list = list(set(variants) & set(ensemble_members)) + return ensemble_member_final_list + + +def _get_variants_and_filter( + session: SearchSession, max_ensemble_members: int, ensemble_members: list[str], logger: logging.Logger +) -> list[str]: + """Helper to retrieve and filter variant labels.""" + variants = session.get_available_facets("variant_label") + + if len(variants) < 1: + # Note: Previous code raised ValueError here but logging info first + return [] + + logger.info(f"Available variants : {variants}\n") + logger.info(f"Length : {len(variants)}") + + ensemble_members_list = _handle_ensemble_members( + variants=variants, ensemble_members=ensemble_members, max_ensemble_members=max_ensemble_members, logger=logger + ) + return ensemble_members_list + + +def search_and_download_esgf_model_single_var( + model: str, + variable: str, + experiment: str, + project: str, + default_grid_label: str, + default_frequency: str, + preferred_version: str, + max_ensemble_members: int, + ensemble_members: list[str], + base_path: Path, + logger=LOGGER, +): + logger.info("Using download_from_model_single_var() function") + + with SearchClient() as client: + try: + session = client.new_session() + + cmip_constraints = CMIP6Constraints( + project=project, experiment_id=experiment, source_id=model, variable=variable + ) + session.constrain(cmip_constraints) + + # 1. Frequency + frequency = get_frequency(session, default_frequency, logger) + if frequency: + session.constrain(BaseSearchConstraints(frequency=frequency)) + + # 2. Grid Label + grid_label = get_grid_label(session, default_grid_label, logger) + if grid_label: + session.constrain(BaseSearchConstraints(grid_label=grid_label)) + + # 3. Variants (Ensemble Members) + ensemble_member_final_list = _get_variants_and_filter( + session, max_ensemble_members, ensemble_members, logger + ) + + if not ensemble_member_final_list: + logger.info( + "No items were found for this request. Please check on the esgf server if the combination of your " + "model/scenarios/variables exists." + ) + if not ensemble_members and len(session.get_available_facets("variant_label")) < 1: + # Replicate original ValueError text + raise ValueError( + f"Downloader did not find any items on esgf for your request with: Project {project}, " + f"Experiment {experiment}, Model {model}, Variable {variable}." + ) + logger.info("WARNING: no overlap between available and desired ensemble members!") + logger.info("Skipping.") + return None + + results_list = [] + + base_constraints_list = [ + cmip_constraints, + BaseSearchConstraints(frequency=frequency) if frequency else BaseSearchConstraints(), + BaseSearchConstraints(grid_label=grid_label) if grid_label else BaseSearchConstraints(), + ] + + for ensemble_member in ensemble_member_final_list: + logger.info(f"Ensembles member: {ensemble_member}") + + sub_session = client.new_session() + for c in base_constraints_list: + sub_session.constrain(c) + + sub_session.constrain(CMIP6Constraints(variant_label=ensemble_member)) + + version = get_upload_version(sub_session, preferred_version, logger) + if version: + sub_session.constrain(BaseSearchConstraints(version=version)) + + results = sub_session.search() + if results: + results_list.append(results) + logger.info(f"Result len {len(results)}") + + logger.info(results_list) + if results_list: + for results in results_list: + download_model_variable( + project=project, + model_id=model, + search_results=results, + variable=variable, + base_path=base_path, + ) + return results_list + logger.error("Could not find anything for configured nodes") + + except ValueError: + raise + except Exception as e: # pylint: disable=broad-exception-caught + logger.error(f"Error: {e}") + + raise RuntimeError("Could not find anything for all urls") diff --git a/climateset/utils.py b/climateset/utils.py index d93464b..720a8b1 100644 --- a/climateset/utils.py +++ b/climateset/utils.py @@ -1,7 +1,6 @@ import logging import pathlib import sys -from typing import Union import yaml @@ -56,7 +55,7 @@ def get_mip(experiment: str): return "CMIP" -def get_yaml_config(yaml_config_file: Union[str, pathlib.Path], logger: logging.Logger = LOGGER) -> dict: +def get_yaml_config(yaml_config_file: str | pathlib.Path, logger: logging.Logger = LOGGER) -> dict: """ Reads a YAML configuration file and returns its contents as a dictionary. @@ -95,7 +94,7 @@ def get_yaml_config(yaml_config_file: Union[str, pathlib.Path], logger: logging. for path in potential_paths: if path.exists(): config_filepath = path - logger.info(f"Yaml config file [{str(path)}] found.") + logger.info(f"Yaml config file [{path!s}] found.") break params = {} diff --git a/configs/core_dataset.yaml b/configs/core_dataset.yaml index e6981f5..21ad438 100644 --- a/configs/core_dataset.yaml +++ b/configs/core_dataset.yaml @@ -1,4 +1,7 @@ -models: ["NorESM2-LM"] -downloader_kwargs: - variables: ["CO2", "BC", "SO2", "CH4", "tas", "pr"] - experiments: ["historical","ssp126", "ssp245", "ssp370", "ssp585"] \ No newline at end of file +CMIP6: + model: "NorESM2-LM" + variables: [ "tas", "pr" ] + experiments: [ "historical","ssp126", "ssp245", "ssp370", "ssp585" ] +input4MIPs: + variables: [ "CO2", "BC", "SO2", "CH4" ] + experiments: [ "historical","ssp126", "ssp245", "ssp370", "ssp585" ] diff --git a/configs/downloader/cmip6/awi_sea-ice-thickness_control-1950.yaml b/configs/downloader/cmip6/awi_sea-ice-thickness_control-1950.yaml index db0c390..7a323b7 100644 --- a/configs/downloader/cmip6/awi_sea-ice-thickness_control-1950.yaml +++ b/configs/downloader/cmip6/awi_sea-ice-thickness_control-1950.yaml @@ -1,4 +1,4 @@ -models: ["AWI-CM-1-1-HR"] -downloader_kwargs: - variables: ["sithick"] - experiments: ["control-1950"] \ No newline at end of file +CMIP6: + models: [ "AWI-CM-1-1-HR" ] + variables: [ "sithick" ] + experiments: [ "control-1950" ] \ No newline at end of file diff --git a/configs/downloader/cmip6/canesm_co2_ssp.yaml b/configs/downloader/cmip6/canesm_co2_ssp.yaml index fc373d0..ebe7823 100644 --- a/configs/downloader/cmip6/canesm_co2_ssp.yaml +++ b/configs/downloader/cmip6/canesm_co2_ssp.yaml @@ -1,4 +1,4 @@ -models: ["CanESM5"] -downloader_kwargs: - variables: ["co2"] - experiments: ["ssp245"] \ No newline at end of file +CMIP6: + models: ["CanESM5"] + variables: [ "co2" ] + experiments: [ "abrupt-2xCO2" ] \ No newline at end of file diff --git a/configs/downloader/cmip6/fgoals_tas_ssp.yaml b/configs/downloader/cmip6/fgoals_tas_ssp.yaml index cfd3eb1..dfec4c3 100644 --- a/configs/downloader/cmip6/fgoals_tas_ssp.yaml +++ b/configs/downloader/cmip6/fgoals_tas_ssp.yaml @@ -1,4 +1,4 @@ -models: ["FGOALS-g3"] -downloader_kwargs: - variables: ["tas"] - experiments: ["ssp245"] \ No newline at end of file +CMIP6: + models: ["FGOALS-g3"] + variables: [ "tas" ] + experiments: [ "ssp245" ] diff --git a/configs/downloader/cmip6/noresm_pr_historical.yaml b/configs/downloader/cmip6/noresm_pr_historical.yaml index 0498820..968a9e1 100644 --- a/configs/downloader/cmip6/noresm_pr_historical.yaml +++ b/configs/downloader/cmip6/noresm_pr_historical.yaml @@ -1,4 +1,5 @@ -models: ["NorESM2-LM"] -downloader_kwargs: - variables: ["pr"] - experiments: ["historical"] \ No newline at end of file +CMIP6: + models: ["NorESM2-LM"] + variables: [ "pr" ] + experiments: [ "historical" ] + max_ensemble_members: 1 \ No newline at end of file diff --git a/configs/downloader/cmip6/noresm_tas_ssp.yaml b/configs/downloader/cmip6/noresm_tas_ssp.yaml index e1e7238..c8f605c 100644 --- a/configs/downloader/cmip6/noresm_tas_ssp.yaml +++ b/configs/downloader/cmip6/noresm_tas_ssp.yaml @@ -1,4 +1,5 @@ -models: ["NorESM2-LM"] -downloader_kwargs: - variables: ["tas"] - experiments: ["ssp245"] \ No newline at end of file +CMIP6: + models: ["NorESM2-LM"] + variables: [ "tas" ] + experiments: [ "ssp245" ] + ensemble_members: [ "r9i1p1f2", "r8i1p1f2" ] \ No newline at end of file diff --git a/configs/downloader/cmip6/ukesm_tas_picontrol.yaml b/configs/downloader/cmip6/ukesm_tas_picontrol.yaml index 2b2a25d..60339d8 100644 --- a/configs/downloader/cmip6/ukesm_tas_picontrol.yaml +++ b/configs/downloader/cmip6/ukesm_tas_picontrol.yaml @@ -1,4 +1,4 @@ -models: ["UKESM1-0-LL"] -downloader_kwargs: - variables: ["tas"] - experiments: ["piControl"] +CMIP6: + models: ["UKESM1-0-LL"] + variables: [ "tas" ] + experiments: [ "piControl" ] diff --git a/configs/downloader/constants/cmip6.yaml b/configs/downloader/constants/cmip6.yaml new file mode 100644 index 0000000..374756c --- /dev/null +++ b/configs/downloader/constants/cmip6.yaml @@ -0,0 +1,916 @@ +node_link: "https://esgf-node.llnl.gov/esg-search/" + +model_sources: + - "4AOP-v1-5" + - "ACCESS-CM2" + - "ACCESS-ESM1-5" + - "ACCESS-OM2" + - "ACCESS-OM2-025" + - "ARTS-2-3" + - "AWI-CM-1-1-HR" + - "AWI-CM-1-1-LR" + - "AWI-CM-1-1-MR" + - "AWI-ESM-1-1-LR" + - "AWI-ESM-2-1-LR" + - "BCC-CSM2-HR" + - "BCC-CSM2-MR" + - "BCC-ESM1" + - "CAM-MPAS-HR" + - "CAM-MPAS-LR" + - "CAMS-CSM1-0" + - "CanESM5" + - "CanESM5-1" + - "CanESM5-CanOE" + - "CAS-ESM2-0" + - "CESM1-1-CAM5-CMIP5" + - "CESM1-CAM5-SE-HR" + - "CESM1-CAM5-SE-LR" + - "CESM1-WACCM-SC" + - "CESM2" + - "CESM2-FV2" + - "CESM2-WACCM" + - "CESM2-WACCM-FV2" + - "CIESM" + - "CMCC-CM2-HR4" + - "CMCC-CM2-SR5" + - "CMCC-CM2-VHR4" + - "CMCC-ESM2" + - "CNRM-CM6-1" + - "CNRM-CM6-1-HR" + - "CNRM-ESM2-1" + - "E3SM-1-0" + - "E3SM-1-1" + - "E3SM-1-1-ECA" + - "E3SM-2-0" + - "EC-Earth3" + - "EC-Earth3-AerChem" + - "EC-Earth3-CC" + - "EC-Earth3-GrIS" + - "EC-Earth3-HR" + - "EC-Earth3-LR" + - "EC-Earth3-Veg" + - "EC-Earth3-Veg-LR" + - "EC-Earth3P" + - "EC-Earth3P-HR" + - "EC-Earth3P-VHR" + - "ECMWF-IFS-HR" + - "ECMWF-IFS-LR" + - "ECMWF-IFS-MR" + - "FGOALS-f3-H" + - "FGOALS-f3-L" + - "FGOALS-g3" + - "FIO-ESM-2-0" + - "GFDL-AM4" + - "GFDL-CM4" + - "GFDL-CM4C192" + - "GFDL-ESM2M" + - "GFDL-ESM4" + - "GFDL-GRTCODE" + - "GFDL-OM4p5B" + - "GFDL-RFM-DISORT" + - "GISS-E2-1-G" + - "GISS-E2-1-G-CC" + - "GISS-E2-1-H" + - "GISS-E2-2-G" + - "GISS-E2-2-H" + - "GISS-E3-G" + - "HadGEM3-GC31-HH" + - "HadGEM3-GC31-HM" + - "HadGEM3-GC31-LL" + - "HadGEM3-GC31-LM" + - "HadGEM3-GC31-MH" + - "HadGEM3-GC31-MM" + - "HiRAM-SIT-HR" + - "HiRAM-SIT-LR" + - "ICON-ESM-LR" + - "IITM-ESM" + - "INM-CM4-8" + - "INM-CM5-0" + - "INM-CM5-H" + - "IPSL-CM5A2-INCA" + - "IPSL-CM6A-ATM-HR" + - "IPSL-CM6A-ATM-ICO-HR" + - "IPSL-CM6A-ATM-ICO-LR" + - "IPSL-CM6A-ATM-ICO-MR" + - "IPSL-CM6A-ATM-ICO-VHR" + - "IPSL-CM6A-ATM-LR-REPROBUS" + - "IPSL-CM6A-LR" + - "IPSL-CM6A-LR-INCA" + - "IPSL-CM6A-MR1" + - "KACE-1-0-G" + - "KIOST-ESM" + - "LBLRTM-12-8" + - "MCM-UA-1-0" + - "MIROC-ES2H" + - "MIROC-ES2H-NB" + - "MIROC-ES2L" + - "MIROC6" + - "MPI-ESM-1-2-HAM" + - "MPI-ESM1-2-HR" + - "MPI-ESM1-2-LR" + - "MPI-ESM1-2-XR" + - "MRI-AGCM3-2-H" + - "MRI-AGCM3-2-S" + - "MRI-ESM2-0" + - "NESM3" + - "NICAM16-7S" + - "NICAM16-8S" + - "NICAM16-9S" + - "NorCPM1" + - "NorESM1-F" + - "NorESM2-LM" + - "NorESM2-MH" + - "RRTMG-LW-4-91" + - "RRTMG-SW-4-02" + - "RTE-RRTMGP-181204" + - "SAM0-UNICON" + - "TaiESM1" + - "TaiESM1-TIMCOM" + - "TaiESM1-TIMCOM2" + - "UKESM1-0-LL" + - "UKESM1-1-LL" + - "UKESM1-ice-LL" + - "E3SM-2-0-NARRM" + - "E3SM-2-1" + - "EC-Earth3-ESM-1" + - "PCMDI-test-1-0" + +var_source_lookup: + - "ztp" + - "zsatcalc" + - "zsatarag" + - "zostoga" + - "zossq" + - "zos" + - "zoocos" + - "zooc" + - "zo2min" + - "zhalfo" + - "zg500" + - "zg1000" + - "zg100" + - "zg10" + - "zg" + - "zfullo" + - "wtd" + - "wo" + - "wmo" + - "wfonocorr" + - "wfo" + - "wetss" + - "wetso4" + - "wetso2" + - "wetlandFrac" + - "wetlandCH4" + - "wetbc" + - "wap500" + - "wap" + - "vsf" + - "volo" + - "volcello" + - "vo" + - "vmo" + - "vegHeight" + - "va" + - "uo" + - "umo" + - "ua" + - "tslsi" + - "tsl" + - "ts" + - "tran" + - "tossq" + - "tosga" + - "tos" + - "tob" + - "thkcello" + - "thetaot700" + - "thetaot300" + - "thetaot2000" + - "thetaot" + - "thetaoga" + - "thetao" + - "tgs" + - "tcs" + - "tauvo" + - "tauv" + - "tauuo" + - "tauu" + - "tasmin" + - "tasmax" + - "tas" + - "talkos" + - "talknat" + - "talk" + - "ta850" + - "ta700" + - "ta500" + - "ta" + - "t20d" + - "spco2" + - "sossq" + - "sosga" + - "sos" + - "sootsn" + - "somint" + - "soga" + - "sob" + - "so2" + - "so" + - "snw" + - "sndmasswindrif" + - "sndmasssnf" + - "sndmasssi" + - "sndmassmelt" + - "snd" + - "snc" + - "sivols" + - "sivoln" + - "sivol" + - "siv" + - "siu" + - "sitimefrac" + - "sithick" + - "sitemptop" + - "sitempsnic" + - "sitempbot" + - "sistryubot" + - "sistrydtop" + - "sistrxubot" + - "sistrxdtop" + - "sispeed" + - "sisnthick" + - "sisnmass" + - "sisnhc" + - "sisnconc" + - "sirdgthick" + - "sirdgconc" + - "sipr" + - "sios" + - "simpconc" + - "simass" + - "siitdthick" + - "siitdsnthick" + - "siitdsnconc" + - "siitdconc" + - "sihc" + - "siforcetilty" + - "siforcetiltx" + - "siforceintstry" + - "siforceintstrx" + - "siforcecorioly" + - "siforcecoriolx" + - "siflswutop" + - "siflswdtop" + - "siflswdbot" + - "siflsensupbot" + - "siflsenstop" + - "sifllwutop" + - "sifllwdtop" + - "sifllatstop" + - "siflfwdrain" + - "siflfwbot" + - "siflcondtop" + - "siflcondbot" + - "sifb" + - "siextents" + - "siextentn" + - "sidmasstrany" + - "sidmasstranx" + - "sidmassth" + - "sidmasssi" + - "sidmassmelttop" + - "sidmassmeltbot" + - "sidmasslat" + - "sidmassgrowthwat" + - "sidmassgrowthbot" + - "sidmassevapsubl" + - "sidmassdyn" + - "sidivvel" + - "sidconcth" + - "sidconcdyn" + - "siconc" + - "sicompstren" + - "siarean" + - "siage" + - "si" + - "sftof" + - "sftlf" + - "sftgif" + - "sfdsi" + - "sfcWind" + - "sf6" + - "rtmt" + - "rsutcsaf" + - "rsutcs" + - "rsutaf" + - "rsut" + - "rsuscs" + - "rsus" + - "rsntds" + - "rsdt" + - "rsdsdiff" + - "rsdscs" + - "rsds" + - "rlutcsaf" + - "rlutcs" + - "rlutaf" + - "rlut" + - "rlus" + - "rldscs" + - "rlds" + - "rh" + - "reffclwtop" + - "ra" + - "rMaint" + - "rGrowth" + - "qgwr" + - "pso" + - "psl" + - "ps" + - "prw" + - "prveg" + - "prsn" + - "prra" + - "prc" + - "pr" + - "ppos" + - "pp" + - "popos" + - "pop" + - "ponos" + - "pon" + - "po4os" + - "po4" + - "phynos" + - "phyn" + - "phyfeos" + - "phyfe" + - "phyc" + - "phos" + - "phnat" + - "phalf" + - "ph" + - "pfull" + - "pctisccp" + - "pbo" + - "orog" + - "opottempmint" + - "oh" + - "od870aer" + - "od550ss" + - "od550so4" + - "od550oa" + - "od550lt1aer" + - "od550dust" + - "od550csaer" + - "od550bc" + - "od550aerh2o" + - "od550aer" + - "od440aer" + - "obvfsq" + - "o3" + - "o2satos" + - "o2sat" + - "o2os" + - "o2min" + - "o2" + - "nppWood" + - "nppRoot" + - "nppLeaf" + - "npp" + - "no3os" + - "no3" + - "nep" + - "nbp" + - "nVeg" + - "nStem" + - "nSoil" + - "nRoot" + - "nMineralNO3" + - "nMineralNH4" + - "nMineral" + - "nLitter" + - "nLeaf" + - "nLand" + - "n2oglobal" + - "msftmzmpa" + - "msftmz" + - "msftmrhompa" + - "msftmrho" + - "msftbarot" + - "mrtws" + - "mrsos" + - "mrsol" + - "mrso" + - "mrsll" + - "mrsfl" + - "mrros" + - "mrrob" + - "mrro" + - "mrlso" + - "mrfso" + - "mmrss" + - "mmrsoa" + - "mmrso4" + - "mmrpm2p5" + - "mmrpm1" + - "mmroa" + - "mmrdust" + - "mmrbc" + - "mmraerh2o" + - "mlotstsq" + - "mlotstmin" + - "mlotstmax" + - "mlotst" + - "mfo" + - "masso" + - "masscello" + - "lwsnl" + - "lwp" + - "loadss" + - "loaddust" + - "lai" + - "isop" + - "intpp" + - "intpoc" + - "intpn2" + - "intdoc" + - "intdic" + - "huss" + - "hus" + - "hurs" + - "hur" + - "hfy" + - "hfx" + - "hfss" + - "hfls" + - "hfds" + - "hfbasinpmdiff" + - "hfbasinpmadv" + - "hfbasinpadv" + - "hfbasin" + - "gpp" + - "fsitherm" + - "froc" + - "frn" + - "friver" + - "fric" + - "frfe" + - "ficeberg" + - "fgo2" + - "fgdms" + - "fgco2nat" + - "fgco2" + - "fVegLitterSenescence" + - "fVegLitterMortality" + - "fVegLitter" + - "fNup" + - "fNnetmin" + - "fNloss" + - "fNleach" + - "fNgasNonFire" + - "fNgasFire" + - "fNgas" + - "fNfert" + - "fNdep" + - "fNProduct" + - "fNOx" + - "fN2O" + - "fLuc" + - "fLitterFire" + - "fHarvestToProduct" + - "fHarvest" + - "fFireNat" + - "fFire" + - "fDeforestToProduct" + - "fBNF" + - "evspsblveg" + - "evspsblsoi" + - "evspsbl" + - "evs" + - "esn" + - "es" + - "epsi100" + - "epp100" + - "epn100" + - "epfe100" + - "epcalc100" + - "epc100" + - "emivoc" + - "emiss" + - "emiso4" + - "emiso2" + - "emioa" + - "emiisop" + - "emidust" + - "emidms" + - "emibvoc" + - "emibc" + - "ec" + - "dryso4" + - "dryso2" + - "drybc" + - "dpco2" + - "dmsos" + - "dms" + - "dmlt" + - "dissocos" + - "dissoc" + - "dissicos" + - "dissicnat" + - "dissic" + - "dfeos" + - "dfe" + - "detocos" + - "detoc" + - "deptho" + - "cod" + - "co3satcalcos" + - "co3satcalc" + - "co3sataragos" + - "co3satarag" + - "co3os" + - "co3nat" + - "co3" + - "co2mass" + - "co2" + - "clwvi" + - "clwmodis" + - "clw" + - "cltmodis" + - "cltisccp" + - "cltcalipso" + - "clt" + - "clmcalipso" + - "cllcalipso" + - "clivi" + - "climodis" + - "cli" + - "clhcalipso" + - "cl" + - "chlos" + - "chl" + - "chepsoa" + - "ch4global" + - "cfc12global" + - "cfc12" + - "cfc11global" + - "cfc11" + - "cdnc" + - "cct" + - "ccn" + - "ccb" + - "calcos" + - "calc" + - "cWood" + - "cVeg" + - "cStem" + - "cSoilSlow" + - "cSoilMedium" + - "cSoilFast" + - "cSoilAbove1m" + - "cSoil" + - "cRoot" + - "cMisc" + - "cLitter" + - "cLeaf" + - "cLand" + - "cCwd" + - "bsios" + - "bsi" + - "bldep" + - "bfeos" + - "bfe" + - "basin" + - "ares" + - "areacello" + - "areacella" + - "albisccp" + - "airmass" + - "agessc" + - "abs550aer" + +supported_experiments: + - "hist-1950HC" + - "lfmip-pdLC" + - "ssp126" + - "ssp126-ssp370Lu" + - "ssp245" + - "ssp370" + - "ssp370-lowNTCF" + - "ssp370-ssp126Lu" + - "ssp370SST" + - "ssp370SST-lowCH4" + - "ssp370SST-lowNTCF" + - "ssp370SST-ssp126Lu" + - "ssp585" + - "hist-resAMO" + - "hist-resIPO" + - "historical-ext" + - "lfmip-initLC" + - "lfmip-pdLC-cruNcep" + - "lfmip-pdLC-princeton" + - "lfmip-pdLC-wfdei" + - "lfmip-rmLC" + - "lfmip-rmLC-cruNcep" + - "lfmip-rmLC-princeton" + - "lfmip-rmLC-wfdei" + - "pa-futAntSIC" + - "pa-futArcSIC" + - "pa-pdSIC" + - "pa-piAntSIC" + - "pa-piArcSIC" + - "ssp119" + - "ssp370pdSST" + - "ssp370SST-lowAer" + - "ssp370SST-lowBC" + - "ssp370SST-lowO3" + - "ssp434" + - "ssp460" + - "dcppC-atl-pacemaker" + - "dcppC-pac-pacemaker" + - "pa-futAntSIC-ext" + - "pa-futArcSIC-ext" + - "pa-pdSIC-ext" + - "ssp370-lowNTCFCH4" + - "ssp370SST-lowNTCFCH4" + - "volc-cluster-21C" + - "yr2010CO2" + - "dcppA-historical-niff" + - "1pctCO2" + - "1pctCO2-bgc" + - "abrupt-4xCO2" + - "dcppC-amv-neg" + - "dcppC-amv-pos" + - "dcppC-atl-control" + - "dcppC-ipv-neg" + - "dcppC-ipv-pos" + - "dcppC-pac-control" + - "deforest-globe" + - "faf-heat" + - "faf-heat-NA50pct" + - "faf-stress" + - "faf-water" + - "G1" + - "hist-aer" + - "hist-GHG" + - "hist-nat" + - "hist-noLu" + - "hist-piNTCF" + - "hist-spAer-all" + - "histSST" + - "histSST-noLu" + - "histSST-piCH4" + - "histSST-piNTCF" + - "piClim-4xCO2" + - "piClim-aer" + - "piClim-anthro" + - "piClim-CH4" + - "piClim-control" + - "piClim-ghg" + - "piClim-HC" + - "piClim-lu" + - "piClim-NTCF" + - "volc-long-eq" + - "volc-pinatubo-full" + - "volc-pinatubo-strat" + - "volc-pinatubo-surf" + - "1pctCO2-rad" + - "1pctCO2Ndep" + - "1pctCO2Ndep-bgc" + - "abrupt-0p5xCO2" + - "abrupt-2xCO2" + - "abrupt-solm4p" + - "abrupt-solp4p" + - "dcppC-amv-ExTrop-neg" + - "dcppC-amv-ExTrop-pos" + - "dcppC-amv-Trop-neg" + - "dcppC-amv-Trop-pos" + - "dcppC-ipv-NexTrop-neg" + - "dcppC-ipv-NexTrop-pos" + - "faf-all" + - "faf-antwater-stress" + - "faf-heat-NA0pct" + - "faf-passiveheat" + - "hist-bgc" + - "hist-piAer" + - "hist-spAer-aer" + - "hist-stratO3" + - "histSST-piAer" + - "histSST-piN2O" + - "histSST-piO3" + - "piClim-2xdust" + - "piClim-2xss" + - "piClim-BC" + - "piClim-histaer" + - "piClim-histall" + - "piClim-histghg" + - "piClim-histnat" + - "piClim-N2O" + - "piClim-O3" + - "piClim-spAer-aer" + - "piClim-spAer-anthro" + - "piClim-spAer-histaer" + - "piClim-spAer-histall" + - "piSST-4xCO2-solar" + - "volc-cluster-ctrl" + - "volc-long-hlN" + - "hist-all-aer2" + - "hist-all-nat2" + - "hist-CO2" + - "hist-sol" + - "hist-totalO3" + - "hist-volc" + - "piClim-2xDMS" + - "piClim-2xfire" + - "piClim-2xNOx" + - "piClim-2xVOC" + - "piClim-NH3" + - "piClim-NOx" + - "piClim-OC" + - "piClim-SO2" + - "piClim-VOC" + - "volc-long-hlS" + - "histSST-1950HC" + - "esm-ssp585" + - "esm-ssp585-ssp126Lu" + - "esm-hist-ext" + - "ssp534-over-bgc" + - "ssp585-bgc" + - "esm-1pct-brch-1000PgC" + - "esm-1pct-brch-750PgC" + - "esm-1pct-brch-2000PgC" + - "esm-hist" + - "esm-pi-cdr-pulse" + - "esm-pi-CO2pulse" + - "esm-1pctCO2" + - "esm-bell-750PgC" + - "esm-bell-1000PgC" + - "esm-bell-2000PgC" + - "esm-yr2010CO2-control" + - "1pctCO2-4xext" + - "1pctCO2-cdr" + - "esm-ssp534-over" + - "esm-ssp585-ocn-alk" + - "esm-ssp585ext" + - "esm-ssp585-ocn-alk-stop" + - "esm-ssp585-ssp126Lu-ext" + - "esm-yr2010CO2-cdr-pulse" + - "esm-yr2010CO2-CO2pulse" + - "esm-yr2010CO2-noemit" + - "amip" + - "amip-4xCO2" + - "amip-future4K" + - "amip-hist" + - "amip-p4K" + - "aqua-4xCO2" + - "aqua-control" + - "aqua-p4K" + - "highresSST-present" + - "ism-ctrl-std" + - "ism-pdControl-std" + - "ism-piControl-self" + - "land-hist" + - "land-hist-altStartYear" + - "land-noLu" + - "land-ssp126" + - "land-ssp585" + - "lgm" + - "lig127k" + - "midHolocene" + - "midPliocene-eoi400" + - "omip1" + - "past1000" + - "piControl-withism" + - "rad-irf" + - "a4SST" + - "a4SSTice" + - "a4SSTice-4xCO2" + - "amip-a4SST-4xCO2" + - "amip-lfmip-pdLC" + - "amip-lfmip-pObs" + - "amip-lfmip-rmLC" + - "amip-lwoff" + - "amip-m4K" + - "amip-p4K-lwoff" + - "amip-piForcing" + - "aqua-control-lwoff" + - "aqua-p4K-lwoff" + - "dcppA-assim" + - "esm-piControl-spinup" + - "land-cClim" + - "land-cCO2" + - "land-crop-grass" + - "land-crop-noFert" + - "land-crop-noIrrig" + - "land-crop-noIrrigFert" + - "land-hist-altLu1" + - "land-hist-altLu2" + - "land-hist-cruNcep" + - "land-hist-princeton" + - "land-hist-wfdei" + - "land-noFire" + - "land-noPasture" + - "land-noShiftCultivate" + - "land-noWoodHarv" + - "land-ssp434" + - "omip1-spunup" + - "past1000-solaronly" + - "past1000-volconly" + - "piControl-spinup" + - "piControl-spinup-cmip5" + - "piSST" + - "piSST-4xCO2" + - "piSST-4xCO2-rad" + - "piSST-pxK" + - "spinup-1950" + - "amip-hld" + - "amip-TIP" + - "amip-TIP-nosh" + - "control-slab" + - "dcppC-atl-spg" + - "esm-past1000" + - "ism-lig127k-std" + - "omip2" + - "omip2-spunup" + - "past2k" + - "esm-piControl" + - "historical" + - "historical-cmip5" + - "hist-aer-cmip5" + - "hist-GHG-cmip5" + - "hist-nat-cmip5" + - "piControl" + - "piControl-cmip5" + - "ssp245-aer" + - "ssp245-cov-strgreen" + - "ssp245-covid" + - "ssp245-cov-aer" + - "ssp245-cov-fossil" + - "ssp245-cov-GHG" + - "ssp245-cov-modgreen" + - "ssp245-GHG" + - "ssp245-nat" + - "ssp245-stratO3" + - "dcppA-hindcast" + - "dcppB-forecast" + - "dcppC-forecast-addPinatubo" + - "dcppC-hindcast-noPinatubo" + - "dcppC-hindcast-noAgung" + - "dcppC-hindcast-noElChichon" + - "dcppC-forecast-addAgung" + - "dcppC-forecast-addElChichon" + - "dcppA-hindcast-niff" + - "futureSST-4xCO2-solar" + - "G6solar" + - "G6sulfur" + - "G6SST1" + - "G7cirrus" + - "G7SST1-cirrus" + - "ssp534-over" + - "G6SST2-solar" + - "G6SST2-sulfur" + - "G7SST2-cirrus" + - "control-1950" + - "hist-1950" + - "highres-future" + - "highresSST-4xCO2" + - "highresSST-future" + - "highresSST-LAI" + - "highresSST-p4K" + - "highresSST-smoothed" + - "1pctCO2to4x-withism" + - "historical-withism" + - "ism-1pctCO2to4x-self" + - "ism-historical-self" + - "ism-1pctCO2to4x-std" + - "ism-historical-std" + - "ism-asmb-std" + - "ism-bsmb-std" + - "ism-amip-std" + - "ism-ssp585-self" + - "ism-ssp585-std" + - "ssp585-withism" + - "pdSST-futAntSIC" + - "pdSST-futArcSIC" + - "pdSST-pdSIC" + - "pdSST-piAntSIC" + - "pdSST-piArcSIC" + - "piSST-pdSIC" + - "futSST-pdSIC" + - "piSST-piSIC" + - "amip-climSIC" + - "amip-climSST" + - "modelSST-futArcSIC" + - "modelSST-pdSIC" + - "pdSST-futArcSICSIT" + - "pdSST-futBKSeasSIC" + - "pdSST-futOkhotskSIC" + - "pdSST-pdSICSIT" + - "rcp26-cmip5" + - "rcp45-cmip5" + - "rcp60-cmip5" + - "rcp85-cmip5" + - "volc-cluster-mill" + - "volc-pinatubo-slab" \ No newline at end of file diff --git a/configs/downloader/constants/cmip6plus.yaml b/configs/downloader/constants/cmip6plus.yaml new file mode 100644 index 0000000..e7e255d --- /dev/null +++ b/configs/downloader/constants/cmip6plus.yaml @@ -0,0 +1,13 @@ +node_link: "http://esgf-data2.llnl.gov" + +model_sources: + - "HasGEM3-GC31-LL" + +var_source_lookup: + - "areacella" + - "mrsofc" + +supported_experiments: + - "hist-lu" + - "hist-piAer" + - "hist-piVolc" \ No newline at end of file diff --git a/configs/downloader/constants/imput4MIPs.yaml b/configs/downloader/constants/imput4MIPs.yaml new file mode 100644 index 0000000..4915f04 --- /dev/null +++ b/configs/downloader/constants/imput4MIPs.yaml @@ -0,0 +1,732 @@ +node_link: "https://esgf-node.llnl.gov/esg-search/" + +emissions_endings: + - "_em_openburning" + - "_em_anthro" + - "_em_AIR_anthro" + +meta_endings_prc: + - "_percentage_AGRI" + - "_percentage_BORF" + - "_percentage_DEFO" + - "_percentage_PEAT" + - "_percentage_SAVA" + - "_percentage_TEMF" + +meta_endings_shar: + - "_openburning_share" + +mip_era: "CMIP6" + +target_mip: "ScenarioMIP" + +supported_experiments: + - "historical" + - "ssp119" + - "ssp126" + - "ssp245" + - "ssp370" + - "ssp434" + - "ssp460" + - "ssp534-over" + - "ssp585" + +var_source_lookup: + - "years" + - "year_weight" + - "year_fr" + - "wlenbinsize" + - "wlen_bnds" + - "wlen" + - "wfo" + - "wetnoy" + - "wetnhx" + - "water_vapor" + - "vos" + - "volume_density" + - "vo" + - "vmro3" + - "vas" + - "urban_to_secdn" + - "urban_to_secdf" + - "urban_to_range" + - "urban_to_pastr" + - "urban_to_c4per" + - "urban_to_c4ann" + - "urban_to_c3per" + - "urban_to_c3nfx" + - "urban_to_c3ann" + - "urban" + - "uos" + - "uo" + - "uas" + - "tsi" + - "ts" + - "total_solar_irradiance" + - "tosbcs" + - "tos" + - "thetao" + - "theta" + - "temp_level" + - "temp_layer" + - "tauv" + - "tauu" + - "tas" + - "surface_temperature" + - "surface_emissivity" + - "surface_albedo" + - "sst" + - "ssn" + - "ssi" + - "ssa550" + - "sos" + - "solar_zenith_angle" + - "so2f2_SH" + - "so2f2_NH" + - "so2f2_GM" + - "so" + - "sithick" + - "sig_lon_W" + - "sig_lon_E" + - "sig_lat_W" + - "sig_lat_E" + - "siconcbcs" + - "siconca" + - "siconc" + - "sftof" + - "sftflf" + - "sf6_SH" + - "sf6_NH" + - "sf6_GM" + - "secyf_harv" + - "secyf_bioh" + - "secnf_harv" + - "secnf_bioh" + - "secmf_harv" + - "secmf_bioh" + - "secmb" + - "secma" + - "secdn_to_urban" + - "secdn_to_secdf" + - "secdn_to_range" + - "secdn_to_pastr" + - "secdn_to_c4per" + - "secdn_to_c4ann" + - "secdn_to_c3per" + - "secdn_to_c3nfx" + - "secdn_to_c3ann" + - "secdn" + - "secdf_to_urban" + - "secdf_to_secdn" + - "secdf_to_range" + - "secdf_to_pastr" + - "secdf_to_c4per" + - "secdf_to_c4ann" + - "secdf_to_c3per" + - "secdf_to_c3nfx" + - "secdf_to_c3ann" + - "secdf" + - "scph" + - "scnum" + - "sad_of_big_particles" + - "sad" + - "rsds" + - "rndwd" + - "rmean" + - "rlds" + - "range_to_urban" + - "range_to_secdn" + - "range_to_secdf" + - "range_to_pastr" + - "range_to_c4per" + - "range_to_c4ann" + - "range_to_c3per" + - "range_to_c3nfx" + - "range_to_c3ann" + - "range" + - "ptbio" + - "psl" + - "prsn" + - "prra" + - "profile_weight" + - "primn_to_urban" + - "primn_to_secdf" + - "primn_to_range" + - "primn_to_pastr" + - "primn_to_c4per" + - "primn_to_c4ann" + - "primn_to_c3per" + - "primn_to_c3nfx" + - "primn_to_c3ann" + - "primn_harv" + - "primn_bioh" + - "primn" + - "primf_to_urban" + - "primf_to_secdn" + - "primf_to_range" + - "primf_to_pastr" + - "primf_to_c4per" + - "primf_to_c4ann" + - "primf_to_c3per" + - "primf_to_c3nfx" + - "primf_to_c3ann" + - "primf_harv" + - "primf_bioh" + - "primf" + - "pressure" + - "pres_level" + - "pres_layer" + - "pr" + - "plume_number" + - "plume_lon" + - "plume_lat" + - "plume_feature" + - "percentage_TEMF" + - "percentage_SAVA" + - "percentage_PEAT" + - "percentage_DEFO" + - "percentage_BORF" + - "percentage_AGRI" + - "pastr_to_urban" + - "pastr_to_secdn" + - "pastr_to_secdf" + - "pastr_to_range" + - "pastr_to_c4per" + - "pastr_to_c4ann" + - "pastr_to_c3per" + - "pastr_to_c3nfx" + - "pastr_to_c3ann" + - "pastr" + - "ozone" + - "oxygen_GM" + - "nitrous_oxide_SH" + - "nitrous_oxide_NH" + - "nitrous_oxide_GM" + - "nitrogen_GM" + - "nf3_SH" + - "nf3_NH" + - "nf3_GM" + - "mrro" + - "month" + - "mole_fraction_of_so2f2_in_air" + - "mole_fraction_of_sf6_in_air" + - "mole_fraction_of_nitrous_oxide_in_air" + - "mole_fraction_of_nf3_in_air" + - "mole_fraction_of_methyl_chloride_in_air" + - "mole_fraction_of_methyl_bromide_in_air" + - "mole_fraction_of_methane_in_air" + - "mole_fraction_of_hfc4310mee_in_air" + - "mole_fraction_of_hfc365mfc_in_air" + - "mole_fraction_of_hfc32_in_air" + - "mole_fraction_of_hfc245fa_in_air" + - "mole_fraction_of_hfc23_in_air" + - "mole_fraction_of_hfc236fa_in_air" + - "mole_fraction_of_hfc227ea_in_air" + - "mole_fraction_of_hfc152a_in_air" + - "mole_fraction_of_hfc143a_in_air" + - "mole_fraction_of_hfc134aeq_in_air" + - "mole_fraction_of_hfc134a_in_air" + - "mole_fraction_of_hfc125_in_air" + - "mole_fraction_of_hcfc22_in_air" + - "mole_fraction_of_hcfc142b_in_air" + - "mole_fraction_of_hcfc141b_in_air" + - "mole_fraction_of_halon2402_in_air" + - "mole_fraction_of_halon1301_in_air" + - "mole_fraction_of_halon1211_in_air" + - "mole_fraction_of_co2eq_in_air" + - "mole_fraction_of_chcl3_in_air" + - "mole_fraction_of_ch3ccl3_in_air" + - "mole_fraction_of_ch2cl2_in_air" + - "mole_fraction_of_cfc12eq_in_air" + - "mole_fraction_of_cfc12_in_air" + - "mole_fraction_of_cfc11eq_in_air" + - "mole_fraction_of_cfc11_in_air" + - "mole_fraction_of_cfc115_in_air" + - "mole_fraction_of_cfc114_in_air" + - "mole_fraction_of_cfc113_in_air" + - "mole_fraction_of_cf4_in_air" + - "mole_fraction_of_carbon_tetrachloride_in_air" + - "mole_fraction_of_carbon_dioxide_in_air" + - "mole_fraction_of_c_c4f8_in_air" + - "mole_fraction_of_c8f18_in_air" + - "mole_fraction_of_c7f16_in_air" + - "mole_fraction_of_c6f14_in_air" + - "mole_fraction_of_c5f12_in_air" + - "mole_fraction_of_c4f10_in_air" + - "mole_fraction_of_c3f8_in_air" + - "mole_fraction_of_c2f6_in_air" + - "methyl_chloride_SH" + - "methyl_chloride_NH" + - "methyl_chloride_GM" + - "methyl_bromide_SH" + - "methyl_bromide_NH" + - "methyl_bromide_GM" + - "methane_SH" + - "methane_NH" + - "methane_GM" + - "mask4resto_ipv_Nextrop" + - "mask4resto_ipv" + - "mask4resto_amv_trop" + - "mask4resto_amv_extrop" + - "mask4resto_amv" + - "lon_bounds" + - "licalvf" + - "lat_bounds" + - "kp" + - "is_biomass" + - "irrig_c4per" + - "irrig_c4ann" + - "irrig_c3per" + - "irrig_c3nfx" + - "irrig_c3ann" + - "ipv_index" + - "iprp" + - "iprm" + - "iprg" + - "icwtr" + - "huss" + - "hfds" + - "hfc4310mee_SH" + - "hfc4310mee_NH" + - "hfc4310mee_GM" + - "hfc365mfc_SH" + - "hfc365mfc_NH" + - "hfc365mfc_GM" + - "hfc32_SH" + - "hfc32_NH" + - "hfc32_GM" + - "hfc245fa_SH" + - "hfc245fa_NH" + - "hfc245fa_GM" + - "hfc23_SH" + - "hfc23_NH" + - "hfc23_GM" + - "hfc236fa_SH" + - "hfc236fa_NH" + - "hfc236fa_GM" + - "hfc227ea_SH" + - "hfc227ea_NH" + - "hfc227ea_GM" + - "hfc152a_SH" + - "hfc152a_NH" + - "hfc152a_GM" + - "hfc143a_SH" + - "hfc143a_NH" + - "hfc143a_GM" + - "hfc134aeq_SH" + - "hfc134aeq_NH" + - "hfc134aeq_GM" + - "hfc134a_SH" + - "hfc134a_NH" + - "hfc134a_GM" + - "hfc125_SH" + - "hfc125_NH" + - "hfc125_GM" + - "hcfc22_SH" + - "hcfc22_NH" + - "hcfc22_GM" + - "hcfc142b_SH" + - "hcfc142b_NH" + - "hcfc142b_GM" + - "hcfc141b_SH" + - "hcfc141b_NH" + - "hcfc141b_GM" + - "halon2402_SH" + - "halon2402_NH" + - "halon2402_GM" + - "halon1301_SH" + - "halon1301_NH" + - "halon1301_GM" + - "halon1211_SH" + - "halon1211_NH" + - "halon1211_GM" + - "gzdis" + - "gridcellarea" + - "gpbio" + - "gldis" + - "glat_bnds" + - "glat" + - "fulwd" + - "ftr_weight" + - "fstnf" + - "friver" + - "flood" + - "fill_flag" + - "fharv_c4per" + - "fharv_c3per" + - "fertl_c4per" + - "fertl_c4ann" + - "fertl_c3per" + - "fertl_c3nfx" + - "fertl_c3ann" + - "f107" + - "expt_label" + - "evspsbl" + - "drynoy" + - "drynhx" + - "delta13co2_in_air" + - "datasource" + - "crpbf_total" + - "crpbf_c4per" + - "crpbf_c4ann" + - "crpbf_c3per" + - "crpbf_c3nfx" + - "crpbf_c3ann" + - "combf" + - "co2eq_SH" + - "co2eq_NH" + - "co2eq_GM" + - "chcl3_SH" + - "chcl3_NH" + - "chcl3_GM" + - "ch3ccl3_SH" + - "ch3ccl3_NH" + - "ch3ccl3_GM" + - "ch2cl2_SH" + - "ch2cl2_NH" + - "ch2cl2_GM" + - "cfc12eq_SH" + - "cfc12eq_NH" + - "cfc12eq_GM" + - "cfc12_SH" + - "cfc12_NH" + - "cfc12_GM" + - "cfc11eq_SH" + - "cfc11eq_NH" + - "cfc11eq_GM" + - "cfc11_SH" + - "cfc11_NH" + - "cfc11_GM" + - "cfc115_SH" + - "cfc115_NH" + - "cfc115_GM" + - "cfc114_SH" + - "cfc114_NH" + - "cfc114_GM" + - "cfc113_SH" + - "cfc113_NH" + - "cfc113_GM" + - "cf4_SH" + - "cf4_NH" + - "cf4_GM" + - "ccode" + - "carea" + - "carbon_tetrachloride_SH" + - "carbon_tetrachloride_NH" + - "carbon_tetrachloride_GM" + - "carbon_monoxide_GM" + - "carbon_dioxide_SH" + - "carbon_dioxide_NH" + - "carbon_dioxide_GM" + - "calyear" + - "calmonth" + - "calday" + - "c_c4f8_SH" + - "c_c4f8_NH" + - "c_c4f8_GM" + - "c8f18_SH" + - "c8f18_NH" + - "c8f18_GM" + - "c7f16_SH" + - "c7f16_NH" + - "c7f16_GM" + - "c6f14_SH" + - "c6f14_NH" + - "c6f14_GM" + - "c5f12_SH" + - "c5f12_NH" + - "c5f12_GM" + - "c4per_to_urban" + - "c4per_to_secdn" + - "c4per_to_secdf" + - "c4per_to_range" + - "c4per_to_pastr" + - "c4per_to_c4ann" + - "c4per_to_c3per" + - "c4per_to_c3nfx" + - "c4per_to_c3ann" + - "c4per" + - "c4f10_SH" + - "c4f10_NH" + - "c4f10_GM" + - "c4ann_to_urban" + - "c4ann_to_secdn" + - "c4ann_to_secdf" + - "c4ann_to_range" + - "c4ann_to_pastr" + - "c4ann_to_c4per" + - "c4ann_to_c3per" + - "c4ann_to_c3nfx" + - "c4ann_to_c3ann" + - "c4ann" + - "c3per_to_urban" + - "c3per_to_secdn" + - "c3per_to_secdf" + - "c3per_to_range" + - "c3per_to_pastr" + - "c3per_to_c4per" + - "c3per_to_c4ann" + - "c3per_to_c3nfx" + - "c3per_to_c3ann" + - "c3per" + - "c3nfx_to_urban" + - "c3nfx_to_secdn" + - "c3nfx_to_secdf" + - "c3nfx_to_range" + - "c3nfx_to_pastr" + - "c3nfx_to_c4per" + - "c3nfx_to_c4ann" + - "c3nfx_to_c3per" + - "c3nfx_to_c3ann" + - "c3nfx" + - "c3f8_SH" + - "c3f8_NH" + - "c3f8_GM" + - "c3ann_to_urban" + - "c3ann_to_secdn" + - "c3ann_to_secdf" + - "c3ann_to_range" + - "c3ann_to_pastr" + - "c3ann_to_c4per" + - "c3ann_to_c4ann" + - "c3ann_to_c3per" + - "c3ann_to_c3nfx" + - "c3ann" + - "c2f6_SH" + - "c2f6_NH" + - "c2f6_GM" + - "bounds_time" + - "bounds_sector" + - "bounds_latitude" + - "bounds_altitude" + - "beta_b" + - "beta_a" + - "asy550" + - "asl" + - "areacello" + - "areacellg" + - "areacella" + - "ap" + - "aod_spmx" + - "aod_fmbg" + - "ann_cycle" + - "angstrom" + - "amv_index" + - "altitude" + - "added_tree_cover" + - "acabf" + - "WST" + - "VOC_openburning_share" + - "VOC_em_openburning" + - "VOC_em_anthro" + - "VOC_em_AIR_anthro" + - "VOC25_other_voc_em_speciated_VOC_anthro" + - "VOC25_other_voc_em_speciated_VOC" + - "VOC25-other_voc_em_speciated_VOC" + - "VOC24_acids_em_speciated_VOC_anthro" + - "VOC24_acids_em_speciated_VOC" + - "VOC24-acids_em_speciated_VOC" + - "VOC23_ketones_em_speciated_VOC_anthro" + - "VOC23_ketones_em_speciated_VOC" + - "VOC23-ketones_em_speciated_VOC" + - "VOC22_other_alka_em_speciated_VOC_anthro" + - "VOC22_other_alka_em_speciated_VOC" + - "VOC22-other_alka_em_speciated_VOC" + - "VOC21_methanal_em_speciated_VOC_anthro" + - "VOC21_methanal_em_speciated_VOC" + - "VOC21-methanal_em_speciated_VOC" + - "VOC20_chlorinate_em_speciated_VOC_anthro" + - "VOC20_chlorinate_em_speciated_VOC" + - "VOC20-chlorinate_em_speciated_VOC" + - "VOC19_ethers_em_speciated_VOC_anthro" + - "VOC19_ethers_em_speciated_VOC" + - "VOC19-ethers_em_speciated_VOC" + - "VOC18_esters_em_speciated_VOC_anthro" + - "VOC18_esters_em_speciated_VOC" + - "VOC18-esters_em_speciated_VOC" + - "VOC17_other_arom_em_speciated_VOC_anthro" + - "VOC17_other_arom_em_speciated_VOC" + - "VOC17-other_arom_em_speciated_VOC" + - "VOC16_trimethylb_em_speciated_VOC_anthro" + - "VOC16_trimethylb_em_speciated_VOC" + - "VOC16-trimethylb_em_speciated_VOC" + - "VOC15_xylene_em_speciated_VOC_anthro" + - "VOC15_xylene_em_speciated_VOC" + - "VOC15-xylene_em_speciated_VOC" + - "VOC14_toluene_em_speciated_VOC_anthro" + - "VOC14_toluene_em_speciated_VOC" + - "VOC14-toluene_em_speciated_VOC" + - "VOC13_benzene_em_speciated_VOC_anthro" + - "VOC13_benzene_em_speciated_VOC" + - "VOC13-benzene_em_speciated_VOC" + - "VOC12_other_alke_em_speciated_VOC_anthro" + - "VOC12_other_alke_em_speciated_VOC" + - "VOC12-other_alke_em_speciated_VOC" + - "VOC09_ethyne_em_speciated_VOC_anthro" + - "VOC09_ethyne_em_speciated_VOC" + - "VOC09-ethyne_em_speciated_VOC" + - "VOC08_propene_em_speciated_VOC_anthro" + - "VOC08_propene_em_speciated_VOC" + - "VOC08-propene_em_speciated_VOC" + - "VOC07_ethene_em_speciated_VOC_anthro" + - "VOC07_ethene_em_speciated_VOC" + - "VOC07-ethene_em_speciated_VOC" + - "VOC06_hexanes_pl_em_speciated_VOC_anthro" + - "VOC06_hexanes_pl_em_speciated_VOC" + - "VOC06-hexanes_pl_em_speciated_VOC" + - "VOC05_pentanes_em_speciated_VOC_anthro" + - "VOC05_pentanes_em_speciated_VOC" + - "VOC05-pentanes_em_speciated_VOC" + - "VOC04_butanes_em_speciated_VOC_anthro" + - "VOC04_butanes_em_speciated_VOC" + - "VOC04-butanes_em_speciated_VOC" + - "VOC03_propane_em_speciated_VOC_anthro" + - "VOC03_propane_em_speciated_VOC" + - "VOC03-propane_em_speciated_VOC" + - "VOC02_ethane_em_speciated_VOC_anthro" + - "VOC02_ethane_em_speciated_VOC" + - "VOC02-ethane_em_speciated_VOC" + - "VOC01_alcohols_em_speciated_VOC_anthro" + - "VOC01_alcohols_em_speciated_VOC" + - "VOC01-alcohols_em_speciated_VOC" + - "Toluene_lump" + - "TRA" + - "SO2_openburning_share" + - "SO2_em_openburning" + - "SO2_em_anthro" + - "SO2_em_SOLID_BIOFUEL_anthro" + - "SO2_em_AIR_anthro" + - "SO2" + - "SLV" + - "SHP" + - "RSLossRem" + - "RCO" + - "OC_openburning_share" + - "OC_em_openburning" + - "OC_em_anthro" + - "OC_em_SOLID_BIOFUEL_anthro" + - "OC_em_AIR_anthro" + - "OC" + - "NOx_openburning_share" + - "NOx_em_openburning" + - "NOx_em_anthro" + - "NOx_em_SOLID_BIOFUEL_anthro" + - "NOx_em_AIR_anthro" + - "NOx" + - "NMVOC_openburning_share" + - "NMVOC_em_openburning" + - "NMVOC_em_anthro" + - "NMVOC_em_SOLID_BIOFUEL_anthro" + - "NMVOC_em_AIR_anthro" + - "NMVOC_Toluene_lump_speciated_VOC_openburning_share" + - "NMVOC_Toluene_lump_em_speciated_VOC_openburning" + - "NMVOC_MEK_speciated_VOC_openburning_share" + - "NMVOC_MEK_em_speciated_VOC_openburning" + - "NMVOC_Higher_Alkenes_speciated_VOC_openburning_share" + - "NMVOC_Higher_Alkenes_em_speciated_VOC_openburning" + - "NMVOC_Higher_Alkanes_speciated_VOC_openburning_share" + - "NMVOC_Higher_Alkanes_em_speciated_VOC_openburning" + - "NMVOC_HOCH2CHO_speciated_VOC_openburning_share" + - "NMVOC_HOCH2CHO_em_speciated_VOC_openburning" + - "NMVOC_HCOOH_speciated_VOC_openburning_share" + - "NMVOC_HCOOH_em_speciated_VOC_openburning" + - "NMVOC_HCN_speciated_VOC_openburning_share" + - "NMVOC_HCN_em_speciated_VOC_openburning" + - "NMVOC_CH3OH_speciated_VOC_openburning_share" + - "NMVOC_CH3OH_em_speciated_VOC_openburning" + - "NMVOC_CH3COOH_speciated_VOC_openburning_share" + - "NMVOC_CH3COOH_em_speciated_VOC_openburning" + - "NMVOC_CH3COCHO_speciated_VOC_openburning_share" + - "NMVOC_CH3COCHO_em_speciated_VOC_openburning" + - "NMVOC_CH2O_speciated_VOC_openburning_share" + - "NMVOC_CH2O_em_speciated_VOC_openburning" + - "NMVOC_C8H10_speciated_VOC_openburning_share" + - "NMVOC_C8H10_em_speciated_VOC_openburning" + - "NMVOC_C7H8_speciated_VOC_openburning_share" + - "NMVOC_C7H8_em_speciated_VOC_openburning" + - "NMVOC_C6H6_speciated_VOC_openburning_share" + - "NMVOC_C6H6_em_speciated_VOC_openburning" + - "NMVOC_C5H8_speciated_VOC_openburning_share" + - "NMVOC_C5H8_em_speciated_VOC_openburning" + - "NMVOC_C3H8_speciated_VOC_openburning_share" + - "NMVOC_C3H8_em_speciated_VOC_openburning" + - "NMVOC_C3H6_speciated_VOC_openburning_share" + - "NMVOC_C3H6_em_speciated_VOC_openburning" + - "NMVOC_C3H6O_speciated_VOC_openburning_share" + - "NMVOC_C3H6O_em_speciated_VOC_openburning" + - "NMVOC_C2H6_speciated_VOC_openburning_share" + - "NMVOC_C2H6_em_speciated_VOC_openburning" + - "NMVOC_C2H6S_speciated_VOC_openburning_share" + - "NMVOC_C2H6S_em_speciated_VOC_openburning" + - "NMVOC_C2H5OH_speciated_VOC_openburning_share" + - "NMVOC_C2H5OH_em_speciated_VOC_openburning" + - "NMVOC_C2H4_speciated_VOC_openburning_share" + - "NMVOC_C2H4_em_speciated_VOC_openburning" + - "NMVOC_C2H4O_speciated_VOC_openburning_share" + - "NMVOC_C2H4O_em_speciated_VOC_openburning" + - "NMVOC_C2H2_speciated_VOC_openburning_share" + - "NMVOC_C2H2_em_speciated_VOC_openburning" + - "NMVOC_C10H16_speciated_VOC_openburning_share" + - "NMVOC_C10H16_em_speciated_VOC_openburning" + - "NMVOC" + - "NH3_openburning_share" + - "NH3_em_openburning" + - "NH3_em_anthro" + - "NH3_em_SOLID_BIOFUEL_anthro" + - "NH3_em_AIR_anthro" + - "NH3" + - "N2O" + - "MEK" + - "IND" + - "Higher_Alkenes" + - "Higher_Alkanes" + - "HOCH2CHO" + - "HCOOH" + - "HCN" + - "H2_openburning_share" + - "H2_em_openburning" + - "H2SO4_mass" + - "H2" + - "ENE" + - "Delta14co2_in_air" + - "CO_openburning_share" + - "CO_em_openburning" + - "CO_em_anthro" + - "CO_em_SOLID_BIOFUEL_anthro" + - "CO_em_AIR_anthro" + - "CO2_em_anthro" + - "CO2_em_AIR_anthro" + - "CO2" + - "CO" + - "CH4_openburning_share" + - "CH4_em_openburning" + - "CH4_em_anthro" + - "CH4_em_SOLID_BIOFUEL_anthro" + - "CH4_em_AIR_anthro" + - "CH4" + - "CH3OH" + - "CH3COOH" + - "CH3COCHO" + - "CH2O" + - "C8H10" + - "C7H8" + - "C6H6" + - "C5H8" + - "C3H8" + - "C3H6O" + - "C3H6" + - "C2H6S" + - "C2H6" + - "C2H5OH" + - "C2H4O" + - "C2H4" + - "C2H2" + - "C10H16" + - "BC_openburning_share" + - "BC_em_openburning" + - "BC_em_anthro" + - "BC_em_SOLID_BIOFUEL_anthro" + - "BC_em_AIR_anthro" + - "BC" + - "AIR" + - "AGR" diff --git a/configs/downloader/future_usecases/cmip7.yaml b/configs/downloader/future_usecases/cmip7.yaml index 1a31530..100460e 100644 --- a/configs/downloader/future_usecases/cmip7.yaml +++ b/configs/downloader/future_usecases/cmip7.yaml @@ -1,4 +1,4 @@ -models: ["HasGEM3-GC31-LL"] -downloader_kwargs: - variables: ["mrsofc"] - experiments: ["hist-lu"] \ No newline at end of file +CMIP6Plus: + models: ["HasGEM3-GC31-LL"] + variables: [ "mrsofc" ] + experiments: [ "hist-lu" ] diff --git a/configs/downloader/future_usecases/e3sm.yaml b/configs/downloader/future_usecases/e3sm.yaml index 56d4fc6..16e9bdb 100644 --- a/configs/downloader/future_usecases/e3sm.yaml +++ b/configs/downloader/future_usecases/e3sm.yaml @@ -1,4 +1,3 @@ -models: ["E3SM"] -downloader_kwargs: - variables: ["???"] - experiments: ["ssp585"] \ No newline at end of file +E3SM: + variables: [ "???" ] + experiments: [ "ssp585" ] \ No newline at end of file diff --git a/configs/downloader/future_usecases/noresm_ocean.yaml b/configs/downloader/future_usecases/noresm_ocean.yaml new file mode 100644 index 0000000..76b58d6 --- /dev/null +++ b/configs/downloader/future_usecases/noresm_ocean.yaml @@ -0,0 +1,5 @@ +OMIP: + models: ["NorESM2-LM"] + variables: [ "omldamax" ] # sea-surface temperature (often used as forcing for atmo models) + experiments: [ "omip1" ] + ensemble_members: [ "r1i1p1f1" ] \ No newline at end of file diff --git a/configs/downloader/future_usecases/obs4mips.yaml b/configs/downloader/future_usecases/obs4mips.yaml index 8f7f853..9441903 100644 --- a/configs/downloader/future_usecases/obs4mips.yaml +++ b/configs/downloader/future_usecases/obs4mips.yaml @@ -1,4 +1,4 @@ -models: ["ESACCI-CLOUD-ATSR2-AATSR-3-0"] -downloader_kwargs: +#ESACCI-CLOUD-ATSR2-AATSR-3-0: +obs4MIPs: variables: ["pctCCI"] experiments: [""] \ No newline at end of file diff --git a/configs/downloader/input4mips/bc_historical.yaml b/configs/downloader/input4mips/bc_historical.yaml index 717e0c2..194b790 100644 --- a/configs/downloader/input4mips/bc_historical.yaml +++ b/configs/downloader/input4mips/bc_historical.yaml @@ -1,4 +1,3 @@ -models: ["NorESM2-LM"] -downloader_kwargs: - variables: ["BC"] - experiments: ["historical"] \ No newline at end of file +input4MIPs: + variables: [ "BC" ] + experiments: [ "historical" ] \ No newline at end of file diff --git a/configs/downloader/input4mips/bc_ssp.yaml b/configs/downloader/input4mips/bc_ssp.yaml index 1608f92..2d88ff6 100644 --- a/configs/downloader/input4mips/bc_ssp.yaml +++ b/configs/downloader/input4mips/bc_ssp.yaml @@ -1,4 +1,3 @@ -models: ["NorESM2-LM"] -downloader_kwargs: - variables: ["BC"] - experiments: ["ssp585"] \ No newline at end of file +input4MIPs: + variables: [ "BC" ] + experiments: [ "ssp585" ] \ No newline at end of file diff --git a/configs/downloader/input4mips/ch4_historical.yaml b/configs/downloader/input4mips/ch4_historical.yaml index 1e5ad30..00784f7 100644 --- a/configs/downloader/input4mips/ch4_historical.yaml +++ b/configs/downloader/input4mips/ch4_historical.yaml @@ -1,4 +1,3 @@ -models: ["NorESM2-LM"] -downloader_kwargs: - variables: ["CH4"] - experiments: ["historical"] \ No newline at end of file +input4MIPs: + variables: [ "CH4" ] + experiments: [ "historical" ] \ No newline at end of file diff --git a/configs/downloader/input4mips/ch4_ssp.yaml b/configs/downloader/input4mips/ch4_ssp.yaml index 4282283..e003e77 100644 --- a/configs/downloader/input4mips/ch4_ssp.yaml +++ b/configs/downloader/input4mips/ch4_ssp.yaml @@ -1,4 +1,3 @@ -models: ["NorESM2-LM"] -downloader_kwargs: - variables: ["CH4"] - experiments: ["ssp245"] \ No newline at end of file +input4MIPs: + variables: [ "CH4" ] + experiments: [ "ssp245" ] \ No newline at end of file diff --git a/configs/downloader/input4mips/co2_historical.yaml b/configs/downloader/input4mips/co2_historical.yaml index c415eb8..28a7f8f 100644 --- a/configs/downloader/input4mips/co2_historical.yaml +++ b/configs/downloader/input4mips/co2_historical.yaml @@ -1,4 +1,3 @@ -models: ["NorESM2-LM"] -downloader_kwargs: - variables: ["CO2"] - experiments: ["historical"] \ No newline at end of file +input4MIPs: + variables: [ "CO2" ] + experiments: [ "historical" ] \ No newline at end of file diff --git a/configs/downloader/input4mips/co2_ssp.yaml b/configs/downloader/input4mips/co2_ssp.yaml index 2a166d4..1646a9d 100644 --- a/configs/downloader/input4mips/co2_ssp.yaml +++ b/configs/downloader/input4mips/co2_ssp.yaml @@ -1,4 +1,3 @@ -models: ["NorESM2-LM"] -downloader_kwargs: - variables: ["C02"] - experiments: ["ssp460"] \ No newline at end of file +input4MIPs: + variables: [ "CO2" ] + experiments: [ "ssp460" ] \ No newline at end of file diff --git a/configs/downloader/input4mips/so2_historical.yaml b/configs/downloader/input4mips/so2_historical.yaml index f639cd3..9be5c8d 100644 --- a/configs/downloader/input4mips/so2_historical.yaml +++ b/configs/downloader/input4mips/so2_historical.yaml @@ -1,4 +1,3 @@ -models: ["NorESM2-LM"] -downloader_kwargs: - variables: ["SO2"] - experiments: ["historical"] \ No newline at end of file +input4MIPs: + variables: [ "SO2" ] + experiments: [ "historical" ] \ No newline at end of file diff --git a/configs/downloader/input4mips/so2_ssp.yaml b/configs/downloader/input4mips/so2_ssp.yaml index 56c5b5b..683b597 100644 --- a/configs/downloader/input4mips/so2_ssp.yaml +++ b/configs/downloader/input4mips/so2_ssp.yaml @@ -1,4 +1,3 @@ -models: ["NorESM2-LM"] -downloader_kwargs: - variables: ["SO2"] - experiments: ["ssp370"] \ No newline at end of file +input4MIPs: + variables: [ "SO2" ] + experiments: [ "ssp370" ] \ No newline at end of file diff --git a/configs/fgoals_minimal.yaml b/configs/fgoals_minimal.yaml deleted file mode 100644 index 145fa02..0000000 --- a/configs/fgoals_minimal.yaml +++ /dev/null @@ -1,4 +0,0 @@ -models: ["FGOALS-g3"] -downloader_kwargs: - variables: ["tas"] - experiments: ["ssp370"] \ No newline at end of file diff --git a/configs/minimal_dataset.yaml b/configs/minimal_dataset.yaml index af4c0c0..f53da4f 100644 --- a/configs/minimal_dataset.yaml +++ b/configs/minimal_dataset.yaml @@ -1,9 +1,9 @@ -models: ["NorESM2-LM"] -downloader_kwargs: - variables: ["tas", "CH4", "CO2"] - experiments: ["historical", "ssp126"] +CMIP6: + models: [ "NorESM2-LM" ] + variables: [ "tas" ] + experiments: [ "historical", "ssp126" ] max_ensemble_members: 1 - overwrite: true - download_biomassburning: false - start_year: 1990 - end_year: 2030 \ No newline at end of file + ensemble_members: ["r2i1p1f1"] +input4MIPs: + variables: [ "CH4", "CO2" ] + experiments: [ "historical","ssp126" ] \ No newline at end of file diff --git a/docs/agents/planning/esgpull_implementation/ESGPULL_CLIENT_IMPLEMENTATION.md b/docs/agents/planning/esgpull_implementation/ESGPULL_CLIENT_IMPLEMENTATION.md new file mode 100644 index 0000000..dd55809 --- /dev/null +++ b/docs/agents/planning/esgpull_implementation/ESGPULL_CLIENT_IMPLEMENTATION.md @@ -0,0 +1,69 @@ +# SPEC: Add ESGF Download Client using `esgpull` + +## 1. Goal Description +The objective is to implement a new asynchronous, stateful `esgpull` downloader client alongside the existing `esgf-pyclient` implementation for ESGF download operations within `climateset`. + +This addition introduces an architecture paradigm shift: providing a database-backed approach driven by `asyncio` as an alternative to the existing dynamic, stateless HTTP search scripts operating over `subprocess`. As the **Orchestrator**, the goal is a contract-first implementation integrating asynchronous Python fetching with strict SLURM cluster isolation requirements, while maintaining backwards compatibility with the existing client. + +## 2. Constraints & Assumptions +### Non-Functional Requirements +1. **Isolated Contexts**: `esgpull` must not initialize in `$HOME`. It must initialize in a unique, isolated path natively inside `RAW_DATA` (e.g. `RAW_DATA/.esgpull_jobs/`) to avoid file lock SQLite collisions between parallel SLURM batch jobs. +2. **Storage Cleanup**: `esgpull` downloads files natively to its internal cache. We must `shutil.move()` them to our strict local directory schema and cleanly delete the `.esgpull_jobs/` context immediately afterward to avoid cluttering disk space. +3. **Deterministic Search Testing**: Standard unit testing uses heavy mocking. **CRITICAL:** the *search querying phase* mechanism of this application must NEVER be mocked to ensure constraints map accurately against real ESGF nodes. + +## 3. Acceptance Criteria (AC) +- **AC1:** A new `EsgpullDownloader` (or similar) is added as an alternative search and download engine, co-existing with the current `esgf-pyclient` based downloader. +- **AC2:** Existing download clients and their related helper scripts/wrappers are preserved without breaking changes. +- **AC3:** Complex combinations of multi-value constraints and wildcards route through `esgpull.models.Query` successfully in the new client. +- **AC4:** The `esgpull` execution context initializes in an isolated cluster hash folder within `RAW_DATA`, completely bypassing `~/.esgpull`. +- **AC5:** A finalization block safely transfers `.nc` files from the isolation folder to the target Data Reference Syntax and purges the isolation folder afterward. +- **AC6:** End-to-end extraction in the new client is native `asyncio.run(esg.download())` without invoking `subprocess.run(["bash", ...])`. + +--- + +## 4. Orchestrator Implementation Phases + +### Blueprint Phase: Environment & Interfaces +* **Delegation**: `python`, `systemdesign` +* **Implement**: + 1. Modify `pyproject.toml` or `environment.yml`: Add `esgpull` to dependencies while retaining `esgf-pyclient`. + 2. Map out `climateset/download/constraints.py`. Ensure serialization outputs can be seamlessly adapted into dictionaries compatible with `esgpull.models.Query(selection=...)` for the new client, without breaking existing serialization. + +### Foundation Phase: The Isolated Context Engine +* **Delegation**: `systemdesign` +* **Implement**: + 1. Construct a context manager/helper inside `climateset/download/utils.py` (or a new `esgpull_utils.py`) named `isolated_esgpull_context(raw_data_path)`. + 2. Implement uniquely hashed path logic mapping to `RAW_DATA/.esgpull_jobs/`. + 3. Ensure a strict `try/finally` block wraps `shutil.rmtree` to tear down the environment safely regardless of download success/failure. + +### Implementation Phase A: The Search Contract +* **Delegation**: `python` +* **Implement**: + 1. Create a new downloader module (e.g., `climateset/download/esgpull_downloader.py`) that inherits from `AbstractDownloader` (if applicable) or implements the necessary download interface. + 2. Implement dynamic facet lookups (e.g., `get_grid_label`) in the new client using `hints = esg.context.hints(query)`. + 3. Implement `search_and_download_*` equivalents that instantiate `esg = Esgpull(path=hash_path)`. + 4. Trigger searches to return native tracking files: `files = esg.context.search(query, file=True)`. + +### Implementation Phase B: Asynchronous Execution & Integration +* **Delegation**: `python` +* **Implement**: + 1. Ensure the new client does not rely on bash-generation scripts like `_download_result` and `_download_process`. + 2. Inside the new download methods, add tracked files: `esg.db.add(*files)`. + 3. Wrap the retrieval execution: `downloaded, errors = asyncio.run(esg.download(files))`. + 4. Implement the file-moving pipeline transferring `.nc` chunk artifacts from the isolated DB cache to the final `RAW_DATA` path. + +--- + +## 5. Verification Plan + +Every Acceptance Criterion must be verified natively. + +### Automated Verification +- **Test Alignment (AC1, AC2, AC3)**: Add new tests specifically for the `esgpull` client. Ensure existing tests for the `esgf-pyclient` downloader continue to pass unmodified. +- **Search Isolation Guarantee (AC3)**: Review test logs to confirm queries touch real node indices and that actual `File` arrays are returned for the new client. +- **Download Mocking**: Ensure the `Esgpull.download` asynchronous function is the **only** layer intercepted by `unittest.mock` to prevent massive data bandwidth usage in CI. + +### Manual End-To-End Verification +- **Storage Independence (AC4, AC5)**: Execute a download script configured to use the new `esgpull` client. During runtime, verify (via terminal `ls`) that `RAW_DATA/.esgpull_jobs/` spawns the correct tracked directory, and verify it deletes cleanly when the script finishes. Confirm `~/.esgpull` is untouched. +- **Subprocess Elimination (AC6)**: Inspect the output of the `esgpull` download script to ensure Python logs asynchronously track chunks rather than arbitrary `wget` process logs. Confirm the `.nc` files correctly structure themselves in the output directory. +- **Regression Testing**: Execute a download using the existing `esgf-pyclient` implementation to verify it still functions as expected. diff --git a/docs/agents/planning/esgpull_implementation/download_logs.md b/docs/agents/planning/esgpull_implementation/download_logs.md new file mode 100644 index 0000000..a3b6eaa --- /dev/null +++ b/docs/agents/planning/esgpull_implementation/download_logs.md @@ -0,0 +1,639 @@ + $ python scripts/download_example.py download-basic +[2026-03-03 16:18:24] INFO [MainThread][climateset.utils] Yaml config file [/home/francispelletier/projects/ClimateSetExtension/configs/downloader/constants/cmip6.yaml] found. +[2026-03-03 16:18:24] INFO [MainThread][climateset.utils] Loading YAML config file [/home/francispelletier/projects/ClimateSetExtension/configs/downloader/constants/cmip6.yaml]. +[2026-03-03 16:18:24] INFO [MainThread][climateset.utils] Yaml config file [/home/francispelletier/projects/ClimateSetExtension/configs/downloader/constants/cmip6plus.yaml] found. +[2026-03-03 16:18:24] INFO [MainThread][climateset.utils] Loading YAML config file [/home/francispelletier/projects/ClimateSetExtension/configs/downloader/constants/cmip6plus.yaml]. +[2026-03-03 16:18:24] INFO [MainThread][climateset.utils] Yaml config file [/home/francispelletier/projects/ClimateSetExtension/configs/downloader/constants/imput4MIPs.yaml] found. +[2026-03-03 16:18:24] INFO [MainThread][climateset.utils] Loading YAML config file [/home/francispelletier/projects/ClimateSetExtension/configs/downloader/constants/imput4MIPs.yaml]. +[2026-03-03 16:18:24] INFO [MainThread][climateset.utils] Yaml config file [/home/francispelletier/projects/ClimateSetExtension/configs/micro_dataset.yaml] found. +[2026-03-03 16:18:24] INFO [MainThread][climateset.utils] Loading YAML config file [/home/francispelletier/projects/ClimateSetExtension/configs/micro_dataset.yaml]. +[2026-03-03 16:18:24] INFO [MainThread][climateset.download.downloader_config] Cleaned variables : ['CO2'] +[2026-03-03 16:18:24] INFO [MainThread][climateset.download.downloader_config] Emission variables to download: ['CO2_em_anthro', 'CO2_em_AIR_anthro'] +[2026-03-03 16:18:24] INFO [MainThread][climateset.download.downloader_config] Biomass burning vars to download: ['CO2'] +[2026-03-03 16:18:24] INFO [MainThread][climateset.download.downloader_config] Meta emission vars to download: + [] + [] +[2026-03-03 16:18:24] INFO [MainThread][climateset.utils] Yaml config file [/home/francispelletier/projects/ClimateSetExtension/configs/micro_dataset.yaml] found. +[2026-03-03 16:18:24] INFO [MainThread][climateset.utils] Loading YAML config file [/home/francispelletier/projects/ClimateSetExtension/configs/micro_dataset.yaml]. +[2026-03-03 16:18:24] INFO [MainThread][climateset.download.input4mips_downloader] Downloading data for variable: CO2_em_anthro +[2026-03-03 16:18:24] INFO [MainThread][climateset.download.input4mips_downloader] Using download_raw_input_single_var() function +[2026-03-03 16:18:24] INFO [MainThread][climateset.download.client] Connecting to ESGF node: https://esgf-node.llnl.gov/esg-search +[2026-03-03 16:18:25] WARNING [MainThread][climateset.download.client] Error fetching facets from https://esgf-node.llnl.gov/esg-search: 422 Client Error: Unprocessable Content for url: https://esgf-node.ornl.gov/esgf-1-5-bridge?format=application%2Fsolr%2Bjson&limit=0&distrib=false&type=Dataset&project=input4MIPs&variable=CO2_em_anthro&institution_id=PNNL-JGCRI&facets=%2A +[2026-03-03 16:18:25] INFO [MainThread][climateset.download.client] Rotating to next ESGF node... +[2026-03-03 16:18:25] INFO [MainThread][climateset.download.client] Connecting to ESGF node: https://esgf.ceda.ac.uk/esg-search +[2026-03-03 16:18:28] INFO [MainThread][climateset.download.utils] Available grid labels : ['gn'] +[2026-03-03 16:18:28] INFO [MainThread][climateset.download.utils] Choosing grid : gn +[2026-03-03 16:18:31] INFO [MainThread][climateset.download.utils] Available nominal resolution : ['50 km'] +[2026-03-03 16:18:31] INFO [MainThread][climateset.download.utils] Choosing nominal resolution : 50 km +[2026-03-03 16:18:34] INFO [MainThread][climateset.download.utils] Available frequencies : ['mon'] +[2026-03-03 16:18:34] INFO [MainThread][climateset.download.utils] Choosing default frequency : mon +[2026-03-03 16:18:37] INFO [MainThread][climateset.download.utils] Available target mips: ['CMIP'] +[2026-03-03 16:18:37] INFO [MainThread][climateset.download.client] Connecting to ESGF node: https://esgf-node.llnl.gov/esg-search +[2026-03-03 16:18:38] WARNING [MainThread][climateset.download.client] Error fetching facets from https://esgf-node.llnl.gov/esg-search: 422 Client Error: Unprocessable Content for url: https://esgf-node.ornl.gov/esgf-1-5-bridge?format=application%2Fsolr%2Bjson&limit=0&distrib=false&type=Dataset&project=input4MIPs&variable=CO2_em_anthro&institution_id=PNNL-JGCRI&grid_label=gn&nominal_resolution=50+km&frequency=mon&target_mip=CMIP&facets=%2A +[2026-03-03 16:18:38] INFO [MainThread][climateset.download.client] Rotating to next ESGF node... +[2026-03-03 16:18:38] INFO [MainThread][climateset.download.client] Connecting to ESGF node: https://esgf.ceda.ac.uk/esg-search +[2026-03-03 16:18:41] INFO [MainThread][climateset.download.utils] Available versions : ['20250421', '20250325', '20241203'] +[2026-03-03 16:18:41] INFO [MainThread][climateset.download.utils] Choosing latest version: 20250421 +[2026-03-03 16:18:45] INFO [MainThread][climateset.download.utils] Result len for target CMIP: 1 +******************************************************************************** +* * +* Note that new functionality to allow authentication without the need for * +* certificates is available with this version of the wget script. To enable, * +* use the "-H" option and enter your OpenID and password when prompted: * +* * +* $ download -H [options...] * +* * +* For a full description of the available options use the help option: * +* * +* $ download -h * +* * +******************************************************************************** +Running download version: 1.3.2 +Use download -h for help. + +Script created for 6 file(s) +(The count won't match if you manually edit this file!) + + + +CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-CMIP-2025-04-18_gn_175001-179912.nc ...Already downloaded and verified +CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-CMIP-2025-04-18_gn_180001-184912.nc ...Already downloaded and verified +CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-CMIP-2025-04-18_gn_185001-189912.nc ...Already downloaded and verified +CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-CMIP-2025-04-18_gn_190001-194912.nc ...Already downloaded and verified +CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-CMIP-2025-04-18_gn_195001-199912.nc ...Already downloaded and verified +CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-CMIP-2025-04-18_gn_200001-202312.nc ...Already downloaded and verified +done +[2026-03-03 16:18:52] INFO [MainThread][climateset.download.input4mips_downloader] Download results: [] +[2026-03-03 16:18:52] INFO [MainThread][climateset.download.input4mips_downloader] Downloading data for variable: CO2_em_AIR_anthro +[2026-03-03 16:18:52] INFO [MainThread][climateset.download.input4mips_downloader] Using download_raw_input_single_var() function +[2026-03-03 16:18:52] INFO [MainThread][climateset.download.client] Connecting to ESGF node: https://esgf-node.llnl.gov/esg-search +[2026-03-03 16:18:53] WARNING [MainThread][climateset.download.client] Error fetching facets from https://esgf-node.llnl.gov/esg-search: 422 Client Error: Unprocessable Content for url: https://esgf-node.ornl.gov/esgf-1-5-bridge?format=application%2Fsolr%2Bjson&limit=0&distrib=false&type=Dataset&project=input4MIPs&variable=CO2_em_AIR_anthro&institution_id=PNNL-JGCRI&facets=%2A +[2026-03-03 16:18:53] INFO [MainThread][climateset.download.client] Rotating to next ESGF node... +[2026-03-03 16:18:53] INFO [MainThread][climateset.download.client] Connecting to ESGF node: https://esgf.ceda.ac.uk/esg-search +[2026-03-03 16:18:56] INFO [MainThread][climateset.download.utils] Available grid labels : ['gn'] +[2026-03-03 16:18:56] INFO [MainThread][climateset.download.utils] Choosing grid : gn +[2026-03-03 16:18:59] INFO [MainThread][climateset.download.utils] Available nominal resolution : ['50 km'] +[2026-03-03 16:18:59] INFO [MainThread][climateset.download.utils] Choosing nominal resolution : 50 km +[2026-03-03 16:19:03] INFO [MainThread][climateset.download.utils] Available frequencies : ['mon'] +[2026-03-03 16:19:03] INFO [MainThread][climateset.download.utils] Choosing default frequency : mon +[2026-03-03 16:19:06] INFO [MainThread][climateset.download.utils] Available target mips: ['CMIP'] +[2026-03-03 16:19:06] INFO [MainThread][climateset.download.client] Connecting to ESGF node: https://esgf-node.llnl.gov/esg-search +[2026-03-03 16:19:06] WARNING [MainThread][climateset.download.client] Error fetching facets from https://esgf-node.llnl.gov/esg-search: 422 Client Error: Unprocessable Content for url: https://esgf-node.ornl.gov/esgf-1-5-bridge?format=application%2Fsolr%2Bjson&limit=0&distrib=false&type=Dataset&project=input4MIPs&variable=CO2_em_AIR_anthro&institution_id=PNNL-JGCRI&grid_label=gn&nominal_resolution=50+km&frequency=mon&target_mip=CMIP&facets=%2A +[2026-03-03 16:19:06] INFO [MainThread][climateset.download.client] Rotating to next ESGF node... +[2026-03-03 16:19:06] INFO [MainThread][climateset.download.client] Connecting to ESGF node: https://esgf.ceda.ac.uk/esg-search +[2026-03-03 16:19:09] INFO [MainThread][climateset.download.utils] Available versions : ['20250421', '20250325', '20241109'] +[2026-03-03 16:19:09] INFO [MainThread][climateset.download.utils] Choosing latest version: 20250421 +[2026-03-03 16:19:13] INFO [MainThread][climateset.download.utils] Result len for target CMIP: 1 +******************************************************************************** +* * +* Note that new functionality to allow authentication without the need for * +* certificates is available with this version of the wget script. To enable, * +* use the "-H" option and enter your OpenID and password when prompted: * +* * +* $ download -H [options...] * +* * +* For a full description of the available options use the help option: * +* * +* $ download -h * +* * +******************************************************************************** +Running download version: 1.3.2 +Use download -h for help. + +Script created for 6 file(s) +(The count won't match if you manually edit this file!) + + + +CO2-em-AIR-anthro_input4MIPs_emissions_CMIP_CEDS-CMIP-2025-04-18_gn_175001-179912.nc ...Already downloaded and verified +CO2-em-AIR-anthro_input4MIPs_emissions_CMIP_CEDS-CMIP-2025-04-18_gn_180001-184912.nc ...Already downloaded and verified +CO2-em-AIR-anthro_input4MIPs_emissions_CMIP_CEDS-CMIP-2025-04-18_gn_185001-189912.nc ...Already downloaded and verified +CO2-em-AIR-anthro_input4MIPs_emissions_CMIP_CEDS-CMIP-2025-04-18_gn_190001-194912.nc ...Already downloaded and verified +CO2-em-AIR-anthro_input4MIPs_emissions_CMIP_CEDS-CMIP-2025-04-18_gn_195001-199912.nc ...Already downloaded and verified +CO2-em-AIR-anthro_input4MIPs_emissions_CMIP_CEDS-CMIP-2025-04-18_gn_200001-202312.nc ...Already downloaded and verified +done +[2026-03-03 16:19:20] INFO [MainThread][climateset.download.input4mips_downloader] Download results: [] +[2026-03-03 16:19:20] INFO [MainThread][climateset.download.input4mips_downloader] Downloading biomassburing data for variable: CO2 +[2026-03-03 16:19:20] INFO [MainThread][climateset.download.input4mips_downloader] Using download_raw_input_single_var() function +[2026-03-03 16:19:20] INFO [MainThread][climateset.download.client] Connecting to ESGF node: https://esgf-node.llnl.gov/esg-search +[2026-03-03 16:19:20] WARNING [MainThread][climateset.download.client] Error fetching facets from https://esgf-node.llnl.gov/esg-search: 422 Client Error: Unprocessable Content for url: https://esgf-node.ornl.gov/esgf-1-5-bridge?format=application%2Fsolr%2Bjson&limit=0&distrib=false&type=Dataset&project=input4MIPs&variable=CO2&institution_id=VUA&facets=%2A +[2026-03-03 16:19:20] INFO [MainThread][climateset.download.client] Rotating to next ESGF node... +[2026-03-03 16:19:20] INFO [MainThread][climateset.download.client] Connecting to ESGF node: https://esgf.ceda.ac.uk/esg-search +[2026-03-03 16:19:23] INFO [MainThread][climateset.download.utils] Available grid labels : ['gn'] +[2026-03-03 16:19:23] INFO [MainThread][climateset.download.utils] Choosing grid : gn +[2026-03-03 16:19:27] INFO [MainThread][climateset.download.utils] Available nominal resolution : ['25 km'] +[2026-03-03 16:19:27] INFO [MainThread][climateset.download.utils] Choosing nominal resolution : 25 km +[2026-03-03 16:19:30] INFO [MainThread][climateset.download.utils] Available frequencies : ['mon'] +[2026-03-03 16:19:30] INFO [MainThread][climateset.download.utils] Choosing default frequency : mon +[2026-03-03 16:19:33] INFO [MainThread][climateset.download.utils] Available target mips: ['CMIP'] +[2026-03-03 16:19:33] INFO [MainThread][climateset.download.client] Connecting to ESGF node: https://esgf-node.llnl.gov/esg-search +[2026-03-03 16:19:33] WARNING [MainThread][climateset.download.client] Error fetching facets from https://esgf-node.llnl.gov/esg-search: 422 Client Error: Unprocessable Content for url: https://esgf-node.ornl.gov/esgf-1-5-bridge?format=application%2Fsolr%2Bjson&limit=0&distrib=false&type=Dataset&project=input4MIPs&variable=CO2&institution_id=VUA&grid_label=gn&nominal_resolution=25+km&frequency=mon&target_mip=CMIP&facets=%2A +[2026-03-03 16:19:33] INFO [MainThread][climateset.download.client] Rotating to next ESGF node... +[2026-03-03 16:19:33] INFO [MainThread][climateset.download.client] Connecting to ESGF node: https://esgf.ceda.ac.uk/esg-search +[2026-03-03 16:19:36] INFO [MainThread][climateset.download.utils] Available versions : ['20161002', '20160705'] +[2026-03-03 16:19:36] INFO [MainThread][climateset.download.utils] Choosing latest version: 20161002 +[2026-03-03 16:19:40] INFO [MainThread][climateset.download.utils] Result len for target CMIP: 1 +******************************************************************************** +* * +* Note that new functionality to allow authentication without the need for * +* certificates is available with this version of the wget script. To enable, * +* use the "-H" option and enter your OpenID and password when prompted: * +* * +* $ download -H [options...] * +* * +* For a full description of the available options use the help option: * +* * +* $ download -h * +* * +******************************************************************************** +Running download version: 1.3.2 +Use download -h for help. + +Script created for 2 file(s) +(The count won't match if you manually edit this file!) + + + +CO2-em-biomassburning_input4MIPs_emissions_CMIP_VUA-CMIP-BB4CMIP6-1-1_gn_175001-184912.nc ...Already downloaded and verified +CO2-em-biomassburning_input4MIPs_emissions_CMIP_VUA-CMIP-BB4CMIP6-1-1_gn_185001-201512.nc ...Already downloaded and verified +done +[2026-03-03 16:19:47] INFO [MainThread][climateset.download.input4mips_downloader] Download results: [] +[2026-03-03 16:19:47] INFO [MainThread][climateset.download.cmip6_downloader] Downloading data for model: [NorESM2-LM] +[2026-03-03 16:19:47] INFO [MainThread][climateset.download.cmip6_downloader] Downloading data for variable: [tas] +[2026-03-03 16:19:47] INFO [MainThread][climateset.download.cmip6_downloader] Downloading data for experiment: [historical] +[2026-03-03 16:19:47] INFO [MainThread][climateset.download.utils] Using download_from_model_single_var() function +[2026-03-03 16:19:47] INFO [MainThread][climateset.download.client] Connecting to ESGF node: https://esgf-node.llnl.gov/esg-search + +------------------------------------------------------------------------------- +Warning - defaulting to search with facets=* + +This behavior is kept for backward-compatibility, but ESGF indexes might not +successfully perform a distributed search when this option is used, so some +results may be missing. For full results, it is recommended to pass a list of +facets of interest when instantiating a context object. For example, + + ctx = conn.new_context(facets='project,experiment_id') + +Only the facets that you specify will be present in the facets_counts dictionary. + +This warning is displayed when a distributed search is performed while using the +facets=* default, a maximum of once per context object. To suppress this warning, +set the environment variable ESGF_PYCLIENT_NO_FACETS_STAR_WARNING to any value +or explicitly use conn.new_context(facets='*') + +------------------------------------------------------------------------------- +[2026-03-03 16:19:47] WARNING [MainThread][climateset.download.client] Error fetching facets from https://esgf-node.llnl.gov/esg-search: 422 Client Error: Unprocessable Content for url: https://esgf-node.ornl.gov/esgf-1-5-bridge?format=application%2Fsolr%2Bjson&limit=0&distrib=true&type=Dataset&project=CMIP6&variable=tas&experiment_id=historical&source_id=NorESM2-LM&facets=%2A +[2026-03-03 16:19:47] INFO [MainThread][climateset.download.client] Rotating to next ESGF node... +[2026-03-03 16:19:47] INFO [MainThread][climateset.download.client] Connecting to ESGF node: https://esgf.ceda.ac.uk/esg-search + +------------------------------------------------------------------------------- +Warning - defaulting to search with facets=* + +This behavior is kept for backward-compatibility, but ESGF indexes might not +successfully perform a distributed search when this option is used, so some +results may be missing. For full results, it is recommended to pass a list of +facets of interest when instantiating a context object. For example, + + ctx = conn.new_context(facets='project,experiment_id') + +Only the facets that you specify will be present in the facets_counts dictionary. + +This warning is displayed when a distributed search is performed while using the +facets=* default, a maximum of once per context object. To suppress this warning, +set the environment variable ESGF_PYCLIENT_NO_FACETS_STAR_WARNING to any value +or explicitly use conn.new_context(facets='*') + +------------------------------------------------------------------------------- +[2026-03-03 16:19:56] INFO [MainThread][climateset.download.utils] Available frequencies : ['mon', 'day', '6hrPt', '6hr'] +[2026-03-03 16:19:56] INFO [MainThread][climateset.download.utils] Choosing default frequency : mon +[2026-03-03 16:20:04] INFO [MainThread][climateset.download.utils] Available grid labels : ['gn'] +[2026-03-03 16:20:04] INFO [MainThread][climateset.download.utils] Choosing grid : gn +[2026-03-03 16:20:12] INFO [MainThread][climateset.download.utils] Available variants : ['r9i1p1f1', 'r8i1p1f1', 'r7i1p1f1', 'r6i1p1f1', 'r5i1p1f1', 'r4i1p1f1', 'r43i1p1f1', 'r42i1p1f1', 'r41i1p1f1', 'r40i1p1f1', 'r3i1p1f1', 'r39i1p1f1', 'r38i1p1f1', 'r37i1p1f1', 'r36i1p1f1', 'r35i1p1f1', 'r34i1p1f1', 'r33i1p1f1', 'r32i1p1f1', 'r31i1p1f1', 'r30i1p1f1', 'r2i1p1f1', 'r29i1p1f1', 'r28i1p1f1', 'r27i1p1f1', 'r26i1p1f1', 'r25i1p1f1', 'r24i1p1f1', 'r23i1p1f1', 'r22i1p1f1', 'r21i1p1f1', 'r20i1p1f1', 'r1i1p4f1', 'r1i1p1f1', 'r19i1p1f1', 'r18i1p1f1', 'r17i1p1f1', 'r16i1p1f1', 'r15i1p1f1', 'r14i1p1f1', 'r13i1p1f1', 'r12i1p1f1', 'r11i1p1f1', 'r10i1p1f1'] + +[2026-03-03 16:20:12] INFO [MainThread][climateset.download.utils] Length : 44 +[2026-03-03 16:20:12] INFO [MainThread][climateset.download.utils] Desired list of ensemble members given: ['r2i1p1f1'] +[2026-03-03 16:20:12] INFO [MainThread][climateset.download.utils] Ensembles member: r2i1p1f1 +[2026-03-03 16:20:12] INFO [MainThread][climateset.download.client] Connecting to ESGF node: https://esgf-node.llnl.gov/esg-search + +------------------------------------------------------------------------------- +Warning - defaulting to search with facets=* + +This behavior is kept for backward-compatibility, but ESGF indexes might not +successfully perform a distributed search when this option is used, so some +results may be missing. For full results, it is recommended to pass a list of +facets of interest when instantiating a context object. For example, + + ctx = conn.new_context(facets='project,experiment_id') + +Only the facets that you specify will be present in the facets_counts dictionary. + +This warning is displayed when a distributed search is performed while using the +facets=* default, a maximum of once per context object. To suppress this warning, +set the environment variable ESGF_PYCLIENT_NO_FACETS_STAR_WARNING to any value +or explicitly use conn.new_context(facets='*') + +------------------------------------------------------------------------------- +[2026-03-03 16:20:13] WARNING [MainThread][climateset.download.client] Error fetching facets from https://esgf-node.llnl.gov/esg-search: 422 Client Error: Unprocessable Content for url: https://esgf-node.ornl.gov/esgf-1-5-bridge?format=application%2Fsolr%2Bjson&limit=0&distrib=true&type=Dataset&project=CMIP6&variable=tas&experiment_id=historical&source_id=NorESM2-LM&frequency=mon&grid_label=gn&variant_label=r2i1p1f1&facets=%2A +[2026-03-03 16:20:13] INFO [MainThread][climateset.download.client] Rotating to next ESGF node... +[2026-03-03 16:20:13] INFO [MainThread][climateset.download.client] Connecting to ESGF node: https://esgf.ceda.ac.uk/esg-search + +------------------------------------------------------------------------------- +Warning - defaulting to search with facets=* + +This behavior is kept for backward-compatibility, but ESGF indexes might not +successfully perform a distributed search when this option is used, so some +results may be missing. For full results, it is recommended to pass a list of +facets of interest when instantiating a context object. For example, + + ctx = conn.new_context(facets='project,experiment_id') + +Only the facets that you specify will be present in the facets_counts dictionary. + +This warning is displayed when a distributed search is performed while using the +facets=* default, a maximum of once per context object. To suppress this warning, +set the environment variable ESGF_PYCLIENT_NO_FACETS_STAR_WARNING to any value +or explicitly use conn.new_context(facets='*') + +------------------------------------------------------------------------------- +[2026-03-03 16:20:21] INFO [MainThread][climateset.download.utils] Available versions : ['20190920'] +[2026-03-03 16:20:21] INFO [MainThread][climateset.download.utils] Choosing latest version: 20190920 +[2026-03-03 16:20:31] INFO [MainThread][climateset.download.utils] Result len 3 +[2026-03-03 16:20:31] INFO [MainThread][climateset.download.utils] [] + +------------------------------------------------------------------------------- +Warning - defaulting to search with facets=* + +This behavior is kept for backward-compatibility, but ESGF indexes might not +successfully perform a distributed search when this option is used, so some +results may be missing. For full results, it is recommended to pass a list of +facets of interest when instantiating a context object. For example, + + ctx = conn.new_context(facets='project,experiment_id') + +Only the facets that you specify will be present in the facets_counts dictionary. + +This warning is displayed when a distributed search is performed while using the +facets=* default, a maximum of once per context object. To suppress this warning, +set the environment variable ESGF_PYCLIENT_NO_FACETS_STAR_WARNING to any value +or explicitly use conn.new_context(facets='*') + +------------------------------------------------------------------------------- +******************************************************************************** +* * +* Note that new functionality to allow authentication without the need for * +* certificates is available with this version of the wget script. To enable, * +* use the "-H" option and enter your OpenID and password when prompted: * +* * +* $ download -H [options...] * +* * +* For a full description of the available options use the help option: * +* * +* $ download -h * +* * +******************************************************************************** +Running download version: 1.3.2 +Use download -h for help. + +Script created for 17 file(s) +(The count won't match if you manually edit this file!) + + + +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_185001-185912.nc ...Downloading +--2026-03-03 16:20:38-- https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/NCC/NorESM2-LM/historical/r2i1p1f1/Amon/tas/gn/v20190920/tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_185001-185912.nc +Resolving esgf.ceda.ac.uk (esgf.ceda.ac.uk)... 130.246.128.97 +Connecting to esgf.ceda.ac.uk (esgf.ceda.ac.uk)|130.246.128.97|:443... connected. +HTTP request sent, awaiting response... 200 OK +Length: 3921956 (3,7M) [application/octet-stream] +Saving to: ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_185001-185912.nc’ + +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_185001-185912.nc 100%[=========================================================================================================================================================================================================================>] 3,74M 925KB/s in 4,1s + +2026-03-03 16:20:43 (925 KB/s) - ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_185001-185912.nc’ saved [3921956/3921956] + + sha256 ok. done! +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_186001-186912.nc ...Downloading +--2026-03-03 16:20:43-- https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/NCC/NorESM2-LM/historical/r2i1p1f1/Amon/tas/gn/v20190920/tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_186001-186912.nc +Resolving esgf.ceda.ac.uk (esgf.ceda.ac.uk)... 130.246.128.97 +Connecting to esgf.ceda.ac.uk (esgf.ceda.ac.uk)|130.246.128.97|:443... connected. +HTTP request sent, awaiting response... 200 OK +Length: 3921991 (3,7M) [application/octet-stream] +Saving to: ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_186001-186912.nc’ + +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_186001-186912.nc 100%[=========================================================================================================================================================================================================================>] 3,74M 1,04MB/s in 3,6s + +2026-03-03 16:20:47 (1,04 MB/s) - ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_186001-186912.nc’ saved [3921991/3921991] + + sha256 ok. done! +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_187001-187912.nc ...Downloading +--2026-03-03 16:20:47-- https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/NCC/NorESM2-LM/historical/r2i1p1f1/Amon/tas/gn/v20190920/tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_187001-187912.nc +Resolving esgf.ceda.ac.uk (esgf.ceda.ac.uk)... 130.246.128.97 +Connecting to esgf.ceda.ac.uk (esgf.ceda.ac.uk)|130.246.128.97|:443... connected. +HTTP request sent, awaiting response... 200 OK +Length: 3922810 (3,7M) [application/octet-stream] +Saving to: ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_187001-187912.nc’ + +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_187001-187912.nc 100%[=========================================================================================================================================================================================================================>] 3,74M 956KB/s in 4,1s + +2026-03-03 16:20:52 (926 KB/s) - ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_187001-187912.nc’ saved [3922810/3922810] + + sha256 ok. done! +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_188001-188912.nc ...Downloading +--2026-03-03 16:20:52-- https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/NCC/NorESM2-LM/historical/r2i1p1f1/Amon/tas/gn/v20190920/tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_188001-188912.nc +Resolving esgf.ceda.ac.uk (esgf.ceda.ac.uk)... 130.246.128.97 +Connecting to esgf.ceda.ac.uk (esgf.ceda.ac.uk)|130.246.128.97|:443... connected. +HTTP request sent, awaiting response... 200 OK +Length: 3920723 (3,7M) [application/octet-stream] +Saving to: ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_188001-188912.nc’ + +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_188001-188912.nc 100%[=========================================================================================================================================================================================================================>] 3,74M 953KB/s in 4,0s + +2026-03-03 16:20:56 (953 KB/s) - ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_188001-188912.nc’ saved [3920723/3920723] + + sha256 ok. done! +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_189001-189912.nc ...Downloading +--2026-03-03 16:20:56-- https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/NCC/NorESM2-LM/historical/r2i1p1f1/Amon/tas/gn/v20190920/tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_189001-189912.nc +Resolving esgf.ceda.ac.uk (esgf.ceda.ac.uk)... 130.246.128.97 +Connecting to esgf.ceda.ac.uk (esgf.ceda.ac.uk)|130.246.128.97|:443... connected. +HTTP request sent, awaiting response... 200 OK +Length: 3920346 (3,7M) [application/octet-stream] +Saving to: ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_189001-189912.nc’ + +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_189001-189912.nc 100%[=========================================================================================================================================================================================================================>] 3,74M 958KB/s in 4,2s + +2026-03-03 16:21:01 (905 KB/s) - ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_189001-189912.nc’ saved [3920346/3920346] + + sha256 ok. done! +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_190001-190912.nc ...Downloading +--2026-03-03 16:21:01-- https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/NCC/NorESM2-LM/historical/r2i1p1f1/Amon/tas/gn/v20190920/tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_190001-190912.nc +Resolving esgf.ceda.ac.uk (esgf.ceda.ac.uk)... 130.246.128.97 +Connecting to esgf.ceda.ac.uk (esgf.ceda.ac.uk)|130.246.128.97|:443... connected. +HTTP request sent, awaiting response... 200 OK +Length: 3922637 (3,7M) [application/octet-stream] +Saving to: ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_190001-190912.nc’ + +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_190001-190912.nc 100%[=========================================================================================================================================================================================================================>] 3,74M 1019KB/s in 3,8s + +2026-03-03 16:21:06 (1019 KB/s) - ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_190001-190912.nc’ saved [3922637/3922637] + + sha256 ok. done! +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_191001-191912.nc ...Downloading +--2026-03-03 16:21:06-- https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/NCC/NorESM2-LM/historical/r2i1p1f1/Amon/tas/gn/v20190920/tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_191001-191912.nc +Resolving esgf.ceda.ac.uk (esgf.ceda.ac.uk)... 130.246.128.97 +Connecting to esgf.ceda.ac.uk (esgf.ceda.ac.uk)|130.246.128.97|:443... connected. +HTTP request sent, awaiting response... 200 OK +Length: 3921878 (3,7M) [application/octet-stream] +Saving to: ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_191001-191912.nc’ + +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_191001-191912.nc 100%[=========================================================================================================================================================================================================================>] 3,74M 966KB/s in 4,1s + +2026-03-03 16:21:10 (933 KB/s) - ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_191001-191912.nc’ saved [3921878/3921878] + + sha256 ok. done! +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_192001-192912.nc ...Downloading +--2026-03-03 16:21:10-- https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/NCC/NorESM2-LM/historical/r2i1p1f1/Amon/tas/gn/v20190920/tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_192001-192912.nc +Resolving esgf.ceda.ac.uk (esgf.ceda.ac.uk)... 130.246.128.97 +Connecting to esgf.ceda.ac.uk (esgf.ceda.ac.uk)|130.246.128.97|:443... connected. +HTTP request sent, awaiting response... 200 OK +Length: 3918591 (3,7M) [application/octet-stream] +Saving to: ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_192001-192912.nc’ + +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_192001-192912.nc 100%[=========================================================================================================================================================================================================================>] 3,74M 968KB/s in 4,1s + +2026-03-03 16:21:15 (928 KB/s) - ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_192001-192912.nc’ saved [3918591/3918591] + + sha256 ok. done! +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_193001-193912.nc ...Downloading +--2026-03-03 16:21:15-- https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/NCC/NorESM2-LM/historical/r2i1p1f1/Amon/tas/gn/v20190920/tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_193001-193912.nc +Resolving esgf.ceda.ac.uk (esgf.ceda.ac.uk)... 130.246.128.97 +Connecting to esgf.ceda.ac.uk (esgf.ceda.ac.uk)|130.246.128.97|:443... connected. +HTTP request sent, awaiting response... 200 OK +Length: 3920337 (3,7M) [application/octet-stream] +Saving to: ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_193001-193912.nc’ + +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_193001-193912.nc 100%[=========================================================================================================================================================================================================================>] 3,74M 1,02MB/s in 3,7s + +2026-03-03 16:21:19 (1,02 MB/s) - ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_193001-193912.nc’ saved [3920337/3920337] + + sha256 ok. done! +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_194001-194912.nc ...Downloading +--2026-03-03 16:21:19-- https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/NCC/NorESM2-LM/historical/r2i1p1f1/Amon/tas/gn/v20190920/tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_194001-194912.nc +Resolving esgf.ceda.ac.uk (esgf.ceda.ac.uk)... 130.246.128.97 +Connecting to esgf.ceda.ac.uk (esgf.ceda.ac.uk)|130.246.128.97|:443... connected. +HTTP request sent, awaiting response... 200 OK +Length: 3921350 (3,7M) [application/octet-stream] +Saving to: ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_194001-194912.nc’ + +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_194001-194912.nc 100%[=========================================================================================================================================================================================================================>] 3,74M 1011KB/s in 3,8s + +2026-03-03 16:21:24 (1011 KB/s) - ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_194001-194912.nc’ saved [3921350/3921350] + + sha256 ok. done! +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_195001-195912.nc ...Downloading +--2026-03-03 16:21:24-- https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/NCC/NorESM2-LM/historical/r2i1p1f1/Amon/tas/gn/v20190920/tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_195001-195912.nc +Resolving esgf.ceda.ac.uk (esgf.ceda.ac.uk)... 130.246.128.97 +Connecting to esgf.ceda.ac.uk (esgf.ceda.ac.uk)|130.246.128.97|:443... connected. +HTTP request sent, awaiting response... 200 OK +Length: 3920415 (3,7M) [application/octet-stream] +Saving to: ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_195001-195912.nc’ + +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_195001-195912.nc 100%[=========================================================================================================================================================================================================================>] 3,74M 2,85MB/s in 1,3s + +2026-03-03 16:21:26 (2,85 MB/s) - ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_195001-195912.nc’ saved [3920415/3920415] + + sha256 ok. done! +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_196001-196912.nc ...Downloading +--2026-03-03 16:21:26-- https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/NCC/NorESM2-LM/historical/r2i1p1f1/Amon/tas/gn/v20190920/tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_196001-196912.nc +Resolving esgf.ceda.ac.uk (esgf.ceda.ac.uk)... 130.246.128.97 +Connecting to esgf.ceda.ac.uk (esgf.ceda.ac.uk)|130.246.128.97|:443... connected. +HTTP request sent, awaiting response... 200 OK +Length: 3920774 (3,7M) [application/octet-stream] +Saving to: ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_196001-196912.nc’ + +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_196001-196912.nc 100%[=========================================================================================================================================================================================================================>] 3,74M 1,06MB/s in 3,5s + +2026-03-03 16:21:30 (1,06 MB/s) - ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_196001-196912.nc’ saved [3920774/3920774] + + sha256 ok. done! +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_197001-197912.nc ...Downloading +--2026-03-03 16:21:30-- https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/NCC/NorESM2-LM/historical/r2i1p1f1/Amon/tas/gn/v20190920/tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_197001-197912.nc +Resolving esgf.ceda.ac.uk (esgf.ceda.ac.uk)... 130.246.128.97 +Connecting to esgf.ceda.ac.uk (esgf.ceda.ac.uk)|130.246.128.97|:443... connected. +HTTP request sent, awaiting response... 200 OK +Length: 3920052 (3,7M) [application/octet-stream] +Saving to: ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_197001-197912.nc’ + +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_197001-197912.nc 100%[=========================================================================================================================================================================================================================>] 3,74M 1,03MB/s in 3,6s + +2026-03-03 16:21:34 (1,03 MB/s) - ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_197001-197912.nc’ saved [3920052/3920052] + + sha256 ok. done! +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_198001-198912.nc ...Downloading +--2026-03-03 16:21:34-- https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/NCC/NorESM2-LM/historical/r2i1p1f1/Amon/tas/gn/v20190920/tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_198001-198912.nc +Resolving esgf.ceda.ac.uk (esgf.ceda.ac.uk)... 130.246.128.97 +Connecting to esgf.ceda.ac.uk (esgf.ceda.ac.uk)|130.246.128.97|:443... connected. +HTTP request sent, awaiting response... 200 OK +Length: 3919588 (3,7M) [application/octet-stream] +Saving to: ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_198001-198912.nc’ + +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_198001-198912.nc 100%[=========================================================================================================================================================================================================================>] 3,74M 969KB/s in 3,9s + +2026-03-03 16:21:39 (969 KB/s) - ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_198001-198912.nc’ saved [3919588/3919588] + + sha256 ok. done! +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_199001-199912.nc ...Downloading +--2026-03-03 16:21:39-- https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/NCC/NorESM2-LM/historical/r2i1p1f1/Amon/tas/gn/v20190920/tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_199001-199912.nc +Resolving esgf.ceda.ac.uk (esgf.ceda.ac.uk)... 130.246.128.97 +Connecting to esgf.ceda.ac.uk (esgf.ceda.ac.uk)|130.246.128.97|:443... connected. +HTTP request sent, awaiting response... 200 OK +Length: 3919135 (3,7M) [application/octet-stream] +Saving to: ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_199001-199912.nc’ + +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_199001-199912.nc 100%[=========================================================================================================================================================================================================================>] 3,74M 821KB/s in 5,1s + +2026-03-03 16:21:44 (748 KB/s) - ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_199001-199912.nc’ saved [3919135/3919135] + + sha256 ok. done! +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_200001-200912.nc ...Downloading +--2026-03-03 16:21:45-- https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/NCC/NorESM2-LM/historical/r2i1p1f1/Amon/tas/gn/v20190920/tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_200001-200912.nc +Resolving esgf.ceda.ac.uk (esgf.ceda.ac.uk)... 130.246.128.97 +Connecting to esgf.ceda.ac.uk (esgf.ceda.ac.uk)|130.246.128.97|:443... connected. +HTTP request sent, awaiting response... 200 OK +Length: 3915211 (3,7M) [application/octet-stream] +Saving to: ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_200001-200912.nc’ + +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_200001-200912.nc 100%[=========================================================================================================================================================================================================================>] 3,73M 3,63MB/s in 1,0s + +2026-03-03 16:21:46 (3,63 MB/s) - ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_200001-200912.nc’ saved [3915211/3915211] + + sha256 ok. done! +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_201001-201412.nc ...Downloading +--2026-03-03 16:21:46-- https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/NCC/NorESM2-LM/historical/r2i1p1f1/Amon/tas/gn/v20190920/tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_201001-201412.nc +Resolving esgf.ceda.ac.uk (esgf.ceda.ac.uk)... 130.246.128.97 +Connecting to esgf.ceda.ac.uk (esgf.ceda.ac.uk)|130.246.128.97|:443... connected. +HTTP request sent, awaiting response... 200 OK +Length: 1978507 (1,9M) [application/octet-stream] +Saving to: ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_201001-201412.nc’ + +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_201001-201412.nc 100%[=========================================================================================================================================================================================================================>] 1,89M 2,38MB/s in 0,8s + +2026-03-03 16:21:47 (2,38 MB/s) - ‘tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_201001-201412.nc’ saved [1978507/1978507] + + sha256 ok. done! +done + +------------------------------------------------------------------------------- +Warning - defaulting to search with facets=* + +This behavior is kept for backward-compatibility, but ESGF indexes might not +successfully perform a distributed search when this option is used, so some +results may be missing. For full results, it is recommended to pass a list of +facets of interest when instantiating a context object. For example, + + ctx = conn.new_context(facets='project,experiment_id') + +Only the facets that you specify will be present in the facets_counts dictionary. + +This warning is displayed when a distributed search is performed while using the +facets=* default, a maximum of once per context object. To suppress this warning, +set the environment variable ESGF_PYCLIENT_NO_FACETS_STAR_WARNING to any value +or explicitly use conn.new_context(facets='*') + +------------------------------------------------------------------------------- +******************************************************************************** +* * +* Note that new functionality to allow authentication without the need for * +* certificates is available with this version of the wget script. To enable, * +* use the "-H" option and enter your OpenID and password when prompted: * +* * +* $ download -H [options...] * +* * +* For a full description of the available options use the help option: * +* * +* $ download -h * +* * +******************************************************************************** +Running download version: 1.3.2 +Use download -h for help. + +Script created for 17 file(s) +(The count won't match if you manually edit this file!) + + + +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_185001-185912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_186001-186912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_187001-187912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_188001-188912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_189001-189912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_190001-190912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_191001-191912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_192001-192912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_193001-193912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_194001-194912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_195001-195912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_196001-196912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_197001-197912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_198001-198912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_199001-199912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_200001-200912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_201001-201412.nc ...Already downloaded and verified +done + +------------------------------------------------------------------------------- +Warning - defaulting to search with facets=* + +This behavior is kept for backward-compatibility, but ESGF indexes might not +successfully perform a distributed search when this option is used, so some +results may be missing. For full results, it is recommended to pass a list of +facets of interest when instantiating a context object. For example, + + ctx = conn.new_context(facets='project,experiment_id') + +Only the facets that you specify will be present in the facets_counts dictionary. + +This warning is displayed when a distributed search is performed while using the +facets=* default, a maximum of once per context object. To suppress this warning, +set the environment variable ESGF_PYCLIENT_NO_FACETS_STAR_WARNING to any value +or explicitly use conn.new_context(facets='*') + +------------------------------------------------------------------------------- +******************************************************************************** +* * +* Note that new functionality to allow authentication without the need for * +* certificates is available with this version of the wget script. To enable, * +* use the "-H" option and enter your OpenID and password when prompted: * +* * +* $ download -H [options...] * +* * +* For a full description of the available options use the help option: * +* * +* $ download -h * +* * +******************************************************************************** +Running download version: 1.3.2 +Use download -h for help. + +Script created for 17 file(s) +(The count won't match if you manually edit this file!) + + + +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_185001-185912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_186001-186912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_187001-187912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_188001-188912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_189001-189912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_190001-190912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_191001-191912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_192001-192912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_193001-193912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_194001-194912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_195001-195912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_196001-196912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_197001-197912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_198001-198912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_199001-199912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_200001-200912.nc ...Already downloaded and verified +tas_Amon_NorESM2-LM_historical_r2i1p1f1_gn_201001-201412.nc ...Already downloaded and verified +done \ No newline at end of file diff --git a/docs/agents/planning/esgpull_implementation/esgf_pyclient_analysis.md b/docs/agents/planning/esgpull_implementation/esgf_pyclient_analysis.md new file mode 100644 index 0000000..d5af247 --- /dev/null +++ b/docs/agents/planning/esgpull_implementation/esgf_pyclient_analysis.md @@ -0,0 +1,37 @@ +# Analysis of `esgf-pyclient` Usage in the Repository + +This document analyzes how `esgf-pyclient` is integrated and utilized within the `climateset` project, based on the codebase (`climateset/download/client.py`, `climateset/download/utils.py`) and execution logs (`docs/agents/planning/download_logs.md`). + +## 1. Abstraction and Failover Architecture (`client.py`) + +The project wraps `esgf-pyclient` rather than using it directly. `SearchClient` acts as a factory for `SearchSession` objects, which directly manage stateful search building and automatic failover across different ESGF index nodes. + +### Failover Mechanism +- **Node Rotation**: `SearchSession` is initialized with a list of ESGF node URLs (falling back to `NODE_LINK_URLS`). +- **Constraint Replay**: Rather than failing a process entirely when a node times out or returns a 422 Client Error, `SearchSession` maintains a history of constraints in `_constraints_history`. +- When an operation (like `.constrain()`, `.get_available_facets()`, or `.search()`) utilizing the underlying `SearchConnection` fails, the `_rotate_node()` method triggers. It selects the next available URL, establishes a new connection, opens a new dataset context, and iterates over `_constraints_history` to replay all previously applied constraints. +- **Evidence from Logs**: The execution logs demonstrate this behavior in action. When `https://esgf-node.llnl.gov/esg-search` repeatedly returns `422 Unprocessable Content`, the client seamlessly logs `Rotating to next ESGF node...` and switches to `https://esgf.ceda.ac.uk/esg-search`. + +## 2. Iterative Search Pattern (`utils.py`) + +The extraction pipeline is modeled heavily around the dynamic properties of `esgf-pyclient`. It relies on an iterative exploration of facets rather than applying all constraints statically via a single API call. + +### Dynamic Constraint Discovery +Methods like `search_and_download_esgf_raw_single_var` and `search_and_download_esgf_model_single_var` sequentially drive `SearchSession`: +1. **Initial Broad Constraint**: They begin by restricting the query to basic parameters like `project`, `institution_id`, `variable` via wrapper classes (`CMIP6Constraints`, `Input4MIPsConstraints`). +2. **Facet Extraction**: Under the hood, they query the context bounds using `session.get_available_facets(facet_name)` (for schemas like `grid_label`, `nominal_resolution`, `frequency`). This internally surfaces `ctx.facet_counts`. +3. **Refinement**: Helper functions (`get_frequency()`, `get_grid_label()`) verify whether the user's requested parameter exists within the available facets pulled from the ESGF node. If it does, the constraint is tightened; otherwise, it degrades gracefully (usually defaulting to the first available option). +4. **Iterative Branching**: For varying dimension bounds like `target_mip` or `variant_label` (ensemble members), the script forks the session (`sub_session = client.new_session()`), replays the baseline constraints, and searches each combination iteratively rather than fetching bulk records. + +## 3. Distributed vs. Non-Distributed Search + +- `utils.py` selectively applies distributed search semantics. Functions scanning `Input4MIPs` (e.g., `search_and_download_esgf_raw_single_var`) instantiate `SearchClient(distrib=False)` to sidestep `pyesgf` indexing anomalies across distributed networks. +- In contrast, `search_and_download_esgf_model_single_var` allows the initialization to default to `distrib=True`. +- **Evidence from Logs**: This setting generates standard `esgf-pyclient` stderr warnings within the subprocess (`Warning - defaulting to search with facets=*`). The library intentionally surfaces this to notify developers that distributed searches lacking explicit facet requests may not return homogenous results across federated architectures. + +## 4. Download Execution + +When the `pyesgf` search yields results (in the form of instances of `pyesgf.search.results.ResultSet`), the pipeline delegates the actual transfer of large geospatial data arrays directly to ESGF's native mechanisms: +- The script iterates through outputs and triggers `result.file_context().get_download_script()`. This invokes `pyesgf` wrappers that securely request a pre-authenticated `wget`/Bash script generated by the remote THREDDS host. +- The `utils.py` engine then invokes a local worker pool using Python's `subprocess.run(["bash", "-c", wget_script_content, "download", "-s"], ...)`. +- **Evidence from Logs**: The execution logs highlight conventional ESGF Bash outputs (`Running download version: 1.3.2`, `Script created for N file(s)`) interspersed with direct asynchronous `https://esgf.ceda.ac.uk/thredds/fileServer/...` retrievals to local storage. It explicitly demonstrates that Python does not mediate `.nc` file payloads. diff --git a/docs/agents/planning/esgpull_implementation/esgpull_analysis.md b/docs/agents/planning/esgpull_implementation/esgpull_analysis.md new file mode 100644 index 0000000..f78b11e --- /dev/null +++ b/docs/agents/planning/esgpull_implementation/esgpull_analysis.md @@ -0,0 +1,46 @@ +# Analysis of `esgpull` Usage and Search Mechanics + +This document analyzes the `esgpull` library as a modern replacement for `esgf-pyclient`, detailing its approach to asynchronous ESGF data management, search, and downloads. + +## 1. Paradigm Shift: Stateful Database vs. Stateless Execution +`esgf-pyclient` in the current `climateset` pipeline relies on a stateless session (`SearchSession`), tracking constraint history dynamically in memory and executing immediate HTTP transfers upon finding matching facets. + +In contrast, `esgpull` operates on a **stateful, database-backed** paradigm: +1. **Query Definition**: Searches and queries are constructed and tested. +2. **Registration (`add`)**: Queries are explicitly added to a local database (defaulting to SQLite) via an API or CLI command (e.g., `esgpull add project:CMIP6`). +3. **Tracking & Queueing**: Saved queries are monitored (`--track`), adding file references asynchronously to an internal download queue. Multi-step requirements can be chained using UUID queries (`--require`). +4. **Execution (`download`)**: Downloads are triggered separately via asynchronous runners. Instead of synchronous `wget`/bash subprocesses, `esgpull` handles high-concurrency streams natively in Python using `asyncio`. + +## 2. Search Criteria & Constraints Management + +Where `climateset/download/utils.py` currently builds strict hierarchies of Pydantic models constraints (e.g., `CMIP6Constraints` and `Input4MIPsConstraints`) and iteratively loops to discover available constraints via `.get_available_facets()`, `esgpull` introduces a significantly more flexible, native query syntax. + +### Faceted Search +`esgpull` standardizes ESGF facet queries (`name:value`) with robust syntax matching what `SearchConstraints` currently provides: +- **Direct Matching**: `project:CMIP6 variable_id:tas` perfectly replicates our base constraints logic. +- **Multiple Values (Logical OR)**: Our codebase occasionally loops through variants (e.g., `r1i1p1f1`). In `esgpull`, multiple matching criteria can simply be passed by comma: `variable_id:c2h2,c2h6`. +- **Exclusion (Negative Facets)**: Constraints can now actively exclude criteria by prepending a bang `!` (e.g., `!institution_id:IPSL`). Our previous `pyclient` implementation struggled natively with this. + +### Free-Text and Solr Syntax +`esgf-pyclient` heavily restricts discovery to standard facet metadata. `esgpull` passes through Apache Solr text search natively: +- Text not conforming to `key:value` syntax is passed to Solr (`esgpull search "surface AND temperature"`), allowing wildcard and semantic metadata discovery which previously required heavy preprocessing in our pipeline. + +### Wildcard Support +Instead of having to iterate over `session.get_available_facets("variant_label")` and then conditionally matching subsets (like `utils.py: _get_variants_and_filter` does), `esgpull` handles wildcarding internally via asterisks. +- **Example**: Searching for `member_id:r1i*p1f1` handles the traversal intuitively without requiring multi-session iteration over every node. + +## 3. Asynchronous Downloads and Data Retrieval +In the current project logs, `pyesgf` handles downloads by: +1. Contacting a specific THREDDS node. +2. Generating a massive bash string (`wget_script_content = file_context.get_download_script()`). +3. Calling `subprocess.run()`. + +`esgpull` replaces this legacy implementation entirely. It features a custom asynchronous download implementation extending traditional Python fetching via structured coroutines. +- It is far more robust against connection drops and 422 Client Errors, bypassing the need for our manual `_rotate_node()` failover block, thanks to its internal HTTP management architecture. +- Authentication paths are standardized, reducing the complexity of the current parsing routines inside `climateset/download/utils.py`. + +## 4. Conclusion for Refactoring Options +Migrating standard constraints (`project`, `institution_id`, `variable_id`, `grid_label`) from our Pydantic classes to `esgpull` queries will be straightforward. +However, the codebase will need deep architectural changes: +1. Removing iterative `get_available_facets`-based fallback logic in favor of bulk defining searches. +2. Abstracting our "download directly" pipeline into a two-step `add` -> `download` asynchronous tracker model. diff --git a/docs/agents/planning/esgpull_implementation/esgpull_mapping.md b/docs/agents/planning/esgpull_implementation/esgpull_mapping.md new file mode 100644 index 0000000..1eb342c --- /dev/null +++ b/docs/agents/planning/esgpull_implementation/esgpull_mapping.md @@ -0,0 +1,129 @@ +# Comprehensive Mapping: `esgf-pyclient` vs. `esgpull` + +This document serves as a direct technical translation guide for refactoring `climateset/download/utils.py` and `climateset/download/client.py` to use `esgpull`. + +## 1. High-Level Architecture Mapping + +| Concept | `esgf-pyclient` (Current) | `esgpull` (New) | +| :--------------------- | :---------------------------------------- | :------------------------------------------------------------------------------------------------------ | +| **Client Instance** | `pyesgf.search.SearchConnection(url=...)` | `esgpull.Esgpull(path=data_dir)` | +| **Session / State** | Custom `SearchClient` + `SearchSession` | `esg.context` (`esgpull.context.Context`) | +| **Nodes & Failover** | Manually looping `NODE_LINK_URLS` | Handled natively by configuring `esg.config.api.index_node` or passing `index_node=` to search methods. | +| **Distributed Search** | `SearchConnection(distrib=True)` | `esg.context.search(..., query(options=Options(distrib=True)))` | +| **Constraints/Facets** | `ctx.constrain(project="CMIP6", ...)` | `query = Query(selection=dict(project=["CMIP6"], ...))` | + +## 2. Code-Level Translation Guide + +### A. Initialization +**Current `esgf-pyclient`:** +```python +from climateset.download.client import SearchClient + +with SearchClient(distrib=False) as client: + session = client.new_session() + # ... +``` + +**New `esgpull` Equivalent:** +```python +from esgpull import Esgpull + +# Esgpull requires a path to initialize its SQLite DB and configuration +esg = Esgpull(path=data_dir) +``` + +### B. Applying Constraints (Search Definition) +**Current `esgf-pyclient`:** +```python +from climateset.download.constraints import Input4MIPsConstraints + +initial_constraints = Input4MIPsConstraints( + project="input4MIPs", + institution_id="VUA", + variable="CO2" +) +session.constrain(initial_constraints) +``` + +**New `esgpull` Equivalent:** +```python +from esgpull.models import Query + +# Constraints are passed as lists to allow multiple arguments natively +query = Query( + selection=dict( + project=["input4MIPs"], + institution_id=["VUA"], + variable=["CO2"] # Maps to 'variable_id' in newer schemas, verify ESGF node schema mapping + ) +) +``` + +### C. Discovering Facets dynamically +**Current `esgf-pyclient`:** +```python +# To check what grid labels exist before applying one +grid_labels = session.get_available_facets("grid_label") +if default_grid_label in grid_labels: + session.constrain(grid_label=default_grid_label) +``` + +**New `esgpull` Equivalent:** +```python +# Use the context's hints method to fetch available facets +hints = esg.context.hints(query, file=False, facets=["grid_label"]) + +# hints usually returns a list of dictionaries mapping node -> facets -> counts +if hints and "grid_label" in hints[0]: + available_grids = list(hints[0]["grid_label"].keys()) + # Apply logic... +``` +*(Note: Because `esgpull` supports wildcards and native multi-value arguments like `grid_label=["gn", "gr"]`, dynamically checking facets before querying is often no longer strictly necessary, greatly simplifying `utils.py`)* + +### D. Executing the Search +**Current `esgf-pyclient`:** +```python +# Executes against the server and returns a ResultSet of THREDDS pointers +results = session.search() +``` + +**New `esgpull` Equivalent:** +```python +# Executes the Solr query and returns list of `File` objects directly mapped to SQLite +files = esg.context.search(query, file=True) + +print(f"Found {len(files)} files.") +``` + +### E. Executing the Download +**Current `esgf-pyclient`:** +```python +# Iterates over ResultSet, extracts wget bash script, and runs subprocess +for result in search_results: + wget_script_content = result.file_context().get_download_script() + subprocess.run(["bash", "-c", wget_script_content, "download", "-s"], cwd=temp_download_path) +``` + +**New `esgpull` Equivalent:** +```python +import asyncio + +# esgpull completely replaces THREDDS bash scripts with native asynchronous fetching. +# Note: files must be added to the internal queue/database before downloading. +esg.db.add(*files) + +# Run the async download concurrently +async def run_downloads(): + # Will download tracked files concurrently into `esg.path / data` + downloaded, errors = await esg.download(files, show_progress=False) + return downloaded + +# Since utils.py is synchronous right now, we must wrap it: +downloaded_files = asyncio.run(run_downloads()) +``` + +## 3. Notable Edge Cases + +1. `esgpull` strictly requires an installation path where it generates a local SQLite directory (`.esgpull/`). The path in `climateset/download/utils.py` operations (`data_dir`) must be initialized properly so `esg.db` does not throw an error. +2. The current implementation creates folders natively like `RAW_DATA / "{project}/{model_id}/{variable}"`. `esgpull` naturally handles standard DRS (Data Reference Syntax) directory generation inside its configuration, but if we need custom `temp_download_path` mapping, we will need to intercept the downloaded files and physically execute `shutil.move()` or configure `esgpull`'s internal data tree path structures. +3. Because `esgpull` `search()` returns `File` classes rather than PyESGF `ResultSet` objects, existing unit tests evaluating `climateset` modules will require new mocks. diff --git a/docs/agents/planning/esgpull_implementation/tasks/01_environment_setup_and_interfaces.md b/docs/agents/planning/esgpull_implementation/tasks/01_environment_setup_and_interfaces.md new file mode 100644 index 0000000..ab2080d --- /dev/null +++ b/docs/agents/planning/esgpull_implementation/tasks/01_environment_setup_and_interfaces.md @@ -0,0 +1,23 @@ +# Task 1: Environment Setup & Interfaces Blueprint + +**Status:** Completed +**Sequence:** 1 + +## Goal +Update project dependencies to include `esgpull` and prepare the existing constraint definitions for compatibility with `esgpull`'s querying system, without breaking the existing `esgf-pyclient` setup. This is part of the overarching goal to implement a new asynchronous, stateful `esgpull` downloader client alongside the existing implementation. + +## Sub-tasks +- [x] Update `pyproject.toml` or `environment.yml`: Add `esgpull` to the project's dependencies while retaining `esgf-pyclient`. +- [x] Review and map out `climateset/download/constraints.py`. +- [x] Ensure serialization outputs from constraints can be seamlessly adapted into dictionaries compatible with `esgpull.models.Query(selection=...)`. Handle the transition from strict Pydantic models to `esgpull`'s native multi-value lists (e.g., `project=["CMIP6"]`). +- [x] Verify that these modifications do not break the existing constraint serialization logic used by `esgf-pyclient`. + +## Constraints & Assumptions +- Existing code and configurations must be preserved for backward compatibility. +- Transition from iterative constraint building to bulk defining searches where possible, leveraging `esgpull`'s native multi-value, exclusion (`!`), and wildcard (`*`) capabilities. + +## Acceptance Criteria +- **AC2:** Existing download clients and their related helper scripts/wrappers are preserved without breaking changes. + +## Notes +- *Delegation:* python, systemdesign diff --git a/docs/agents/planning/esgpull_implementation/tasks/02_isolated_context_engine.md b/docs/agents/planning/esgpull_implementation/tasks/02_isolated_context_engine.md new file mode 100644 index 0000000..5b73c79 --- /dev/null +++ b/docs/agents/planning/esgpull_implementation/tasks/02_isolated_context_engine.md @@ -0,0 +1,24 @@ +# Task 2: Isolated Context Engine Foundation + +**Status:** Completed +**Sequence:** 2 + +## Goal +Build a safe, isolated execution environment for `esgpull` to avoid file lock SQLite collisions during parallel SLURM batch jobs and prevent pollution of the user's `$HOME` directory. + +## Sub-tasks +- [x] Construct a context manager/helper named `isolated_esgpull_context(raw_data_path)` inside `climateset/download/utils.py` (or a newly created `esgpull_utils.py`). +- [x] Implement path logic to create a uniquely hashed path mapping to `RAW_DATA/.esgpull_jobs/`. +- [x] Ensure that `esgpull` initializes in this unique path (`esg = Esgpull(path=hash_path)`), which will generate its local SQLite directory and configuration, bypassing `~/.esgpull`. +- [x] Implement a strict `try/finally` block that wraps `shutil.rmtree` to tear down and safely purge the isolation folder and its SQLite DB, regardless of whether the download succeeds or fails. + +## Constraints & Assumptions +- **Isolated Contexts**: `esgpull` must not initialize in `$HOME`. It must initialize in a unique, isolated path natively inside `RAW_DATA` (e.g. `RAW_DATA/.esgpull_jobs/`) to avoid file lock SQLite collisions between parallel SLURM batch jobs. +- `esgpull` strictly requires an installation path where it generates a local SQLite directory. The context must manage this lifecycle safely. + +## Acceptance Criteria +- **AC4:** The `esgpull` execution context initializes in an isolated cluster hash folder within `RAW_DATA`, completely bypassing `~/.esgpull`. + +## Notes +- *Delegation:* systemdesign +- Requires Task 1 to be completed. diff --git a/docs/agents/planning/esgpull_implementation/tasks/03_search_contract.md b/docs/agents/planning/esgpull_implementation/tasks/03_search_contract.md new file mode 100644 index 0000000..cecfb70 --- /dev/null +++ b/docs/agents/planning/esgpull_implementation/tasks/03_search_contract.md @@ -0,0 +1,28 @@ +# Task 3: Search Contract Implementation + +**Status:** Completed +**Sequence:** 3 + +## Goal +Implement the new `esgpull` downloader class to handle search querying against ESGF nodes, producing `File` arrays. This replaces the stateless, iterative facet exploration of `esgf-pyclient` with `esgpull`'s bulk, stateful query system. + +## Sub-tasks +- [x] Create a new downloader module (e.g., `climateset/download/esgpull_downloader.py`). +- [x] Implement the `EsgpullDownloader` class, ensuring it adheres to the existing downloader interface (e.g., inheriting from `AbstractDownloader`). +- [x] Implement `search_and_download_*` equivalents that instantiate `esg = Esgpull(path=hash_path)`. +- [x] Replace the iterative `get_available_facets`-based fallback logic used by `esgf-pyclient` with bulk queries, utilizing `esgpull`'s wildcard and multi-value support where appropriate. If dynamic lookup is still required, use `hints = esg.context.hints(query)`. +- [x] Implement distributed search handling (`distrib=True/False`) natively through `esgpull` options (`query(options=Options(distrib=True))`). +- [x] Trigger search queries using the new client that return native tracking files: `files = esg.context.search(query, file=True)`. +- [x] Ensure complex combinations of multi-value constraints and wildcards route through `esgpull.models.Query` successfully. + +## Constraints & Assumptions +- Constraints must seamlessly translate to `esgpull.models.Query` ensuring real ESGF nodes are queried correctly. +- `esgpull` handles failover and HTTP management internally, bypassing the manual `_rotate_node()` logic used by `esgf-pyclient`. + +## Acceptance Criteria +- **AC1:** A new `EsgpullDownloader` (or similar) is added as an alternative search and download engine, co-existing with the current `esgf-pyclient` based downloader. +- **AC3:** Complex combinations of multi-value constraints and wildcards route through `esgpull.models.Query` successfully in the new client. + +## Notes +- *Delegation:* python +- Requires Task 2 to be completed. diff --git a/docs/agents/planning/esgpull_implementation/tasks/04_async_execution_and_integration.md b/docs/agents/planning/esgpull_implementation/tasks/04_async_execution_and_integration.md new file mode 100644 index 0000000..3cc1112 --- /dev/null +++ b/docs/agents/planning/esgpull_implementation/tasks/04_async_execution_and_integration.md @@ -0,0 +1,25 @@ +# Task 4: Asynchronous Execution & Integration + +**Status:** Completed +**Sequence:** 4 + +## Goal +Execute the asynchronous download of files tracked in the isolated `esgpull` database and move the final artifacts into the project's standard Data Reference Syntax schema. This replaces the `subprocess.run` bash scripts with native Python `asyncio`. + +## Sub-tasks +- [x] Ensure the new `esgpull` client does NOT rely on bash-generation scripts (like `_download_result` and `_download_process`). +- [x] Implement the two-step asynchronous tracker model: First, add the tracked search result files to the local database: `esg.db.add(*files)`. +- [x] Wrap the retrieval execution using `asyncio`: `downloaded, errors = asyncio.run(esg.download(files, show_progress=False))`. +- [x] Implement the finalization pipeline: `esgpull` naturally handles standard DRS. We must physically execute `shutil.move()` to safely transfer `.nc` chunk artifacts from the isolated DB cache DRS tree to the final target `RAW_DATA` path matching the project's specific directory schema. +- [x] Ensure extraction is fully native Python (`asyncio`) without invoking `subprocess.run(["bash", ...])`. + +## Constraints & Assumptions +- **Storage Cleanup**: `esgpull` downloads files natively to its internal cache. We must `shutil.move()` them to our strict local directory schema and cleanly delete the `.esgpull_jobs/` context immediately afterward to avoid cluttering disk space. + +## Acceptance Criteria +- **AC5:** A finalization block safely transfers `.nc` files from the isolation folder to the target Data Reference Syntax and purges the isolation folder afterward. +- **AC6:** End-to-end extraction in the new client is native `asyncio.run(esg.download())` without invoking `subprocess.run(["bash", ...])`. + +## Notes +- *Delegation:* python +- Requires Task 3 to be completed. diff --git a/docs/agents/planning/esgpull_implementation/tasks/05_verification_and_testing.md b/docs/agents/planning/esgpull_implementation/tasks/05_verification_and_testing.md new file mode 100644 index 0000000..d07584b --- /dev/null +++ b/docs/agents/planning/esgpull_implementation/tasks/05_verification_and_testing.md @@ -0,0 +1,26 @@ +# Task 5: Verification & Testing + +**Status:** Completed +**Sequence:** 5 + +## Goal +Verify the robustness, correctness, and isolation of the new `esgpull` client while ensuring backward compatibility with the existing `esgf-pyclient` downloader. + +## Sub-tasks +- [x] Add new automated tests specifically for the `esgpull` client. +- [x] Implement new mocks for unit tests, accounting for the fact that `esgpull` `search()` returns `File` classes rather than `PyESGF` `ResultSet` objects. +- [x] Ensure existing tests for the `esgf-pyclient` downloader continue to pass unmodified. +- [x] **Critical:** Ensure the *search querying phase* is NEVER mocked, so queries touch real node indices and actual `File` arrays are returned. +- [x] Implement `unittest.mock` intercept **only** for the `Esgpull.download` asynchronous function to prevent massive data bandwidth usage in CI. +- [x] Perform manual end-to-end verification of storage independence: run a script with the new client and verify via terminal `ls` that `RAW_DATA/.esgpull_jobs/` spawns correctly and cleans up cleanly. Verify `~/.esgpull` is untouched. +- [x] Perform manual verification of subprocess elimination: inspect output logs to ensure Python asynchronously tracks chunks rather than arbitrary `wget` process logs. Confirm `.nc` files are correctly structured in the output directory. +- [x] Perform manual regression testing using the existing `esgf-pyclient` implementation. + +## Constraints & Assumptions +- **Deterministic Search Testing**: Standard unit testing uses heavy mocking. **CRITICAL:** the *search querying phase* mechanism of this application must NEVER be mocked to ensure constraints map accurately against real ESGF nodes. + +## Acceptance Criteria +- All previous Acceptance Criteria (AC1-AC6) are proven correct by fulfilling this verification plan. + +## Notes +- Some tests can be written in parallel with Tasks 3 and 4. Manual verification requires Tasks 1-4 to be fully implemented. diff --git a/docs/agents/planning/refactor-download-client-abstraction/plan-refactor-download-compose.md b/docs/agents/planning/refactor-download-client-abstraction/plan-refactor-download-compose.md new file mode 100644 index 0000000..ab4d2e9 --- /dev/null +++ b/docs/agents/planning/refactor-download-client-abstraction/plan-refactor-download-compose.md @@ -0,0 +1,28 @@ +# 🎯 Scope & Context +The project currently has duplicated classes for downloading data (`Input4MipsDownloader` vs `Input4MipsDownloaderV2` and `CMIP6Downloader` vs `CMIP6DownloaderV2`), where the base versions use `pyesgf` and the `V2` versions use `esgpull`. We need to harmonize these into single `Input4MipsDownloader` and `CMIP6Downloader` classes. The user should instead be able to choose the underlying client implementation (`pyesgf` vs `esgpull`) via configuration, consolidating the shared business logic (e.g., variable iteration, metadata downloading) into a single location. + +# 🏛️ Architectural Approach +We will utilize the **Strategy Pattern** and **Dependency Injection** principles. Instead of duplicating the iterator loop logic via inheritance, the core downloaders will switch between backend functions based on a new configuration field. +- A `ClientType` enumerator (`PYESGF` / `ESGPULL`) will be added to `BaseDownloaderConfig` to formalize the choice. +- **Backend Lifecycle Management:** For `ESGPULL`, the downloader's `download()` method will manage a single `isolated_esgpull_context`. This shared `Esgpull` instance will be injected into all search-and-download calls within the batch to ensure isolation from `$HOME` and performance efficiency (lifting the isolation boundary from per-variable to per-batch). +- The downloader classes will act as orchestrators, reading the configuration and routing the payload parameters to the respective utility functions. + +# 🧪 Verification & FMEA +- **Verification Strategy:** Run the existing unit test suites for `test_downloader.py` using `make test`. We will parametrize the tests to run against both `ClientType.PYESGF` and `ClientType.ESGPULL`. +- **FMEA (Failure Mode and Effects Analysis):** + - **Failure:** User passes an invalid client string in their YAML config. + **Mitigation:** `BaseDownloaderConfig` uses Pydantic or strict Enum validation to fail fast. + - **Failure:** `esg.context` carries stale facet state between variables in `ESGPULL` mode. + **Mitigation:** Verify search results in TASK-05 to ensure fresh `Query` objects are used per call. + - **Failure:** Inconsistent function signatures between `utils.py` and `esgpull_utils.py`. + **Mitigation:** Standardize the orchestrator-to-utility interface to always accept an optional `esg: Esgpull = None` parameter. + +# 🪜 Implementation Steps +1. **Add Client Configuration:** Introduce a `ClientType` Enum and add a `client_type: ClientType = ClientType.PYESGF` property to `BaseDownloaderConfig` in `climateset/download/downloader_config.py`. Update YAML parsing to handle this field. +2. **Harmonize `CMIP6Downloader`:** Update `CMIP6Downloader.download_from_model_single_var` to conditionally call `search_and_download_esgf_model_single_var` (for pyesgf) or `esgpull_search_and_download_esgf_model_single_var` (for esgpull) based on `self.config.client_type`. Delete `CMIP6DownloaderV2`. +3. **Harmonize `Input4MipsDownloader`:** Update `Input4MipsDownloader` functions to conditionally dispatch to the `pyesgf` or `esgpull` utility equivalents. Delete `Input4MipsDownloaderV2`. +4. **Refactor CLI / Entrypoints:** Update `cli.py` and any `scripts/download_example.py` to utilize the unified classes, passing the chosen client type via configuration. +5. **Update Tests:** Refactor the test suite in `tests/test_download/test_downloader.py` to instantiate the base downloaders with the appropriate `ClientType` config rather than importing `*V2` classes. + +# 🤝 Next Step +Are you ready to approve Step 1 of the implementation to add the `ClientType` configuration? \ No newline at end of file diff --git a/docs/agents/planning/refactor-download-client-abstraction/tasks/01_client_configuration.md b/docs/agents/planning/refactor-download-client-abstraction/tasks/01_client_configuration.md new file mode 100644 index 0000000..7f8bb8c --- /dev/null +++ b/docs/agents/planning/refactor-download-client-abstraction/tasks/01_client_configuration.md @@ -0,0 +1,36 @@ +# TASK-01: Client Configuration + +## Goal +Introduce a `ClientType` Enum to configure which ESGF client (`pyesgf` or `esgpull`) the downlaoder should use, and add it to `BaseDownloaderConfig`. + +## Context & References +- **Source Plan**: docs/agents/planning/refactor-download-compose.md +- **Relevant Specs**: N/A +- **Existing Code**: + - `climateset/download/downloader_config.py` + +## Subtasks +1. [ ] Define a `ClientType` Enum (e.g. `PYESGF` and `ESGPULL`). +2. [ ] Add `client_type: ClientType = ClientType.PYESGF` property to `BaseDownloaderConfig`'s `__init__`. +3. [ ] Update YAML configuration parsing or ensure that standard parsing can handle mapping strings like "pyesgf" or "esgpull" to the `ClientType` Enum if provided in the kwargs. + +## Requirements & Constraints +- Must default to `ClientType.PYESGF` to avoid breaking changes in behavior if `client_type` is missing from the config files. +- Configuration must fail fast if an invalid client string is provided in the YAML. + +## Acceptance Criteria (AC) +- [ ] AC 1: `BaseDownloaderConfig` successfully initializes with either `PYESGF` or `ESGPULL` string/enum. +- [ ] AC 2: ValueError or similar validation error is raised when an unsupported client type string is loaded from config. + +## Testing & Validation +- **Command**: `make test-custom TEST_ARGS="tests/test_download/test_downloader.py"` +- **Success State**: Configuration parsing tests pass or downloader tests load default successfully. +- **Manual Verification**: Run a quick script to load `BaseDownloaderConfig` with a test yaml setting `client_type` explicitly. + +## Completion Protocol +1. [ ] All ACs are met. +2. [ ] Tests pass without regressions. +3. [ ] Code is linted via `make precommit` and `make pylint`. +4. [ ] Documentation updated (if applicable). +5. [ ] Commit work: `git commit -m "feat(download): task 01 - add ClientType configuration"` +6. [ ] Update this document: Mark as COMPLETE. \ No newline at end of file diff --git a/docs/agents/planning/refactor-download-client-abstraction/tasks/02_harmonize_cmip6_downloader.md b/docs/agents/planning/refactor-download-client-abstraction/tasks/02_harmonize_cmip6_downloader.md new file mode 100644 index 0000000..9d5c8ce --- /dev/null +++ b/docs/agents/planning/refactor-download-client-abstraction/tasks/02_harmonize_cmip6_downloader.md @@ -0,0 +1,37 @@ +# TASK-02: Harmonize CMIP6Downloader + +## Goal +Harmonize the CMIP6 downloader so that it dispatches the actual download step to the correct utility backend based on the `client_type` configuration, effectively removing the need for `CMIP6DownloaderV2`. + +## Context & References +- **Source Plan**: docs/agents/planning/refactor-download-compose.md +- **Relevant Specs**: N/A +- **Existing Code**: + - `climateset/download/cmip6_downloader.py` + +## Subtasks +1. [ ] Update `CMIP6Downloader.download()` to manage the `isolated_esgpull_context` if `client_type == ClientType.ESGPULL`. +2. [ ] Refactor the inner loops of `CMIP6Downloader.download()` into a helper (or thread `esg` through) so that `download_from_model_single_var` accepts an optional `esg: Esgpull = None`. +3. [ ] If `ESGPULL` mode, call `esgpull_search_and_download_esgf_model_single_var` passing the injected `esg`. +4. [ ] Delete `CMIP6DownloaderV2` class from `cmip6_downloader.py`. + +## Requirements & Constraints +- Both utility functions (`search_and_download_esgf_model_single_var` and its esgpull counterpart) must have consistent argument signatures as called from the class. + +## Acceptance Criteria (AC) +- [ ] AC 1: `CMIP6Downloader` acts as a unified facade for both clients. +- [ ] AC 2: `CMIP6DownloaderV2` no longer exists in the codebase. +- [ ] AC 3: `CMIP6Downloader` successfully passes config parameters to both backends without TypeErrors. + +## Testing & Validation +- **Command**: `make test-custom TEST_ARGS="tests/test_download/test_downloader.py"` (specifically tests targeting `CMIP6Downloader`) +- **Success State**: The harmonized class works for both client types. +- **Manual Verification**: Check if `make precommit` and `make pylint` pass after deleting `CMIP6DownloaderV2`. + +## Completion Protocol +1. [ ] All ACs are met. +2. [ ] Tests pass without regressions. +3. [ ] Code is linted via `make precommit` and `make pylint`. +4. [ ] Documentation updated (if applicable). +5. [ ] Commit work: `git commit -m "refactor(download): task 02 - harmonize CMIP6Downloader and remove V2"` +6. [ ] Update this document: Mark as COMPLETE. \ No newline at end of file diff --git a/docs/agents/planning/refactor-download-client-abstraction/tasks/03_harmonize_input4mips_downloader.md b/docs/agents/planning/refactor-download-client-abstraction/tasks/03_harmonize_input4mips_downloader.md new file mode 100644 index 0000000..d4b0ada --- /dev/null +++ b/docs/agents/planning/refactor-download-client-abstraction/tasks/03_harmonize_input4mips_downloader.md @@ -0,0 +1,38 @@ +# TASK-03: Harmonize Input4MipsDownloader + +## Goal +Harmonize the Input4Mips downloader so that it dispatches the actual download steps to the correct utility backends based on the `client_type` configuration, effectively removing the need for `Input4MipsDownloaderV2`. + +## Context & References +- **Source Plan**: docs/agents/planning/refactor-download-compose.md +- **Relevant Specs**: N/A +- **Existing Code**: + - `climateset/download/input4mips_downloader.py` + +## Subtasks +1. [ ] Update `Input4MipsDownloader.download()` to conditionally manage the `isolated_esgpull_context` when `ClientType.ESGPULL` is selected. +2. [ ] Modify `Input4MipsDownloader.download_raw_input_single_var` to accept an optional `esg: Esgpull = None`. +3. [ ] Modify `Input4MipsDownloader.download_meta_historic_biomassburning_single_var` to accept an optional `esg: Esgpull = None`. +4. [ ] In `ESGPULL` mode, forward `esg` to the corresponding `esgpull_utils` search-and-download functions. +5. [ ] Delete `Input4MipsDownloaderV2` class from `input4mips_downloader.py`. + +## Requirements & Constraints +- Both utility functions for both standard inputs and biomass must accept identical parameters from the caller. + +## Acceptance Criteria (AC) +- [ ] AC 1: `Input4MipsDownloader` acts as a unified facade for both clients. +- [ ] AC 2: `Input4MipsDownloaderV2` no longer exists in the codebase. +- [ ] AC 3: Dispatches to standard and biomass endpoints work for both client types. + +## Testing & Validation +- **Command**: `make test-custom TEST_ARGS="tests/test_download/test_downloader.py"` (specifically tests targeting `Input4MipsDownloader`) +- **Success State**: The harmonized class works for both client types. +- **Manual Verification**: Verify `download_raw_input_single_var` branches correctly using `make test`. + +## Completion Protocol +1. [ ] All ACs are met. +2. [ ] Tests pass without regressions. +3. [ ] Code is linted via `make precommit` and `make pylint`. +4. [ ] Documentation updated (if applicable). +5. [ ] Commit work: `git commit -m "refactor(download): task 03 - harmonize Input4MipsDownloader and remove V2"` +6. [ ] Update this document: Mark as COMPLETE. \ No newline at end of file diff --git a/docs/agents/planning/refactor-download-client-abstraction/tasks/04_refactor_cli_entrypoints.md b/docs/agents/planning/refactor-download-client-abstraction/tasks/04_refactor_cli_entrypoints.md new file mode 100644 index 0000000..b73b8c7 --- /dev/null +++ b/docs/agents/planning/refactor-download-client-abstraction/tasks/04_refactor_cli_entrypoints.md @@ -0,0 +1,37 @@ +# TASK-04: Refactor CLI / Entrypoints + +## Goal +Update the CLI and example scripts to use the unified classes (`Input4MipsDownloader` and `CMIP6Downloader`) and remove references to the `*V2` classes. + +## Context & References +- **Source Plan**: docs/agents/planning/refactor-download-compose.md +- **Relevant Specs**: N/A +- **Existing Code**: + - `climateset/cli.py` + - `scripts/download_example.py` + - Any config yaml references if applicable. + +## Subtasks +1. [ ] Replace imports of `CMIP6DownloaderV2` with `CMIP6Downloader` in `climateset/cli.py` and `scripts/download_example.py`. +2. [ ] Replace imports of `Input4MipsDownloaderV2` with `Input4MipsDownloader` in `climateset/cli.py` and `scripts/download_example.py`. +3. [ ] Update initialization logic in these entrypoints if `client_type` needs to be explicitly passed or parsed from CLI arguments (if desired), or rely on config parsing doing it. + +## Requirements & Constraints +- The user-facing CLI behavior should stay identical, but internally it routes through the unified class rather than picking between V1/V2 classes at the CLI layer. + +## Acceptance Criteria (AC) +- [ ] AC 1: No references to `V2` downloaders exist in the entrypoint scripts. +- [ ] AC 2: CLI scripts run successfully using the new consolidated class. + +## Testing & Validation +- **Command**: `python scripts/download_example.py --help` (or equivalent test execution) +- **Success State**: Script parses without import errors. +- **Manual Verification**: Run `grep -r "DownloaderV2"` to ensure all usages are purged. + +## Completion Protocol +1. [ ] All ACs are met. +2. [ ] Tests pass without regressions. +3. [ ] Code is linted via `make precommit` and `make pylint`. +4. [ ] Documentation updated (if applicable). +5. [ ] Commit work: `git commit -m "refactor(cli): task 04 - replace V2 downloader references in CLI"` +6. [ ] Update this document: Mark as COMPLETE. \ No newline at end of file diff --git a/docs/agents/planning/refactor-download-client-abstraction/tasks/05_update_tests.md b/docs/agents/planning/refactor-download-client-abstraction/tasks/05_update_tests.md new file mode 100644 index 0000000..0840ced --- /dev/null +++ b/docs/agents/planning/refactor-download-client-abstraction/tasks/05_update_tests.md @@ -0,0 +1,38 @@ +# TASK-05: Update Tests + +## Goal +Refactor the test suite in `tests/test_download/test_downloader.py` (and any related test files) to instantiate the base downloaders with the appropriate `ClientType` config rather than importing and testing `*V2` classes separately. + +## Context & References +- **Source Plan**: docs/agents/planning/refactor-download-compose.md +- **Relevant Specs**: N/A +- **Existing Code**: + - `tests/test_download/test_downloader.py` + - Any mocks inside `tests/test_download/` + +## Subtasks +1. [ ] Find tests covering `CMIP6DownloaderV2` and `Input4MipsDownloaderV2`. +2. [ ] Change imports to the base classes. +3. [ ] Modify the fixture or config initialization to explicitly set `client_type=ClientType.ESGPULL` for testing the `esgpull` branches. +4. [ ] Use `pytest.mark.parametrize` to run shared tests for both `PYESGF` and `ESGPULL` where appropriate to ensure identical behavior. + +## Requirements & Constraints +- Coverage for both `pyesgf` and `esgpull` code paths must be maintained. +- Tests should not hang or fail due to improper mocking of the selected backend. + +## Acceptance Criteria (AC) +- [ ] AC 1: All V2 tests are ported to use the base classes with config injection. +- [ ] AC 2: `make test` passes with 100% success rate across both `pyesgf` and `esgpull` configurations. + +## Testing & Validation +- **Command**: `make test-custom TEST_ARGS="tests/test_download/test_downloader.py -v"` +- **Success State**: All tests pass. +- **Manual Verification**: Verify test logs run both branches of the conditional dispatch. + +## Completion Protocol +1. [ ] All ACs are met. +2. [ ] Tests pass without regressions. +3. [ ] Code is linted via `make precommit` and `make pylint`. +4. [ ] Documentation updated (if applicable). +5. [ ] Commit work: `git commit -m "test(download): task 05 - update test suite for harmonized downloaders"` +6. [ ] Update this document: Mark as COMPLETE. \ No newline at end of file diff --git a/docs/agents/planning/refactor-esgpull-isolation-to-batch-scoped/plan-refactor-esgpull-isolation.md b/docs/agents/planning/refactor-esgpull-isolation-to-batch-scoped/plan-refactor-esgpull-isolation.md new file mode 100644 index 0000000..40b5e1d --- /dev/null +++ b/docs/agents/planning/refactor-esgpull-isolation-to-batch-scoped/plan-refactor-esgpull-isolation.md @@ -0,0 +1,57 @@ +# Formal Design Document: Refactor Per-Variable Esgpull Isolation to Batch-Scoped Contexts + +## 1. 🎯 Scope & Context + +`isolated_esgpull_context()` in `esgpull_utils.py` currently creates a new UUID directory, SQLite database, and full `Esgpull(install=True)` instance for **every single variable download call**. In the V2 downloaders, the triple-nested loop `models x variables x experiments` means a configuration with 5 models, 10 variables, and 4 experiments spawns **200 isolated contexts** — each paying the full cost of filesystem setup, SQLite initialization, and teardown via `shutil.rmtree`. This document proposes lifting the isolation boundary from per-variable to per-batch (i.e., per `download()` invocation), so a single `Esgpull` instance is reused across all iterations within one downloader run. The constraint from `ESGPULL_CLIENT_IMPLEMENTATION.md` — isolation from `$HOME` and between parallel SLURM jobs — must be preserved. + +## 2. 🧠 Architectural Approach (Trade-offs & Strategy) + +- **Chosen pattern: Dependency Injection of a shared context.** Instead of each `esgpull_search_and_download_*` function creating its own context internally, the V2 downloader's `download()` method opens **one** `isolated_esgpull_context()` and passes the `Esgpull` instance into each search-and-download call. This applies the **Dependency Inversion Principle (DIP)** — the download functions depend on an abstract `Esgpull` handle, not on how it was constructed. + +- **Why not a module-level singleton?** A singleton would violate the SLURM parallel-job isolation requirement documented in `ESGPULL_CLIENT_IMPLEMENTATION.md` (Constraint 1). Two concurrent SLURM jobs sharing a single SQLite file would deadlock. The batch-scoped context preserves one-context-per-OS-process semantics while eliminating the per-variable churn. + +- **Why not a connection pool / context cache keyed by `data_dir`?** This adds complexity (LRU eviction, thread-safety, lifecycle management) that is not justified. The current execution model is single-threaded and sequential within a downloader. A pool solves a problem that does not yet exist. + +- **Accepted trade-off: SQLite DB grows within a batch.** The `esg.db.add(*files)` calls accumulate rows across the entire batch instead of starting fresh each time. This is acceptable because (a) the DB is still ephemeral and deleted on context exit, (b) file deduplication already happens via `unique_files` in `_download_and_move_files`, and (c) the total row count for a realistic batch (hundreds of files) is trivial for SQLite. + +- **Accepted trade-off: blast radius of a mid-batch failure increases.** If the process crashes at variable 150 of 200, the single context's `finally` block cleans up the isolation directory — including any already-moved files' cache remnants. This is identical to the current behavior per-variable, but now the cleanup is deferred. Already-moved `.nc` files in their final destination are unaffected by the teardown, so no data loss occurs. + +- **Principle: Easier to Change (ETC).** The refactored functions accept an `Esgpull` instance as a parameter, making them testable with a mock or stub `Esgpull` without needing to patch the context manager. This also unblocks future parallelization — a caller could open N contexts and distribute work across them. + +## 3. 🛡️ Verification & Failure Modes (FMEA) + +### Test Strategy + +- **Unit tests (`test_esgpull_utils.py`):** Verify that refactored `esgpull_search_and_download_*` functions accept an `Esgpull` instance parameter and invoke `esg.context.search`, `esg.db.add`, and `esg.download` on it. Mock only the `Esgpull` object — never the context manager boundary. +- **Integration tests (existing real-node tests):** Run the V2 downloaders against a live ESGF node for a minimal config (1 model, 1 variable, 1 experiment) and confirm files land in the correct directory tree. This validates that a shared context does not corrupt search state across iterations. +- **Context lifecycle test:** Assert that exactly **one** UUID directory is created under `.esgpull_jobs/` during a multi-variable `download()` call, and that it is removed after the call completes (both on success and on exception). +- **Regression:** Existing V1 downloader tests must pass unchanged — V1 code is not touched by this refactor. + +### Known Risks + +| Failure Mode | Likelihood | Impact | Mitigation | +|---|---|---|---| +| **SQLite row accumulation causes query slowdown** | Low — realistic batches are < 1 000 files | Minor latency increase | Monitor; if observed, add `esg.db` pruning between iterations | +| **`esg.context` carries stale facet state between variables** | Medium — depends on esgpull internals | Wrong files downloaded | Integration test with 2+ variables asserts correct file counts per variable | +| **`asyncio.run()` called multiple times on same event loop** | Low — each `_download_and_move_files` call uses `asyncio.run()` which creates a fresh loop | RuntimeError if loop is already running | Wrap in `asyncio.new_event_loop()` + `loop.run_until_complete()` if needed; verify in unit test | +| **Interrupted batch leaves orphan `.esgpull_jobs/` dir** | Same as current | Disk space leak | No change — `finally` block + `shutil.rmtree` already handles this. Document manual cleanup in ops runbook | + +## 4. 📋 Granular Implementation Steps + +1. **Add `esg: Esgpull` parameter to search-and-download functions** — Modify `esgpull_search_and_download_esgf_raw_single_var`, `esgpull_search_and_download_esgf_biomass_single_var`, and `esgpull_search_and_download_esgf_model_single_var` in `esgpull_utils.py` to accept an `esg: Esgpull` parameter instead of `data_dir: Path | str`. Remove the internal `with isolated_esgpull_context(...)` block from each function body. + +2. **Lift context creation into V2 downloader `download()` methods** — In `CMIP6DownloaderV2.download()` and `Input4MipsDownloaderV2.download()`, wrap the entire iteration loop in a single `with isolated_esgpull_context(self.config.data_dir) as esg:` block. Pass `esg` through to each `download_from_model_single_var` / `download_raw_input_single_var` / `download_meta_historic_biomassburning_single_var` call. + +3. **Update intermediate methods to thread `esg` through** — Modify `CMIP6DownloaderV2.download_from_model_single_var`, `Input4MipsDownloaderV2.download_raw_input_single_var`, and `Input4MipsDownloaderV2.download_meta_historic_biomassburning_single_var` to accept and forward the `esg` parameter to the corresponding `esgpull_utils` function. + +4. **Re-derive `dest_dir` from `data_dir` inside search-and-download functions** — Since `data_dir` is no longer passed directly, either (a) add `data_dir` as a separate parameter alongside `esg`, or (b) derive it from `esg.config.paths` (the esgpull instance knows its root path's parent). Option (a) is simpler and more explicit — prefer it. + +5. **Verify `esg.context` statefulness** — Write a focused integration test that calls `esgpull_search_and_download_esgf_model_single_var` twice with different `(variable, experiment)` pairs using the **same** `esg` instance, and asserts that each call returns files matching only its own constraints. This guards against facet bleed between iterations. + +6. **Update existing unit tests** — Adjust `test_esgpull_utils.py` and any V2 downloader tests to reflect the new signatures. Add a lifecycle assertion that `isolated_esgpull_context` is entered exactly once per `download()` invocation. + +7. **Run full test suite and linting** — Execute `make test`, `make precommit`, `make pylint`, and `make mypy` to confirm no regressions. + +## 5. ⏭️ Next Step + +> Shall I proceed with Step 1 — adding the `esg: Esgpull` parameter to the three search-and-download functions in `esgpull_utils.py` and removing their internal `isolated_esgpull_context` blocks? diff --git a/docs/agents/planning/refactor-esgpull-isolation-to-batch-scoped/tasks/TASK-001-refactor-esgpull-utils-signatures.md b/docs/agents/planning/refactor-esgpull-isolation-to-batch-scoped/tasks/TASK-001-refactor-esgpull-utils-signatures.md new file mode 100644 index 0000000..7be7272 --- /dev/null +++ b/docs/agents/planning/refactor-esgpull-isolation-to-batch-scoped/tasks/TASK-001-refactor-esgpull-utils-signatures.md @@ -0,0 +1,96 @@ +# TASK-001: Refactor search-and-download functions to accept an injected `Esgpull` instance + +## 1. Goal +Replace the per-call `isolated_esgpull_context` usage inside the three `esgpull_search_and_download_*` functions with an externally provided `esg: Esgpull` parameter, so callers control the context lifecycle. + +## 2. Context & References +- **Plan section:** Steps 1 and 4 from `plan-refactor-esgpull-isolation.md`: + > 1. Add `esg: Esgpull` parameter to search-and-download functions — Modify ... to accept an `esg: Esgpull` parameter instead of `data_dir: Path | str`. Remove the internal `with isolated_esgpull_context(...)` block from each function body. + > 4. Re-derive `dest_dir` from `data_dir` inside search-and-download functions — ... add `data_dir` as a separate parameter alongside `esg` ... Option (a) is simpler and more explicit — prefer it. +- **Upstream tasks:** None — this is the first task. +- **Key files:** + - `climateset/download/esgpull_utils.py` (lines 56–167) — the three functions to modify +- **Relevant skills:** `python` (DIP, strict typing), `systemdesign` (decoupling context lifecycle from business logic) + +### Current signatures (to be changed) +```python +# esgpull_utils.py:56 +def esgpull_search_and_download_esgf_raw_single_var( + variable: str, institution_id: str, project: str, + default_grid_label: str, default_frequency: str, preferred_version: str, + data_dir: Path | str, # <-- drives both isolation AND dest_dir + distrib: bool = False, logger: logging.Logger = LOGGER, +): ... + +# esgpull_utils.py:94 +def esgpull_search_and_download_esgf_biomass_single_var( + variable: str, variable_id: str, institution_id: str, project: str, + default_grid_label: str, default_frequency: str, preferred_version: str, + data_dir: Path | str, + distrib: bool = False, logger: logging.Logger = LOGGER, +): ... + +# esgpull_utils.py:129 +def esgpull_search_and_download_esgf_model_single_var( + model: str, variable: str, experiment: str, project: str, + default_grid_label: str, default_frequency: str, preferred_version: str, + max_ensemble_members: int, ensemble_members: list[str], + data_dir: Path | str, + distrib: bool = False, logger: logging.Logger = LOGGER, +): ... +``` + +## 3. Subtasks +- [x] 1. Replace `data_dir: Path | str` with `esg: Esgpull` and `data_dir: Path` (keeping `data_dir` for `dest_dir` derivation) in `esgpull_search_and_download_esgf_raw_single_var` +- [x] 2. Remove the `with isolated_esgpull_context(data_dir) as esg:` block — dedent the function body one level so it uses the injected `esg` directly +- [x] 3. Repeat subtasks 1–2 for `esgpull_search_and_download_esgf_biomass_single_var` +- [x] 4. Repeat subtasks 1–2 for `esgpull_search_and_download_esgf_model_single_var` +- [x] 5. Verify `isolated_esgpull_context` is no longer called inside any of the three functions (but remains defined in the module for external use) +- [x] 6. Run `python3 -c "import ast; ast.parse(open('climateset/download/esgpull_utils.py').read())"` to confirm syntax validity + +## 4. Requirements & Constraints +- **Technical:** + - `esg` parameter must be typed as `Esgpull` (imported from `esgpull`). + - `data_dir` parameter changes type from `Path | str` to `Path` — the `str` coercion responsibility moves to the caller (the V2 downloader). + - `isolated_esgpull_context()` definition must remain in the module — it is still the public API for creating contexts. Only its **call sites** inside the three functions are removed. +- **Business:** The function bodies must remain functionally identical — only the context acquisition changes. +- **Out of scope:** Modifying V2 downloader classes or tests — those are TASK-002 and TASK-004. + +## 5. Acceptance Criteria +- [x] AC-1: All three `esgpull_search_and_download_*` functions accept `esg: Esgpull` as their first parameter. +- [x] AC-2: All three functions retain `data_dir: Path` as a parameter (used only for `dest_dir` derivation). +- [x] AC-3: No call to `isolated_esgpull_context` exists inside any of the three function bodies. +- [x] AC-4: `isolated_esgpull_context` remains defined and importable from `esgpull_utils`. +- [x] AC-5: `python3 -c "import ast; ast.parse(open('climateset/download/esgpull_utils.py').read())"` exits 0. +- [x] AC-6: `make precommit` and `make pylint` exit 0 for `climateset/download/esgpull_utils.py`. + +## 6. Testing & Validation +```bash +# Syntax check +python3 -c "import ast; ast.parse(open('climateset/download/esgpull_utils.py').read())" +# Expected: exit 0, no output + +# Lint +make precommit +make pylint +# Expected: exit 0 (or only pre-existing warnings unrelated to this change) + +# Verify isolated_esgpull_context is still defined +grep -n "def isolated_esgpull_context" climateset/download/esgpull_utils.py +# Expected: one match (the definition) + +# Verify no internal calls remain +grep -n "isolated_esgpull_context" climateset/download/esgpull_utils.py +# Expected: only the definition line and any import/export — no calls inside the three functions +``` + +## 7. Completion Protocol +1. Verify every AC is checked off in Section 5. +2. Run all commands in Section 6 and confirm expected output. +3. Stage and commit with a scoped message: + ```bash + git add climateset/download/esgpull_utils.py + git commit -m "refactor(esgpull): inject Esgpull instance into search-and-download functions — closes TASK-001" + ``` +4. Update this file: check off completed subtasks and ACs, note any deviations. +5. Notify the user with a concise summary and request approval before proceeding to TASK-002. diff --git a/docs/agents/planning/refactor-esgpull-isolation-to-batch-scoped/tasks/TASK-002-lift-context-into-v2-downloaders.md b/docs/agents/planning/refactor-esgpull-isolation-to-batch-scoped/tasks/TASK-002-lift-context-into-v2-downloaders.md new file mode 100644 index 0000000..242f5a5 --- /dev/null +++ b/docs/agents/planning/refactor-esgpull-isolation-to-batch-scoped/tasks/TASK-002-lift-context-into-v2-downloaders.md @@ -0,0 +1,97 @@ +# TASK-002: Lift `isolated_esgpull_context` into V2 downloader `download()` methods + +## 1. Goal +Move the `isolated_esgpull_context` lifecycle from the utility functions (removed in TASK-001) into the V2 downloader `download()` methods, so each full download run opens exactly one isolated context and threads `esg` through all iterations. + +## 2. Context & References +- **Plan section:** Steps 2 and 3 from `plan-refactor-esgpull-isolation.md`: + > 2. Lift context creation into V2 downloader `download()` methods — wrap the entire iteration loop in a single `with isolated_esgpull_context(self.config.data_dir) as esg:` block. + > 3. Update intermediate methods to thread `esg` through — Modify ... to accept and forward the `esg` parameter. +- **Upstream tasks:** TASK-001 — after which the three `esgpull_search_and_download_*` functions have this signature pattern: + ```python + def esgpull_search_and_download_esgf_model_single_var( + esg: Esgpull, # <-- NEW: injected instance + model: str, + variable: str, + experiment: str, + project: str, + default_grid_label: str, + default_frequency: str, + preferred_version: str, + max_ensemble_members: int, + ensemble_members: list[str], + data_dir: Path, # <-- now Path only, used for dest_dir + distrib: bool = False, + logger: logging.Logger = LOGGER, + ): ... + ``` +- **Key files:** + - `climateset/download/cmip6_downloader.py` (lines 86–156) — `CMIP6DownloaderV2` + - `climateset/download/input4mips_downloader.py` (lines 122–224) — `Input4MipsDownloaderV2` +- **Relevant skills:** `python` (DIP, composition), `systemdesign` (lifecycle management, SRP) + +## 3. Subtasks +- [x] 1. Add `from climateset.download.esgpull_utils import isolated_esgpull_context` to `cmip6_downloader.py` (if not already imported) +- [x] 2. Wrap the triple-nested loop in `CMIP6DownloaderV2.download()` with `with isolated_esgpull_context(self.config.data_dir) as esg:` +- [x] 3. Add `esg: Esgpull` parameter to `CMIP6DownloaderV2.download_from_model_single_var()` and pass it through to `esgpull_search_and_download_esgf_model_single_var` +- [x] 4. Update the call site in `CMIP6DownloaderV2.download()` to pass `esg` to `self.download_from_model_single_var()` +- [x] 5. Add `from climateset.download.esgpull_utils import isolated_esgpull_context` to `input4mips_downloader.py` (if not already imported) +- [x] 6. Wrap all iteration loops in `Input4MipsDownloaderV2.download()` with a single `with isolated_esgpull_context(self.config.data_dir) as esg:` block +- [x] 7. Add `esg: Esgpull` parameter to `Input4MipsDownloaderV2.download_raw_input_single_var()` and `Input4MipsDownloaderV2.download_meta_historic_biomassburning_single_var()` — forward to the corresponding `esgpull_utils` function +- [x] 8. Update all call sites in `Input4MipsDownloaderV2.download()` to pass `esg` +- [x] 9. Verify syntax validity of both modified files + +## 4. Requirements & Constraints +- **Technical:** + - The `with` block must encompass **all** iteration loops in `download()`, not just one sub-loop. For `Input4MipsDownloaderV2`, this includes the raw variables loop, biomass loop, and metafiles loops. + - `self.config.data_dir` may be `str` or `Path` — `isolated_esgpull_context` already handles coercion, so pass it directly. + - When passing `data_dir` to the refactored `esgpull_utils` functions, coerce to `Path` at the call site: `Path(self.config.data_dir)`. +- **Business:** Download behaviour must be identical — same files, same destination paths, same error handling. +- **Out of scope:** Modifying V1 downloaders (`CMIP6Downloader`, `Input4MipsDownloader`), tests (TASK-004), or the `isolated_esgpull_context` function itself. + +## 5. Acceptance Criteria +- [x] AC-1: `CMIP6DownloaderV2.download()` opens exactly one `isolated_esgpull_context` wrapping all iterations. +- [x] AC-2: `Input4MipsDownloaderV2.download()` opens exactly one `isolated_esgpull_context` wrapping all iterations. +- [x] AC-3: `download_from_model_single_var`, `download_raw_input_single_var`, and `download_meta_historic_biomassburning_single_var` all accept and forward `esg: Esgpull`. +- [x] AC-4: No direct calls to `isolated_esgpull_context` exist outside the two `download()` methods. +- [x] AC-5: `python3 -c "import ast; ast.parse(open('climateset/download/cmip6_downloader.py').read())"` exits 0. +- [x] AC-6: `python3 -c "import ast; ast.parse(open('climateset/download/input4mips_downloader.py').read())"` exits 0. +- [x] AC-7: `make precommit` and `make pylint` exit 0 for modified files. + +## 6. Testing & Validation +```bash +# Syntax check both files +python3 -c "import ast; ast.parse(open('climateset/download/cmip6_downloader.py').read())" +python3 -c "import ast; ast.parse(open('climateset/download/input4mips_downloader.py').read())" +# Expected: exit 0, no output + +# Lint +make precommit +make pylint +# Expected: exit 0 + +# Verify context is opened exactly once per download() method +grep -n "isolated_esgpull_context" climateset/download/cmip6_downloader.py +# Expected: import line + one call inside download() + +grep -n "isolated_esgpull_context" climateset/download/input4mips_downloader.py +# Expected: import line + one call inside download() + +# Verify esg is threaded through intermediate methods +grep -n "esg:" climateset/download/cmip6_downloader.py +# Expected: parameter in download_from_model_single_var signature + +grep -n "esg:" climateset/download/input4mips_downloader.py +# Expected: parameter in download_raw_input_single_var and download_meta_historic_biomassburning_single_var +``` + +## 7. Completion Protocol +1. Verify every AC is checked off in Section 5. +2. Run all commands in Section 6 and confirm expected output. +3. Stage and commit with a scoped message: + ```bash + git add climateset/download/cmip6_downloader.py climateset/download/input4mips_downloader.py + git commit -m "refactor(downloaders): lift esgpull context to batch scope in V2 downloaders — closes TASK-002" + ``` +4. Update this file: check off completed subtasks and ACs, note any deviations. +5. Notify the user with a concise summary and request approval before proceeding to TASK-003. diff --git a/docs/agents/planning/refactor-esgpull-isolation-to-batch-scoped/tasks/TASK-003-verify-context-statefulness.md b/docs/agents/planning/refactor-esgpull-isolation-to-batch-scoped/tasks/TASK-003-verify-context-statefulness.md new file mode 100644 index 0000000..7191555 --- /dev/null +++ b/docs/agents/planning/refactor-esgpull-isolation-to-batch-scoped/tasks/TASK-003-verify-context-statefulness.md @@ -0,0 +1,75 @@ +# TASK-003: Verify `esg.context` statefulness across sequential queries + +## 1. Goal +Prove that reusing a single `Esgpull` instance across multiple sequential search-and-download calls does not cause facet bleed — i.e., constraints from call N do not leak into call N+1. + +## 2. Context & References +- **Plan section:** Step 5 from `plan-refactor-esgpull-isolation.md`: + > 5. Verify `esg.context` statefulness — Write a focused integration test that calls `esgpull_search_and_download_esgf_model_single_var` twice with different `(variable, experiment)` pairs using the **same** `esg` instance, and asserts that each call returns files matching only its own constraints. +- **FMEA risk:** The plan identifies "`esg.context` carries stale facet state between variables" as **medium likelihood**. +- **Upstream tasks:** TASK-001 (signatures changed), TASK-002 (V2 downloaders updated). The refactored function signature from TASK-001: + ```python + def esgpull_search_and_download_esgf_model_single_var( + esg: Esgpull, model: str, variable: str, experiment: str, + project: str, default_grid_label: str, default_frequency: str, + preferred_version: str, max_ensemble_members: int, + ensemble_members: list[str], data_dir: Path, + distrib: bool = False, logger: logging.Logger = LOGGER, + ) -> list[Path] | None: ... + ``` +- **Key files:** + - `climateset/download/esgpull_utils.py` — functions under test + - `tests/test_download/test_utils.py` — existing test file (extend or add sibling) +- **Relevant skills:** `tdd` (Red-Green-Refactor), `python` (pytest fixtures) + +## 3. Subtasks +- [x] 1. Create `tests/test_download/test_esgpull_context_reuse.py` with a focused integration test class +- [x] 2. Write a test that opens one `isolated_esgpull_context`, calls `esgpull_search_and_download_esgf_model_single_var` twice with different `(variable, experiment)` pairs, and asserts each call's `Query` was constructed with the correct constraints (TDD Red — expect this to pass if esgpull creates fresh `Query` objects per call, or fail if state bleeds) +- [x] 3. If facet bleed is detected: add a `Query` reset or fresh `Query()` construction guard at the top of each function — then re-run to green +- [x] 4. Add a lifecycle test asserting that `isolated_esgpull_context` creates exactly one `.esgpull_jobs/` directory and cleans it up on exit +- [x] 5. Run the new tests in isolation to confirm they pass + +## 4. Requirements & Constraints +- **Technical:** + - Integration tests that hit real ESGF nodes should be marked with `@pytest.mark.integration` (or `@pytest.mark.slow`) so they can be skipped in CI fast-path runs. + - Mock `esg.download` (the async download) to avoid bandwidth usage — only the search/hints layer should hit the network. + - Use `tmp_path` fixture for the `data_dir` / `raw_data_path` to avoid polluting the workspace. +- **Business:** If facet bleed is confirmed, the fix must be minimal (e.g., ensuring `Query` is freshly constructed per call — which the current code already does). Do not refactor `esgpull` internals. +- **Out of scope:** Testing V1 downloaders. Performance benchmarking of shared vs. per-variable contexts. + +## 5. Acceptance Criteria +- [x] AC-1: A test exists that reuses one `Esgpull` instance across 2+ calls to `esgpull_search_and_download_esgf_model_single_var` with different parameters. +- [x] AC-2: The test asserts that each call's search results contain only files matching its own constraints (no cross-contamination). +- [x] AC-3: A lifecycle test asserts exactly one UUID directory is created and cleaned up per `isolated_esgpull_context` usage. +- [x] AC-4: All new tests pass: `make test-custom TEST_ARGS="tests/test_download/test_esgpull_context_reuse.py -v"` exits 0. +- [x] AC-5: `make precommit` exits 0. + +## 6. Testing & Validation +```bash +# Run the new test file +make test-custom TEST_ARGS="tests/test_download/test_esgpull_context_reuse.py -v" +# Expected: all tests pass + +# Run with integration marker (if tests hit real nodes) +make test-marker TEST_ARGS="integration" +# Expected: integration tests pass (may be slow) + +# Lint +make precommit +# Expected: exit 0 + +# Verify no regressions in existing tests +make test-custom TEST_ARGS="tests/test_download/ -v" +# Expected: all tests pass +``` + +## 7. Completion Protocol +1. Verify every AC is checked off in Section 5. +2. Run all commands in Section 6 and confirm expected output. +3. Stage and commit with a scoped message: + ```bash + git add tests/test_download/test_esgpull_context_reuse.py + git commit -m "test(esgpull): add context reuse and lifecycle tests — closes TASK-003" + ``` +4. Update this file: check off completed subtasks and ACs, note any deviations. +5. Notify the user with a concise summary and request approval before proceeding to TASK-004. diff --git a/docs/agents/planning/refactor-esgpull-isolation-to-batch-scoped/tasks/TASK-004-update-existing-tests.md b/docs/agents/planning/refactor-esgpull-isolation-to-batch-scoped/tasks/TASK-004-update-existing-tests.md new file mode 100644 index 0000000..1eac719 --- /dev/null +++ b/docs/agents/planning/refactor-esgpull-isolation-to-batch-scoped/tasks/TASK-004-update-existing-tests.md @@ -0,0 +1,74 @@ +# TASK-004: Update existing tests and run full validation suite + +## 1. Goal +Align all existing test files with the refactored function signatures from TASK-001/TASK-002, add a lifecycle assertion for single-context-per-batch, and confirm zero regressions across the full test suite and linting pipeline. + +## 2. Context & References +- **Plan section:** Steps 6 and 7 from `plan-refactor-esgpull-isolation.md`: + > 6. Update existing unit tests — Adjust `test_esgpull_utils.py` and any V2 downloader tests to reflect the new signatures. Add a lifecycle assertion that `isolated_esgpull_context` is entered exactly once per `download()` invocation. + > 7. Run full test suite and linting — Execute `make test`, `make precommit`, `make pylint`, and `make mypy` to confirm no regressions. +- **Upstream tasks:** + - TASK-001: `esgpull_search_and_download_*` now accept `esg: Esgpull` + `data_dir: Path` instead of `data_dir: Path | str` (no internal context). + - TASK-002: V2 downloader `download()` methods wrap loops in `isolated_esgpull_context`; intermediate methods accept `esg: Esgpull`. + - TASK-003: New test file `test_esgpull_context_reuse.py` exists and passes. +- **Key files:** + - `tests/test_download/test_utils.py` — likely contains tests for esgpull utility functions + - `tests/test_download/test_downloader.py` — likely contains tests for V2 downloaders + - `tests/test_download/test_constraints.py` — should be unaffected (no signature changes) + - `tests/test_download/test_search_client.py` — should be unaffected +- **Relevant skills:** `tdd` (green/refactor), `qa` (full validation pass), `python` (testing, type checking) + +## 3. Subtasks +- [x] 1. Read `tests/test_download/test_utils.py` and identify all tests that call `esgpull_search_and_download_*` functions directly — update call sites to pass a mock `Esgpull` instance as `esg` and `Path` as `data_dir` +- [x] 2. Read `tests/test_download/test_downloader.py` and identify all tests that instantiate `CMIP6DownloaderV2` or `Input4MipsDownloaderV2` — update mocks/patches for the new `isolated_esgpull_context` call site (now in `download()`, not in `esgpull_utils`) +- [x] 3. Add a lifecycle assertion: patch `isolated_esgpull_context` and assert it is called exactly **once** during a `download()` invocation with multiple variables +- [x] 4. Run `make test-custom TEST_ARGS="tests/test_download/ -v"` and fix any failures +- [x] 5. Run `make test` (full suite) and confirm no regressions +- [x] 6. Run `make precommit`, `make pylint`, and `make mypy` across modified files + +## 4. Requirements & Constraints +- **Technical:** + - When mocking `Esgpull`, mock only the instance — not the `isolated_esgpull_context` context manager — in tests that exercise the utility functions directly. For downloader-level tests, mock the context manager to avoid filesystem side effects. + - Do not modify V1 downloader tests — V1 code was not touched by this refactor. +- **Business:** All tests that passed before TASK-001 must still pass. Zero regressions. +- **Out of scope:** Writing new integration tests against live ESGF nodes (covered in TASK-003). Performance benchmarking. + +## 5. Acceptance Criteria +- [x] AC-1: All tests in `tests/test_download/test_utils.py` pass with the updated signatures. +- [x] AC-2: All tests in `tests/test_download/test_downloader.py` pass with the updated V2 downloader code. +- [x] AC-3: A test asserts `isolated_esgpull_context` is entered exactly once per `download()` call. +- [x] AC-4: `make test` (full suite) exits 0 with no failures or errors. +- [x] AC-5: `make precommit` exits 0. +- [x] AC-6: `make pylint` exits 0 for modified files. +- [x] AC-7: `make mypy` exits 0 for `climateset/download/esgpull_utils.py climateset/download/cmip6_downloader.py climateset/download/input4mips_downloader.py` (or only pre-existing errors unrelated to this change). + +## 6. Testing & Validation +```bash +# Targeted test run +make test-custom TEST_ARGS="tests/test_download/ -v" +# Expected: all tests pass + +# Full test suite +make test +# Expected: exit 0, no failures + +# Lint +make precommit +make pylint +# Expected: exit 0 + +# Type check +make mypy +# Expected: exit 0 or only pre-existing errors +``` + +## 7. Completion Protocol +1. Verify every AC is checked off in Section 5. +2. Run all commands in Section 6 and confirm expected output. +3. Stage and commit with a scoped message: + ```bash + git add tests/test_download/ + git commit -m "test(esgpull): update tests for batch-scoped context refactor — closes TASK-004" + ``` +4. Update this file: check off completed subtasks and ACs, note any deviations. +5. Notify the user with a concise summary. This is the final task — the refactor is complete. diff --git a/climateset/download/constants/selected_scenariosMIPs.json b/docs/selected_scenariosMIPs.json similarity index 100% rename from climateset/download/constants/selected_scenariosMIPs.json rename to docs/selected_scenariosMIPs.json diff --git a/noxfile.py b/noxfile.py index 23c23cb..f6d4309 100644 --- a/noxfile.py +++ b/noxfile.py @@ -176,9 +176,7 @@ def test_custom(session): if not ARG_RE.match(a): session.error(f"unsafe pytest argument detected: {a!r}") - session.run( - "python", "-m", "pytest", external=True, *session.posargs - ) # Pass additional arguments directly to pytest + session.run("pytest", *session.posargs, external=True) # Pass additional arguments directly to pytest @nox.session() diff --git a/poetry.lock b/poetry.lock index bf4e591..a81620c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,55 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.3.1 and should not be changed by hand. + +[[package]] +name = "aiofiles" +version = "25.1.0" +description = "File support for asyncio." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "aiofiles-25.1.0-py3-none-any.whl", hash = "sha256:abe311e527c862958650f9438e859c1fa7568a141b22abcd015e120e86a85695"}, + {file = "aiofiles-25.1.0.tar.gz", hash = "sha256:a8d728f0a29de45dc521f18f07297428d56992a742f0cd2701ba86e44d23d5b2"}, +] + +[[package]] +name = "aiostream" +version = "0.7.1" +description = "Generator-based operators for asynchronous iteration" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "aiostream-0.7.1-py3-none-any.whl", hash = "sha256:ea8739e9158ee6a606b3feedf3762721c3507344e540d09a10984c5e88a13b37"}, + {file = "aiostream-0.7.1.tar.gz", hash = "sha256:272aaa0d8f83beb906f5aa9022bb59046bb7a103fa3770f807c31f918595acf6"}, +] + +[package.dependencies] +typing-extensions = "*" + +[package.extras] +dev = ["pytest", "pytest-asyncio", "pytest-cov"] + +[[package]] +name = "alembic" +version = "1.18.4" +description = "A database migration tool for SQLAlchemy." +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "alembic-1.18.4-py3-none-any.whl", hash = "sha256:a5ed4adcf6d8a4cb575f3d759f071b03cd6e5c7618eb796cb52497be25bfe19a"}, + {file = "alembic-1.18.4.tar.gz", hash = "sha256:cb6e1fd84b6174ab8dbb2329f86d631ba9559dd78df550b57804d607672cedbc"}, +] + +[package.dependencies] +Mako = "*" +SQLAlchemy = ">=1.4.23" +tomli = {version = "*", markers = "python_version < \"3.11\""} +typing-extensions = ">=4.12" + +[package.extras] +tz = ["tzdata"] [[package]] name = "annotated-types" @@ -6,6 +57,7 @@ version = "0.6.0" description = "Reusable constraint types to use with typing.Annotated" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "annotated_types-0.6.0-py3-none-any.whl", hash = "sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43"}, {file = "annotated_types-0.6.0.tar.gz", hash = "sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d"}, @@ -17,6 +69,7 @@ version = "4.3.0" description = "High level compatibility layer for multiple asynchronous event loop implementations" optional = false python-versions = ">=3.8" +groups = ["main", "lab"] files = [ {file = "anyio-4.3.0-py3-none-any.whl", hash = "sha256:048e05d0f6caeed70d731f3db756d35dcc1f35747c8c403364a8332c630441b8"}, {file = "anyio-4.3.0.tar.gz", hash = "sha256:f75253795a87df48568485fd18cdd2a3fa5c4f7c5be8e5e36637733fce06fed6"}, @@ -30,7 +83,7 @@ typing-extensions = {version = ">=4.1", markers = "python_version < \"3.11\""} [package.extras] doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] -test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17) ; platform_python_implementation == \"CPython\" and platform_system != \"Windows\""] trio = ["trio (>=0.23)"] [[package]] @@ -39,6 +92,8 @@ version = "0.1.4" description = "Disable App Nap on macOS >= 10.9" optional = false python-versions = ">=3.6" +groups = ["lab"] +markers = "platform_system == \"Darwin\"" files = [ {file = "appnope-0.1.4-py2.py3-none-any.whl", hash = "sha256:502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c"}, {file = "appnope-0.1.4.tar.gz", hash = "sha256:1de3860566df9caf38f01f86f65e0e13e379af54f9e4bee1e66b48f2efffd1ee"}, @@ -50,6 +105,7 @@ version = "3.5.0" description = "Bash tab completion for argparse" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "argcomplete-3.5.0-py3-none-any.whl", hash = "sha256:d4bcf3ff544f51e16e54228a7ac7f486ed70ebf2ecfe49a63a91171c76bf029b"}, {file = "argcomplete-3.5.0.tar.gz", hash = "sha256:4349400469dccfb7950bb60334a680c58d88699bff6159df61251878dc6bf74b"}, @@ -64,6 +120,7 @@ version = "23.1.0" description = "Argon2 for Python" optional = false python-versions = ">=3.7" +groups = ["lab"] files = [ {file = "argon2_cffi-23.1.0-py3-none-any.whl", hash = "sha256:c670642b78ba29641818ab2e68bd4e6a78ba53b7eff7b4c3815ae16abf91c7ea"}, {file = "argon2_cffi-23.1.0.tar.gz", hash = "sha256:879c3e79a2729ce768ebb7d36d4609e3a78a4ca2ec3a9f12286ca057e3d0db08"}, @@ -84,6 +141,7 @@ version = "21.2.0" description = "Low-level CFFI bindings for Argon2" optional = false python-versions = ">=3.6" +groups = ["lab"] files = [ {file = "argon2-cffi-bindings-21.2.0.tar.gz", hash = "sha256:bb89ceffa6c791807d1305ceb77dbfacc5aa499891d2c55661c6459651fc39e3"}, {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ccb949252cb2ab3a08c02024acb77cfb179492d5701c7cbdbfd776124d4d2367"}, @@ -121,6 +179,7 @@ version = "1.3.0" description = "Better dates & times for Python" optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "arrow-1.3.0-py3-none-any.whl", hash = "sha256:c728b120ebc00eb84e01882a6f5e7927a53960aa990ce7dd2b10f39005a67f80"}, {file = "arrow-1.3.0.tar.gz", hash = "sha256:d4540617648cb5f895730f1ad8c82a65f2dad0166f57b75f3ca54759c4d67a85"}, @@ -140,6 +199,7 @@ version = "0.8.1" description = "Read/rewrite/write Python ASTs" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" +groups = ["dev"] files = [ {file = "astor-0.8.1-py2.py3-none-any.whl", hash = "sha256:070a54e890cefb5b3739d19f30f5a5ec840ffc9c50ffa7d23cc9fc1a38ebbfc5"}, {file = "astor-0.8.1.tar.gz", hash = "sha256:6a6effda93f4e1ce9f618779b2dd1d9d84f1e32812c23a29b3fff6fd7f63fa5e"}, @@ -151,6 +211,7 @@ version = "3.1.0" description = "An abstract syntax tree for Python with inference support." optional = false python-versions = ">=3.8.0" +groups = ["dev"] files = [ {file = "astroid-3.1.0-py3-none-any.whl", hash = "sha256:951798f922990137ac090c53af473db7ab4e70c770e6d7fae0cec59f74411819"}, {file = "astroid-3.1.0.tar.gz", hash = "sha256:ac248253bfa4bd924a0de213707e7ebeeb3138abeb48d798784ead1e56d419d4"}, @@ -165,6 +226,7 @@ version = "2.4.1" description = "Annotate AST trees with source code positions" optional = false python-versions = "*" +groups = ["lab"] files = [ {file = "asttokens-2.4.1-py2.py3-none-any.whl", hash = "sha256:051ed49c3dcae8913ea7cd08e46a606dba30b79993209636c4875bc1d637bc24"}, {file = "asttokens-2.4.1.tar.gz", hash = "sha256:b03869718ba9a6eb027e134bfdf69f38a236d681c83c160d510768af11254ba0"}, @@ -174,8 +236,8 @@ files = [ six = ">=1.12.0" [package.extras] -astroid = ["astroid (>=1,<2)", "astroid (>=2,<4)"] -test = ["astroid (>=1,<2)", "astroid (>=2,<4)", "pytest"] +astroid = ["astroid (>=1,<2) ; python_version < \"3\"", "astroid (>=2,<4) ; python_version >= \"3\""] +test = ["astroid (>=1,<2) ; python_version < \"3\"", "astroid (>=2,<4) ; python_version >= \"3\"", "pytest"] [[package]] name = "async-lru" @@ -183,6 +245,7 @@ version = "2.0.4" description = "Simple LRU cache for asyncio" optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "async-lru-2.0.4.tar.gz", hash = "sha256:b8a59a5df60805ff63220b2a0c5b5393da5521b113cd5465a44eb037d81a5627"}, {file = "async_lru-2.0.4-py3-none-any.whl", hash = "sha256:ff02944ce3c288c5be660c42dbcca0742b32c3b279d6dceda655190240b99224"}, @@ -197,6 +260,7 @@ version = "23.2.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.7" +groups = ["main", "lab"] files = [ {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, @@ -207,8 +271,40 @@ cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] dev = ["attrs[tests]", "pre-commit"] docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] tests = ["attrs[tests-no-zope]", "zope-interface"] -tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] -tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.6) ; platform_python_implementation == \"CPython\" and python_version >= \"3.8\"", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.8\""] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "autoflake" +version = "2.3.1" +description = "Removes unused imports and unused variables" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "autoflake-2.3.1-py3-none-any.whl", hash = "sha256:3ae7495db9084b7b32818b4140e6dc4fc280b712fb414f5b8fe57b0a8e85a840"}, + {file = "autoflake-2.3.1.tar.gz", hash = "sha256:c98b75dc5b0a86459c4f01a1d32ac7eb4338ec4317a4469515ff1e687ecd909e"}, +] + +[package.dependencies] +pyflakes = ">=3.0.0" +tomli = {version = ">=2.0.1", markers = "python_version < \"3.11\""} + +[[package]] +name = "autopep8" +version = "2.3.2" +description = "A tool that automatically formats Python code to conform to the PEP 8 style guide" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "autopep8-2.3.2-py2.py3-none-any.whl", hash = "sha256:ce8ad498672c845a0c3de2629c15b635ec2b05ef8177a6e7c91c74f3e9b51128"}, + {file = "autopep8-2.3.2.tar.gz", hash = "sha256:89440a4f969197b69a995e4ce0661b031f455a9f776d2c5ba3dbd83466931758"}, +] + +[package.dependencies] +pycodestyle = ">=2.12.0" +tomli = {version = "*", markers = "python_version < \"3.11\""} [[package]] name = "babel" @@ -216,6 +312,7 @@ version = "2.14.0" description = "Internationalization utilities" optional = false python-versions = ">=3.7" +groups = ["lab"] files = [ {file = "Babel-2.14.0-py3-none-any.whl", hash = "sha256:efb1a25b7118e67ce3a259bed20545c29cb68be8ad2c784c83689981b7a57287"}, {file = "Babel-2.14.0.tar.gz", hash = "sha256:6919867db036398ba21eb5c7a0f6b28ab8cbc3ae7a73a44ebe34ae74a4e7d363"}, @@ -230,6 +327,7 @@ version = "4.12.3" description = "Screen-scraping library" optional = false python-versions = ">=3.6.0" +groups = ["lab"] files = [ {file = "beautifulsoup4-4.12.3-py3-none-any.whl", hash = "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed"}, {file = "beautifulsoup4-4.12.3.tar.gz", hash = "sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051"}, @@ -251,6 +349,7 @@ version = "24.4.2" description = "The uncompromising code formatter." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "black-24.4.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dd1b5a14e417189db4c7b64a6540f31730713d173f0b63e55fabd52d61d8fdce"}, {file = "black-24.4.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e537d281831ad0e71007dcdcbe50a71470b978c453fa41ce77186bbe0ed6021"}, @@ -287,7 +386,7 @@ typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} [package.extras] colorama = ["colorama (>=0.4.3)"] -d = ["aiohttp (>=3.7.4)", "aiohttp (>=3.7.4,!=3.9.0)"] +d = ["aiohttp (>=3.7.4) ; sys_platform != \"win32\" or implementation_name != \"pypy\"", "aiohttp (>=3.7.4,!=3.9.0) ; sys_platform == \"win32\" and implementation_name == \"pypy\""] jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] uvloop = ["uvloop (>=0.15.2)"] @@ -297,6 +396,7 @@ version = "6.1.0" description = "An easy safelist-based HTML-sanitizing tool." optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "bleach-6.1.0-py3-none-any.whl", hash = "sha256:3225f354cfc436b9789c66c4ee030194bee0568fbf9cbdad3bc8b5c26c5f12b6"}, {file = "bleach-6.1.0.tar.gz", hash = "sha256:0a31f1837963c41d46bbf1331b8778e1308ea0791db03cc4e7357b97cf42a8fe"}, @@ -315,6 +415,7 @@ version = "0.16.2" description = "Version bump your Python project" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "bump-my-version-0.16.2.tar.gz", hash = "sha256:966dfc6cf9765a1d4a48fbaeb587a2e3a70ed5c13b04b5ee10c6e0134d4b8342"}, {file = "bump_my_version-0.16.2-py3-none-any.whl", hash = "sha256:83820caf4f64465c674f41346a2135999d7c3359c35e42a67708cb957c5135be"}, @@ -339,6 +440,7 @@ version = "23.2.3" description = "Composable complex class support for attrs and dataclasses." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, @@ -353,7 +455,7 @@ typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_ver bson = ["pymongo (>=4.4.0)"] cbor2 = ["cbor2 (>=5.4.6)"] msgpack = ["msgpack (>=1.0.5)"] -orjson = ["orjson (>=3.9.2)"] +orjson = ["orjson (>=3.9.2) ; implementation_name == \"cpython\""] pyyaml = ["pyyaml (>=6.0)"] tomlkit = ["tomlkit (>=0.11.8)"] ujson = ["ujson (>=5.7.0)"] @@ -364,6 +466,7 @@ version = "2024.2.2" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" +groups = ["main", "lab"] files = [ {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, @@ -375,6 +478,7 @@ version = "0.9.1" description = "A convenience wrapper for using CF attributes on xarray objects" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "cf_xarray-0.9.1-py3-none-any.whl", hash = "sha256:a2f328b4e7c391b3473b8973d57640d6a9e866fe6f435af89b16562056b35da8"}, {file = "cf_xarray-0.9.1.tar.gz", hash = "sha256:9aa965762d3da7015322cd7b78a24ed538e3671229adeebec5590ab429b5202c"}, @@ -392,6 +496,7 @@ version = "1.16.0" description = "Foreign Function Interface for Python calling C code." optional = false python-versions = ">=3.8" +groups = ["main", "lab"] files = [ {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, @@ -446,6 +551,7 @@ files = [ {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, ] +markers = {main = "platform_python_implementation != \"PyPy\""} [package.dependencies] pycparser = "*" @@ -456,6 +562,7 @@ version = "3.4.0" description = "Validate configuration and produce human readable error messages." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9"}, {file = "cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560"}, @@ -467,6 +574,7 @@ version = "1.6.3" description = "Time-handling functionality from netcdf4-python" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "cftime-1.6.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b62d42546fa5c914dfea5b15a9aaed2087ea1211cc36d08c374502ef95892038"}, {file = "cftime-1.6.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:eb6dd70b2ccabfe1a14b7fbb0bbdce0418e71697094373c0d573c880790fa291"}, @@ -500,6 +608,7 @@ version = "3.3.2" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7.0" +groups = ["main", "dev", "lab"] files = [ {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, @@ -599,6 +708,7 @@ version = "8.1.7" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" +groups = ["main", "dev"] files = [ {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, @@ -607,12 +717,30 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} +[[package]] +name = "click-params" +version = "0.5.0" +description = "A bunch of useful click parameter types" +optional = false +python-versions = ">=3.8.1,<4.0.0" +groups = ["main"] +files = [ + {file = "click_params-0.5.0-py3-none-any.whl", hash = "sha256:bbb2efe44197ab896bffcb50f42f22240fb077e6756b568fbdab3e1700b859d6"}, + {file = "click_params-0.5.0.tar.gz", hash = "sha256:5fe97b9459781a3b43b84fe4ec0065193e1b0d5cf6dc77897fe20c31f478d7ff"}, +] + +[package.dependencies] +click = ">=7.0,<9.0" +deprecated = ">=1.2.14,<2.0.0" +validators = ">=0.22,<0.23" + [[package]] name = "cloudpickle" version = "3.0.0" description = "Pickler class to extend the standard pickle.Pickler functionality" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "cloudpickle-3.0.0-py3-none-any.whl", hash = "sha256:246ee7d0c295602a036e86369c77fecda4ab17b506496730f2f576d9016fd9c7"}, {file = "cloudpickle-3.0.0.tar.gz", hash = "sha256:996d9a482c6fb4f33c1a35335cf8afd065d2a56e973270364840712d9131a882"}, @@ -624,10 +752,12 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main", "dev", "lab"] files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +markers = {main = "platform_system == \"Windows\"", dev = "platform_system == \"Windows\" or sys_platform == \"win32\"", lab = "sys_platform == \"win32\""} [[package]] name = "colorlog" @@ -635,6 +765,7 @@ version = "6.8.2" description = "Add colours to the output of Python's logging module." optional = false python-versions = ">=3.6" +groups = ["dev"] files = [ {file = "colorlog-6.8.2-py3-none-any.whl", hash = "sha256:4dcbb62368e2800cb3c5abd348da7e53f6c362dda502ec27c560b2e58a66bd33"}, {file = "colorlog-6.8.2.tar.gz", hash = "sha256:3e3e079a41feb5a1b64f978b5ea4f46040a94f11f0e8bbb8261e3dbbeca64d44"}, @@ -652,6 +783,7 @@ version = "0.2.2" description = "Jupyter Python Comm implementation, for usage in ipykernel, xeus-python etc." optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "comm-0.2.2-py3-none-any.whl", hash = "sha256:e6fb86cb70ff661ee8c9c14e7d36d6de3b4066f1441be4063df9c5009f0a64d3"}, {file = "comm-0.2.2.tar.gz", hash = "sha256:3fd7a84065306e07bea1773df6eb8282de51ba82f77c72f9c85716ab11fe980e"}, @@ -669,6 +801,7 @@ version = "42.0.7" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "cryptography-42.0.7-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:a987f840718078212fdf4504d0fd4c6effe34a7e4740378e59d47696e8dfb477"}, {file = "cryptography-42.0.7-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:bd13b5e9b543532453de08bcdc3cc7cebec6f9883e886fd20a92f26940fd3e7a"}, @@ -723,6 +856,7 @@ version = "2024.5.1" description = "Parallel PyData with Task Scheduling" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "dask-2024.5.1-py3-none-any.whl", hash = "sha256:af1cadd1fd1d1d44600ff5de43dd029e5668fdf87422131f4e3e3aa2a6a63555"}, {file = "dask-2024.5.1.tar.gz", hash = "sha256:e071fda67031c314569e37ca70b3e88bb30f1d91ff8ee4122b541845847cc264"}, @@ -731,7 +865,7 @@ files = [ [package.dependencies] click = ">=8.1" cloudpickle = ">=1.5.0" -fsspec = ">=2021.09.0" +fsspec = ">=2021.9.0" importlib-metadata = {version = ">=4.13.0", markers = "python_version < \"3.12\""} packaging = ">=20.0" partd = ">=1.2.0" @@ -752,6 +886,7 @@ version = "1.8.1" description = "An implementation of the Debug Adapter Protocol for Python" optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "debugpy-1.8.1-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:3bda0f1e943d386cc7a0e71bfa59f4137909e2ed947fb3946c506e113000f741"}, {file = "debugpy-1.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dda73bf69ea479c8577a0448f8c707691152e6c4de7f0c4dec5a4bc11dee516e"}, @@ -783,6 +918,7 @@ version = "5.1.1" description = "Decorators for Humans" optional = false python-versions = ">=3.5" +groups = ["lab"] files = [ {file = "decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"}, {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"}, @@ -794,17 +930,37 @@ version = "0.7.1" description = "XML bomb protection for Python stdlib modules" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +groups = ["main", "lab"] files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, ] +[[package]] +name = "deprecated" +version = "1.3.1" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" +groups = ["main"] +files = [ + {file = "deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f"}, + {file = "deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223"}, +] + +[package.dependencies] +wrapt = ">=1.10,<3" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "setuptools ; python_version >= \"3.12\"", "tox"] + [[package]] name = "dill" version = "0.3.8" description = "serialize all of Python" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7"}, {file = "dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca"}, @@ -820,6 +976,7 @@ version = "0.3.8" description = "Distribution utilities" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "distlib-0.3.8-py2.py3-none-any.whl", hash = "sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784"}, {file = "distlib-0.3.8.tar.gz", hash = "sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64"}, @@ -831,6 +988,7 @@ version = "1.7.5" description = "Formats docstrings to follow PEP 257" optional = false python-versions = ">=3.7,<4.0" +groups = ["dev"] files = [ {file = "docformatter-1.7.5-py3-none-any.whl", hash = "sha256:a24f5545ed1f30af00d106f5d85dc2fce4959295687c24c8f39f5263afaf9186"}, {file = "docformatter-1.7.5.tar.gz", hash = "sha256:ffed3da0daffa2e77f80ccba4f0e50bfa2755e1c10e130102571c890a61b246e"}, @@ -842,7 +1000,7 @@ tomli = {version = ">=2.0.0,<3.0.0", optional = true, markers = "python_version untokenize = ">=0.1.1,<0.2.0" [package.extras] -tomli = ["tomli (>=2.0.0,<3.0.0)"] +tomli = ["tomli (>=2.0.0,<3.0.0) ; python_version < \"3.11\""] [[package]] name = "esgf-pyclient" @@ -850,6 +1008,7 @@ version = "0.3.1" description = "A library interacting with ESGF services within Python" optional = false python-versions = ">=3.6.0" +groups = ["main"] files = [ {file = "esgf-pyclient-0.3.1.tar.gz", hash = "sha256:d79fafc6ec2a1eff9645e098554103f1e9f3c270947b1ac4f4a8a07fece59f3b"}, ] @@ -865,12 +1024,48 @@ webob = "*" [package.extras] dev = ["defusedxml", "flake8", "ipykernel", "jinja2", "myproxyclient", "nbconvert", "nbsphinx", "nbval", "pandoc", "pytest", "requests", "requests_cache", "sphinx", "webob"] +[[package]] +name = "esgpull" +version = "0.9.6" +description = "ESGF data discovery, download, replication tool" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "esgpull-0.9.6-py3-none-any.whl", hash = "sha256:56413a50bb259627bc16cc62d4f62bae2672515350e19c5ed811d4fa796826c5"}, + {file = "esgpull-0.9.6.tar.gz", hash = "sha256:1c11a02f5f03e4b411e4a63d756b414def936f839fdfe929d36d5280518d28e7"}, +] + +[package.dependencies] +aiofiles = ">=22.1.0" +aiostream = ">=0.4.5" +alembic = ">=1.8.1" +attrs = ">=22.1.0" +cattrs = ">=22.2.0" +click = ">=8.1.3" +click-params = ">=0.4.0" +httpx = ">=0.23.0" +nest-asyncio = ">=1.5.6" +packaging = ">=25.0" +platformdirs = ">=2.6.2" +pydantic = ">=2.11.7" +pydantic-settings = ">=2.10.1" +pyopenssl = ">=22.1.0" +pyparsing = ">=3.0.9" +pyyaml = ">=6.0" +rich = ">=12.6.0" +setuptools = ">=65.4.1" +sqlalchemy = ">=2.0.0b2,<2.1" +tomlkit = ">=0.11.5" + [[package]] name = "exceptiongroup" version = "1.2.1" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" +groups = ["main", "dev", "lab"] +markers = "python_version == \"3.10\"" files = [ {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"}, {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"}, @@ -885,13 +1080,14 @@ version = "2.0.1" description = "Get the currently executing AST node of a frame, and other information" optional = false python-versions = ">=3.5" +groups = ["lab"] files = [ {file = "executing-2.0.1-py2.py3-none-any.whl", hash = "sha256:eac49ca94516ccc753f9fb5ce82603156e590b27525a8bc32cce8ae302eb61bc"}, {file = "executing-2.0.1.tar.gz", hash = "sha256:35afe2ce3affba8ee97f2d69927fa823b08b472b7b994e36a52a964b93d16147"}, ] [package.extras] -tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich"] +tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich ; python_version >= \"3.11\""] [[package]] name = "fastjsonschema" @@ -899,6 +1095,7 @@ version = "2.19.1" description = "Fastest Python implementation of JSON schema" optional = false python-versions = "*" +groups = ["lab"] files = [ {file = "fastjsonschema-2.19.1-py3-none-any.whl", hash = "sha256:3672b47bc94178c9f23dbb654bf47440155d4db9df5f7bc47643315f9c405cd0"}, {file = "fastjsonschema-2.19.1.tar.gz", hash = "sha256:e3126a94bdc4623d3de4485f8d468a12f02a67921315ddc87836d6e456dc789d"}, @@ -913,6 +1110,7 @@ version = "3.13.4" description = "A platform independent file lock." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "filelock-3.13.4-py3-none-any.whl", hash = "sha256:404e5e9253aa60ad457cae1be07c0f0ca90a63931200a47d9b6a6af84fd7b45f"}, {file = "filelock-3.13.4.tar.gz", hash = "sha256:d13f466618bfde72bd2c18255e269f72542c6e70e7bac83a0232d6b1cc5c8cf4"}, @@ -921,22 +1119,23 @@ files = [ [package.extras] docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8.0.1)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)"] -typing = ["typing-extensions (>=4.8)"] +typing = ["typing-extensions (>=4.8) ; python_version < \"3.11\""] [[package]] name = "flake8" -version = "7.0.0" +version = "7.1.2" description = "the modular source code checker: pep8 pyflakes and co" optional = false python-versions = ">=3.8.1" +groups = ["dev"] files = [ - {file = "flake8-7.0.0-py2.py3-none-any.whl", hash = "sha256:a6dfbb75e03252917f2473ea9653f7cd799c3064e54d4c8140044c5c065f53c3"}, - {file = "flake8-7.0.0.tar.gz", hash = "sha256:33f96621059e65eec474169085dc92bf26e7b2d47366b70be2f67ab80dc25132"}, + {file = "flake8-7.1.2-py2.py3-none-any.whl", hash = "sha256:1cbc62e65536f65e6d754dfe6f1bada7f5cf392d6f5db3c2b85892466c3e7c1a"}, + {file = "flake8-7.1.2.tar.gz", hash = "sha256:c586ffd0b41540951ae41af572e6790dbd49fc12b3aa2541685d253d9bd504bd"}, ] [package.dependencies] mccabe = ">=0.7.0,<0.8.0" -pycodestyle = ">=2.11.0,<2.12.0" +pycodestyle = ">=2.12.0,<2.13.0" pyflakes = ">=3.2.0,<3.3.0" [[package]] @@ -945,6 +1144,7 @@ version = "1.2.3" description = "Flake8 plug-in loading the configuration from pyproject.toml" optional = false python-versions = ">= 3.6" +groups = ["dev"] files = [ {file = "flake8_pyproject-1.2.3-py3-none-any.whl", hash = "sha256:6249fe53545205af5e76837644dc80b4c10037e73a0e5db87ff562d75fb5bd4a"}, ] @@ -962,6 +1162,7 @@ version = "1.0.1" description = "CLI tool to convert a python project's %-formatted strings to f-strings." optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "flynt-1.0.1-py3-none-any.whl", hash = "sha256:65d1c546434827275123222a98408e9561bcd67db832dd58f530ff17b8329ec1"}, {file = "flynt-1.0.1.tar.gz", hash = "sha256:988aac00672a5469726cc0a17cef7d1178c284a9fe8563458db2475d0aaed965"}, @@ -980,6 +1181,7 @@ version = "1.5.1" description = "Validates fully-qualified domain names against RFC 1123, so that they are acceptable to modern bowsers" optional = false python-versions = ">=2.7, !=3.0, !=3.1, !=3.2, !=3.3, !=3.4, <4" +groups = ["lab"] files = [ {file = "fqdn-1.5.1-py3-none-any.whl", hash = "sha256:3a179af3761e4df6eb2e026ff9e1a3033d3587bf980a0b1b2e1e5d08d7358014"}, {file = "fqdn-1.5.1.tar.gz", hash = "sha256:105ed3677e767fb5ca086a0c1f4bb66ebc3c100be518f0e0d755d9eae164d89f"}, @@ -991,6 +1193,7 @@ version = "2024.5.0" description = "File-system specification" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "fsspec-2024.5.0-py3-none-any.whl", hash = "sha256:e0fdbc446d67e182f49a70b82cf7889028a63588fde6b222521f10937b2b670c"}, {file = "fsspec-2024.5.0.tar.gz", hash = "sha256:1d021b0b0f933e3b3029ed808eb400c08ba101ca2de4b3483fbc9ca23fcee94a"}, @@ -1029,17 +1232,87 @@ version = "1.0.0" description = "Clean single-source support for Python 3 and 2" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +groups = ["main"] files = [ {file = "future-1.0.0-py3-none-any.whl", hash = "sha256:929292d34f5872e70396626ef385ec22355a1fae8ad29e1a734c3e43f9fbc216"}, {file = "future-1.0.0.tar.gz", hash = "sha256:bd2968309307861edae1458a4f8a4f3598c03be43b97521076aebf5d94c07b05"}, ] +[[package]] +name = "greenlet" +version = "3.3.2" +description = "Lightweight in-process concurrent programming" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\"" +files = [ + {file = "greenlet-3.3.2-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:9bc885b89709d901859cf95179ec9f6bb67a3d2bb1f0e88456461bd4b7f8fd0d"}, + {file = "greenlet-3.3.2-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b568183cf65b94919be4438dc28416b234b678c608cafac8874dfeeb2a9bbe13"}, + {file = "greenlet-3.3.2-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:527fec58dc9f90efd594b9b700662ed3fb2493c2122067ac9c740d98080a620e"}, + {file = "greenlet-3.3.2-cp310-cp310-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:508c7f01f1791fbc8e011bd508f6794cb95397fdb198a46cb6635eb5b78d85a7"}, + {file = "greenlet-3.3.2-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad0c8917dd42a819fe77e6bdfcb84e3379c0de956469301d9fd36427a1ca501f"}, + {file = "greenlet-3.3.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:97245cc10e5515dbc8c3104b2928f7f02b6813002770cfaffaf9a6e0fc2b94ef"}, + {file = "greenlet-3.3.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8c1fdd7d1b309ff0da81d60a9688a8bd044ac4e18b250320a96fc68d31c209ca"}, + {file = "greenlet-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:5d0e35379f93a6d0222de929a25ab47b5eb35b5ef4721c2b9cbcc4036129ff1f"}, + {file = "greenlet-3.3.2-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:c56692189a7d1c7606cb794be0a8381470d95c57ce5be03fb3d0ef57c7853b86"}, + {file = "greenlet-3.3.2-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ebd458fa8285960f382841da585e02201b53a5ec2bac6b156fc623b5ce4499f"}, + {file = "greenlet-3.3.2-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a443358b33c4ec7b05b79a7c8b466f5d275025e750298be7340f8fc63dff2a55"}, + {file = "greenlet-3.3.2-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4375a58e49522698d3e70cc0b801c19433021b5c37686f7ce9c65b0d5c8677d2"}, + {file = "greenlet-3.3.2-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e2cd90d413acbf5e77ae41e5d3c9b3ac1d011a756d7284d7f3f2b806bbd6358"}, + {file = "greenlet-3.3.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:442b6057453c8cb29b4fb36a2ac689382fc71112273726e2423f7f17dc73bf99"}, + {file = "greenlet-3.3.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:45abe8eb6339518180d5a7fa47fa01945414d7cca5ecb745346fc6a87d2750be"}, + {file = "greenlet-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:1e692b2dae4cc7077cbb11b47d258533b48c8fde69a33d0d8a82e2fe8d8531d5"}, + {file = "greenlet-3.3.2-cp311-cp311-win_arm64.whl", hash = "sha256:02b0a8682aecd4d3c6c18edf52bc8e51eacdd75c8eac52a790a210b06aa295fd"}, + {file = "greenlet-3.3.2-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:ac8d61d4343b799d1e526db579833d72f23759c71e07181c2d2944e429eb09cd"}, + {file = "greenlet-3.3.2-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ceec72030dae6ac0c8ed7591b96b70410a8be370b6a477b1dbc072856ad02bd"}, + {file = "greenlet-3.3.2-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a2a5be83a45ce6188c045bcc44b0ee037d6a518978de9a5d97438548b953a1ac"}, + {file = "greenlet-3.3.2-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ae9e21c84035c490506c17002f5c8ab25f980205c3e61ddb3a2a2a2e6c411fcb"}, + {file = "greenlet-3.3.2-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43e99d1749147ac21dde49b99c9abffcbc1e2d55c67501465ef0930d6e78e070"}, + {file = "greenlet-3.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4c956a19350e2c37f2c48b336a3afb4bff120b36076d9d7fb68cb44e05d95b79"}, + {file = "greenlet-3.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6c6f8ba97d17a1e7d664151284cb3315fc5f8353e75221ed4324f84eb162b395"}, + {file = "greenlet-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:34308836d8370bddadb41f5a7ce96879b72e2fdfb4e87729330c6ab52376409f"}, + {file = "greenlet-3.3.2-cp312-cp312-win_arm64.whl", hash = "sha256:d3a62fa76a32b462a97198e4c9e99afb9ab375115e74e9a83ce180e7a496f643"}, + {file = "greenlet-3.3.2-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:aa6ac98bdfd716a749b84d4034486863fd81c3abde9aa3cf8eff9127981a4ae4"}, + {file = "greenlet-3.3.2-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab0c7e7901a00bc0a7284907273dc165b32e0d109a6713babd04471327ff7986"}, + {file = "greenlet-3.3.2-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d248d8c23c67d2291ffd47af766e2a3aa9fa1c6703155c099feb11f526c63a92"}, + {file = "greenlet-3.3.2-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ccd21bb86944ca9be6d967cf7691e658e43417782bce90b5d2faeda0ff78a7dd"}, + {file = "greenlet-3.3.2-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b6997d360a4e6a4e936c0f9625b1c20416b8a0ea18a8e19cabbefc712e7397ab"}, + {file = "greenlet-3.3.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:64970c33a50551c7c50491671265d8954046cb6e8e2999aacdd60e439b70418a"}, + {file = "greenlet-3.3.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1a9172f5bf6bd88e6ba5a84e0a68afeac9dc7b6b412b245dd64f52d83c81e55b"}, + {file = "greenlet-3.3.2-cp313-cp313-win_amd64.whl", hash = "sha256:a7945dd0eab63ded0a48e4dcade82939783c172290a7903ebde9e184333ca124"}, + {file = "greenlet-3.3.2-cp313-cp313-win_arm64.whl", hash = "sha256:394ead29063ee3515b4e775216cb756b2e3b4a7e55ae8fd884f17fa579e6b327"}, + {file = "greenlet-3.3.2-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:8d1658d7291f9859beed69a776c10822a0a799bc4bfe1bd4272bb60e62507dab"}, + {file = "greenlet-3.3.2-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:18cb1b7337bca281915b3c5d5ae19f4e76d35e1df80f4ad3c1a7be91fadf1082"}, + {file = "greenlet-3.3.2-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2e47408e8ce1c6f1ceea0dffcdf6ebb85cc09e55c7af407c99f1112016e45e9"}, + {file = "greenlet-3.3.2-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e3cb43ce200f59483eb82949bf1835a99cf43d7571e900d7c8d5c62cdf25d2f9"}, + {file = "greenlet-3.3.2-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63d10328839d1973e5ba35e98cccbca71b232b14051fd957b6f8b6e8e80d0506"}, + {file = "greenlet-3.3.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8e4ab3cfb02993c8cc248ea73d7dae6cec0253e9afa311c9b37e603ca9fad2ce"}, + {file = "greenlet-3.3.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:94ad81f0fd3c0c0681a018a976e5c2bd2ca2d9d94895f23e7bb1af4e8af4e2d5"}, + {file = "greenlet-3.3.2-cp314-cp314-win_amd64.whl", hash = "sha256:8c4dd0f3997cf2512f7601563cc90dfb8957c0cff1e3a1b23991d4ea1776c492"}, + {file = "greenlet-3.3.2-cp314-cp314-win_arm64.whl", hash = "sha256:cd6f9e2bbd46321ba3bbb4c8a15794d32960e3b0ae2cc4d49a1a53d314805d71"}, + {file = "greenlet-3.3.2-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:e26e72bec7ab387ac80caa7496e0f908ff954f31065b0ffc1f8ecb1338b11b54"}, + {file = "greenlet-3.3.2-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b466dff7a4ffda6ca975979bab80bdadde979e29fc947ac3be4451428d8b0e4"}, + {file = "greenlet-3.3.2-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b8bddc5b73c9720bea487b3bffdb1840fe4e3656fba3bd40aa1489e9f37877ff"}, + {file = "greenlet-3.3.2-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:59b3e2c40f6706b05a9cd299c836c6aa2378cabe25d021acd80f13abf81181cf"}, + {file = "greenlet-3.3.2-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b26b0f4428b871a751968285a1ac9648944cea09807177ac639b030bddebcea4"}, + {file = "greenlet-3.3.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1fb39a11ee2e4d94be9a76671482be9398560955c9e568550de0224e41104727"}, + {file = "greenlet-3.3.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:20154044d9085151bc309e7689d6f7ba10027f8f5a8c0676ad398b951913d89e"}, + {file = "greenlet-3.3.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c04c5e06ec3e022cbfe2cd4a846e1d4e50087444f875ff6d2c2ad8445495cf1a"}, + {file = "greenlet-3.3.2.tar.gz", hash = "sha256:2eaf067fc6d886931c7962e8c6bede15d2f01965560f3359b27c80bde2d151f2"}, +] + +[package.extras] +docs = ["Sphinx", "furo"] +test = ["objgraph", "psutil", "setuptools"] + [[package]] name = "h11" version = "0.14.0" description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" optional = false python-versions = ">=3.7" +groups = ["main", "lab"] files = [ {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, @@ -1051,6 +1324,7 @@ version = "1.3.0" description = "netCDF4 via h5py" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "h5netcdf-1.3.0-py3-none-any.whl", hash = "sha256:f2df69dcd3665dc9c4d43eb6529dedd113b2508090d12ac973573305a8406465"}, {file = "h5netcdf-1.3.0.tar.gz", hash = "sha256:a171c027daeb34b24c24a3b6304195b8eabbb6f10c748256ed3cfe19806383cf"}, @@ -1069,6 +1343,7 @@ version = "3.11.0" description = "Read and write HDF5 files from Python" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "h5py-3.11.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1625fd24ad6cfc9c1ccd44a66dac2396e7ee74940776792772819fc69f3a3731"}, {file = "h5py-3.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c072655ad1d5fe9ef462445d3e77a8166cbfa5e599045f8aa3c19b75315f10e5"}, @@ -1102,6 +1377,7 @@ version = "1.0.5" description = "A minimal low-level HTTP client." optional = false python-versions = ">=3.8" +groups = ["main", "lab"] files = [ {file = "httpcore-1.0.5-py3-none-any.whl", hash = "sha256:421f18bac248b25d310f3cacd198d55b8e6125c107797b609ff9b7a6ba7991b5"}, {file = "httpcore-1.0.5.tar.gz", hash = "sha256:34a38e2f9291467ee3b44e89dd52615370e152954ba21721378a87b2960f7a61"}, @@ -1123,6 +1399,7 @@ version = "0.27.0" description = "The next generation HTTP client." optional = false python-versions = ">=3.8" +groups = ["main", "lab"] files = [ {file = "httpx-0.27.0-py3-none-any.whl", hash = "sha256:71d5465162c13681bff01ad59b2cc68dd838ea1f10e51574bac27103f00c91a5"}, {file = "httpx-0.27.0.tar.gz", hash = "sha256:a0cb88a46f32dc874e04ee956e4c2764aba2aa228f650b06788ba6bda2962ab5"}, @@ -1136,7 +1413,7 @@ idna = "*" sniffio = "*" [package.extras] -brotli = ["brotli", "brotlicffi"] +brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""] cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] @@ -1147,6 +1424,7 @@ version = "2.5.36" description = "File identification library for Python" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "identify-2.5.36-py2.py3-none-any.whl", hash = "sha256:37d93f380f4de590500d9dba7db359d0d3da95ffe7f9de1753faa159e71e7dfa"}, {file = "identify-2.5.36.tar.gz", hash = "sha256:e5e00f54165f9047fbebeb4a560f9acfb8af4c88232be60a488e9b68d122745d"}, @@ -1161,6 +1439,7 @@ version = "3.7" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.5" +groups = ["main", "lab"] files = [ {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, @@ -1172,6 +1451,7 @@ version = "7.1.0" description = "Read metadata from Python packages" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "importlib_metadata-7.1.0-py3-none-any.whl", hash = "sha256:30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570"}, {file = "importlib_metadata-7.1.0.tar.gz", hash = "sha256:b78938b926ee8d5f020fc4772d487045805a55ddbad2ecf21c6d60938dc7fcd2"}, @@ -1183,7 +1463,7 @@ zipp = ">=0.5" [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] perf = ["ipython"] -testing = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-perf (>=0.9.2)", "pytest-ruff (>=0.2.1)"] +testing = ["flufl.flake8", "importlib-resources (>=1.3) ; python_version < \"3.9\"", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy ; platform_python_implementation != \"PyPy\"", "pytest-perf (>=0.9.2)", "pytest-ruff (>=0.2.1)"] [[package]] name = "iniconfig" @@ -1191,6 +1471,7 @@ version = "2.0.0" description = "brain-dead simple config-ini parsing" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, @@ -1202,6 +1483,7 @@ version = "6.29.4" description = "IPython Kernel for Jupyter" optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "ipykernel-6.29.4-py3-none-any.whl", hash = "sha256:1181e653d95c6808039c509ef8e67c4126b3b3af7781496c7cbfb5ed938a27da"}, {file = "ipykernel-6.29.4.tar.gz", hash = "sha256:3d44070060f9475ac2092b760123fadf105d2e2493c24848b6691a7c4f42af5c"}, @@ -1235,6 +1517,7 @@ version = "8.24.0" description = "IPython: Productive Interactive Computing" optional = false python-versions = ">=3.10" +groups = ["lab"] files = [ {file = "ipython-8.24.0-py3-none-any.whl", hash = "sha256:d7bf2f6c4314984e3e02393213bab8703cf163ede39672ce5918c51fe253a2a3"}, {file = "ipython-8.24.0.tar.gz", hash = "sha256:010db3f8a728a578bb641fdd06c063b9fb8e96a9464c63aec6310fbcb5e80501"}, @@ -1273,6 +1556,7 @@ version = "20.11.0" description = "Operations with ISO 8601 durations" optional = false python-versions = ">=3.7" +groups = ["lab"] files = [ {file = "isoduration-20.11.0-py3-none-any.whl", hash = "sha256:b2904c2a4228c3d44f409c8ae8e2370eb21a26f7ac2ec5446df141dde3452042"}, {file = "isoduration-20.11.0.tar.gz", hash = "sha256:ac2f9015137935279eac671f94f89eb00584f940f5dc49462a0c4ee692ba1bd9"}, @@ -1287,6 +1571,7 @@ version = "5.13.2" description = "A Python utility / library to sort Python imports." optional = false python-versions = ">=3.8.0" +groups = ["dev"] files = [ {file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"}, {file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"}, @@ -1301,6 +1586,7 @@ version = "0.19.1" description = "An autocompletion tool for Python that can be used for text editors." optional = false python-versions = ">=3.6" +groups = ["lab"] files = [ {file = "jedi-0.19.1-py2.py3-none-any.whl", hash = "sha256:e983c654fe5c02867aef4cdfce5a2fbb4a50adc0af145f70504238f18ef5e7e0"}, {file = "jedi-0.19.1.tar.gz", hash = "sha256:cf0496f3651bc65d7174ac1b7d043eff454892c708a87d1b683e57b569927ffd"}, @@ -1320,6 +1606,7 @@ version = "3.1.4" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" +groups = ["main", "lab", "secondary"] files = [ {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"}, {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"}, @@ -1337,6 +1624,7 @@ version = "0.9.25" description = "A Python implementation of the JSON5 data format." optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "json5-0.9.25-py3-none-any.whl", hash = "sha256:34ed7d834b1341a86987ed52f3f76cd8ee184394906b6e22a1e0deb9ab294e8f"}, {file = "json5-0.9.25.tar.gz", hash = "sha256:548e41b9be043f9426776f05df8635a00fe06104ea51ed24b67f908856e151ae"}, @@ -1348,6 +1636,7 @@ version = "2.4" description = "Identify specific nodes in a JSON document (RFC 6901)" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" +groups = ["lab"] files = [ {file = "jsonpointer-2.4-py2.py3-none-any.whl", hash = "sha256:15d51bba20eea3165644553647711d150376234112651b4f1811022aecad7d7a"}, {file = "jsonpointer-2.4.tar.gz", hash = "sha256:585cee82b70211fa9e6043b7bb89db6e1aa49524340dde8ad6b63206ea689d88"}, @@ -1359,6 +1648,7 @@ version = "4.21.1" description = "An implementation of JSON Schema validation for Python" optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "jsonschema-4.21.1-py3-none-any.whl", hash = "sha256:7996507afae316306f9e2290407761157c6f78002dcf7419acb99822143d1c6f"}, {file = "jsonschema-4.21.1.tar.gz", hash = "sha256:85727c00279f5fa6bedbe6238d2aa6403bedd8b4864ab11207d07df3cc1b2ee5"}, @@ -1370,7 +1660,7 @@ fqdn = {version = "*", optional = true, markers = "extra == \"format-nongpl\""} idna = {version = "*", optional = true, markers = "extra == \"format-nongpl\""} isoduration = {version = "*", optional = true, markers = "extra == \"format-nongpl\""} jsonpointer = {version = ">1.13", optional = true, markers = "extra == \"format-nongpl\""} -jsonschema-specifications = ">=2023.03.6" +jsonschema-specifications = ">=2023.3.6" referencing = ">=0.28.4" rfc3339-validator = {version = "*", optional = true, markers = "extra == \"format-nongpl\""} rfc3986-validator = {version = ">0.1.0", optional = true, markers = "extra == \"format-nongpl\""} @@ -1388,6 +1678,7 @@ version = "2023.12.1" description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "jsonschema_specifications-2023.12.1-py3-none-any.whl", hash = "sha256:87e4fdf3a94858b8a2ba2778d9ba57d8a9cafca7c7489c46ba0d30a8bc6a9c3c"}, {file = "jsonschema_specifications-2023.12.1.tar.gz", hash = "sha256:48a76787b3e70f5ed53f1160d2b81f586e4ca6d1548c5de7085d1682674764cc"}, @@ -1402,6 +1693,7 @@ version = "8.6.1" description = "Jupyter protocol implementation and client libraries" optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "jupyter_client-8.6.1-py3-none-any.whl", hash = "sha256:3b7bd22f058434e3b9a7ea4b1500ed47de2713872288c0d511d19926f99b459f"}, {file = "jupyter_client-8.6.1.tar.gz", hash = "sha256:e842515e2bab8e19186d89fdfea7abd15e39dd581f94e399f00e2af5a1652d3f"}, @@ -1416,7 +1708,7 @@ traitlets = ">=5.3" [package.extras] docs = ["ipykernel", "myst-parser", "pydata-sphinx-theme", "sphinx (>=4)", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling"] -test = ["coverage", "ipykernel (>=6.14)", "mypy", "paramiko", "pre-commit", "pytest", "pytest-cov", "pytest-jupyter[client] (>=0.4.1)", "pytest-timeout"] +test = ["coverage", "ipykernel (>=6.14)", "mypy", "paramiko ; sys_platform == \"win32\"", "pre-commit", "pytest", "pytest-cov", "pytest-jupyter[client] (>=0.4.1)", "pytest-timeout"] [[package]] name = "jupyter-core" @@ -1424,6 +1716,7 @@ version = "5.7.2" description = "Jupyter core package. A base package on which Jupyter projects rely." optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "jupyter_core-5.7.2-py3-none-any.whl", hash = "sha256:4f7315d2f6b4bcf2e3e7cb6e46772eba760ae459cd1f59d29eb57b0a01bd7409"}, {file = "jupyter_core-5.7.2.tar.gz", hash = "sha256:aa5f8d32bbf6b431ac830496da7392035d6f61b4f54872f15c4bd2a9c3f536d9"}, @@ -1444,6 +1737,7 @@ version = "0.10.0" description = "Jupyter Event System library" optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "jupyter_events-0.10.0-py3-none-any.whl", hash = "sha256:4b72130875e59d57716d327ea70d3ebc3af1944d3717e5a498b8a06c6c159960"}, {file = "jupyter_events-0.10.0.tar.gz", hash = "sha256:670b8229d3cc882ec782144ed22e0d29e1c2d639263f92ca8383e66682845e22"}, @@ -1469,6 +1763,7 @@ version = "2.2.5" description = "Multi-Language Server WebSocket proxy for Jupyter Notebook/Lab server" optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "jupyter-lsp-2.2.5.tar.gz", hash = "sha256:793147a05ad446f809fd53ef1cd19a9f5256fd0a2d6b7ce943a982cb4f545001"}, {file = "jupyter_lsp-2.2.5-py3-none-any.whl", hash = "sha256:45fbddbd505f3fbfb0b6cb2f1bc5e15e83ab7c79cd6e89416b248cb3c00c11da"}, @@ -1483,6 +1778,7 @@ version = "2.14.0" description = "The backend—i.e. core services, APIs, and REST endpoints—to Jupyter web applications." optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "jupyter_server-2.14.0-py3-none-any.whl", hash = "sha256:fb6be52c713e80e004fac34b35a0990d6d36ba06fd0a2b2ed82b899143a64210"}, {file = "jupyter_server-2.14.0.tar.gz", hash = "sha256:659154cea512083434fd7c93b7fe0897af7a2fd0b9dd4749282b42eaac4ae677"}, @@ -1519,6 +1815,7 @@ version = "0.5.3" description = "A Jupyter Server Extension Providing Terminals." optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "jupyter_server_terminals-0.5.3-py3-none-any.whl", hash = "sha256:41ee0d7dc0ebf2809c668e0fc726dfaf258fcd3e769568996ca731b6194ae9aa"}, {file = "jupyter_server_terminals-0.5.3.tar.gz", hash = "sha256:5ae0295167220e9ace0edcfdb212afd2b01ee8d179fe6f23c899590e9b8a5269"}, @@ -1538,6 +1835,7 @@ version = "4.1.8" description = "JupyterLab computational environment" optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "jupyterlab-4.1.8-py3-none-any.whl", hash = "sha256:c3baf3a2f91f89d110ed5786cd18672b9a357129d4e389d2a0dead15e11a4d2c"}, {file = "jupyterlab-4.1.8.tar.gz", hash = "sha256:3384aded8680e7ce504fd63b8bb89a39df21c9c7694d9e7dc4a68742cdb30f9b"}, @@ -1571,6 +1869,7 @@ version = "5.1.0" description = "Coding assistance for JupyterLab with Language Server Protocol" optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "jupyterlab-lsp-5.1.0.tar.gz", hash = "sha256:aeac84093ada6d20ef57ae0e97811cc5796a0cab7237b32f8eddf993c0bb0356"}, {file = "jupyterlab_lsp-5.1.0-py3-none-any.whl", hash = "sha256:290f473b001c7ebe1edfe391a545f50bc4aefe89d4a64db4f17e8de301db3a37"}, @@ -1586,6 +1885,7 @@ version = "0.3.0" description = "Pygments theme using JupyterLab CSS variables" optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "jupyterlab_pygments-0.3.0-py3-none-any.whl", hash = "sha256:841a89020971da1d8693f1a99997aefc5dc424bb1b251fd6322462a1b8842780"}, {file = "jupyterlab_pygments-0.3.0.tar.gz", hash = "sha256:721aca4d9029252b11cfa9d185e5b5af4d54772bb8072f9b7036f4170054d35d"}, @@ -1597,6 +1897,7 @@ version = "2.27.1" description = "A set of server components for JupyterLab and JupyterLab like applications." optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "jupyterlab_server-2.27.1-py3-none-any.whl", hash = "sha256:f5e26156e5258b24d532c84e7c74cc212e203bff93eb856f81c24c16daeecc75"}, {file = "jupyterlab_server-2.27.1.tar.gz", hash = "sha256:097b5ac709b676c7284ac9c5e373f11930a561f52cd5a86e4fc7e5a9c8a8631d"}, @@ -1622,6 +1923,7 @@ version = "0.42.0" description = "lightweight wrapper around basic LLVM functionality" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "llvmlite-0.42.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3366938e1bf63d26c34fbfb4c8e8d2ded57d11e0567d5bb243d89aab1eb56098"}, {file = "llvmlite-0.42.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c35da49666a21185d21b551fc3caf46a935d54d66969d32d72af109b5e7d2b6f"}, @@ -1652,17 +1954,39 @@ version = "1.0.0" description = "File-based locks for Python on Linux and Windows" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +groups = ["main"] files = [ {file = "locket-1.0.0-py2.py3-none-any.whl", hash = "sha256:b6c819a722f7b6bd955b80781788e4a66a55628b858d347536b7e81325a3a5e3"}, {file = "locket-1.0.0.tar.gz", hash = "sha256:5c0d4c052a8bbbf750e056a8e65ccd309086f4f0f18a2eac306a8dfa4112a632"}, ] +[[package]] +name = "mako" +version = "1.3.10" +description = "A super-fast templating language that borrows the best ideas from the existing templating languages." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "mako-1.3.10-py3-none-any.whl", hash = "sha256:baef24a52fc4fc514a0887ac600f9f1cff3d82c61d4d700a1fa84d597b88db59"}, + {file = "mako-1.3.10.tar.gz", hash = "sha256:99579a6f39583fa7e5630a28c3c1f440e4e97a414b80372649c0ce338da2ea28"}, +] + +[package.dependencies] +MarkupSafe = ">=0.9.2" + +[package.extras] +babel = ["Babel"] +lingua = ["lingua"] +testing = ["pytest"] + [[package]] name = "markdown-it-py" version = "3.0.0" description = "Python port of markdown-it. Markdown parsing, done right!" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, @@ -1687,6 +2011,7 @@ version = "2.1.5" description = "Safely add untrusted strings to HTML/XML markup." optional = false python-versions = ">=3.7" +groups = ["main", "lab", "secondary"] files = [ {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, @@ -1756,6 +2081,7 @@ version = "0.1.7" description = "Inline Matplotlib backend for Jupyter" optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "matplotlib_inline-0.1.7-py3-none-any.whl", hash = "sha256:df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca"}, {file = "matplotlib_inline-0.1.7.tar.gz", hash = "sha256:8423b23ec666be3d16e16b60bdd8ac4e86e840ebd1dd11a30b9f117f2fa0ab90"}, @@ -1770,6 +2096,7 @@ version = "0.7.0" description = "McCabe checker, plugin for flake8" optional = false python-versions = ">=3.6" +groups = ["dev"] files = [ {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, @@ -1781,6 +2108,7 @@ version = "0.1.2" description = "Markdown URL utilities" optional = false python-versions = ">=3.7" +groups = ["main", "dev"] files = [ {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, @@ -1792,6 +2120,7 @@ version = "3.0.2" description = "A sane and fast Markdown parser with useful plugins and renderers" optional = false python-versions = ">=3.7" +groups = ["lab"] files = [ {file = "mistune-3.0.2-py3-none-any.whl", hash = "sha256:71481854c30fdbc938963d3605b72501f5c10a9320ecd412c121c163a1c7d205"}, {file = "mistune-3.0.2.tar.gz", hash = "sha256:fc7f93ded930c92394ef2cb6f04a8aabab4117a91449e72dcc8dfa646a508be8"}, @@ -1803,9 +2132,11 @@ version = "2.1.1" description = "MyProxy Client" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "MyProxyClient-2.1.1-py3-none-any.whl", hash = "sha256:7e95927fec0e0981f1911aa8a766f77b81ecfd3a696a2bb7e20381c04649d9ae"}, {file = "MyProxyClient-2.1.1.tar.gz", hash = "sha256:7e998f1cf2246abdd1fbeba067f87d00d7a18d3bc138334dd93ae9a5f5ae3e5e"}, + {file = "myproxyclient-2.1.1-py3-none-any.whl", hash = "sha256:7e9bba51029e16b5493127eaffb3650679f85df7a062a14a4c6688cbbc84ad53"}, ] [package.dependencies] @@ -1818,6 +2149,7 @@ version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." optional = false python-versions = ">=3.5" +groups = ["dev"] files = [ {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, @@ -1829,6 +2161,7 @@ version = "0.10.0" description = "A client library for executing notebooks. Formerly nbconvert's ExecutePreprocessor." optional = false python-versions = ">=3.8.0" +groups = ["lab"] files = [ {file = "nbclient-0.10.0-py3-none-any.whl", hash = "sha256:f13e3529332a1f1f81d82a53210322476a168bb7090a0289c795fe9cc11c9d3f"}, {file = "nbclient-0.10.0.tar.gz", hash = "sha256:4b3f1b7dba531e498449c4db4f53da339c91d449dc11e9af3a43b4eb5c5abb09"}, @@ -1851,6 +2184,7 @@ version = "7.16.3" description = "Converting Jupyter Notebooks (.ipynb files) to other formats. Output formats include asciidoc, html, latex, markdown, pdf, py, rst, script. nbconvert can be used both as a Python library (`import nbconvert`) or as a command line tool (invoked as `jupyter nbconvert ...`)." optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "nbconvert-7.16.3-py3-none-any.whl", hash = "sha256:ddeff14beeeedf3dd0bc506623e41e4507e551736de59df69a91f86700292b3b"}, {file = "nbconvert-7.16.3.tar.gz", hash = "sha256:a6733b78ce3d47c3f85e504998495b07e6ea9cf9bf6ec1c98dda63ec6ad19142"}, @@ -1888,6 +2222,7 @@ version = "5.10.4" description = "The Jupyter Notebook format" optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "nbformat-5.10.4-py3-none-any.whl", hash = "sha256:3b48d6c8fbca4b299bf3982ea7db1af21580e4fec269ad087b9e81588891200b"}, {file = "nbformat-5.10.4.tar.gz", hash = "sha256:322168b14f937a5d11362988ecac2a4952d3d8e3a2cbeb2319584631226d5b3a"}, @@ -1909,6 +2244,7 @@ version = "1.6.0" description = "Patch asyncio to allow nested event loops" optional = false python-versions = ">=3.5" +groups = ["main", "lab"] files = [ {file = "nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c"}, {file = "nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe"}, @@ -1920,6 +2256,7 @@ version = "1.7.1.post2" description = "Provides an object-oriented python interface to the netCDF version 4 library" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "netCDF4-1.7.1.post2-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:a1006ae117a754e3cf41a9e704032bf3837cbf53a695cd71deaad3e02e93d570"}, {file = "netCDF4-1.7.1.post2-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:7530d60cf6450d997ea0607f8b68b9b088f2382c42648cddf5e66e6f1280b692"}, @@ -1962,6 +2299,7 @@ version = "1.8.0" description = "Node.js virtual environment builder" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" +groups = ["dev"] files = [ {file = "nodeenv-1.8.0-py2.py3-none-any.whl", hash = "sha256:df865724bb3c3adc86b3876fa209771517b0cfe596beff01a92700e0e8be4cec"}, {file = "nodeenv-1.8.0.tar.gz", hash = "sha256:d51e0c37e64fbf47d017feac3145cdbb58836d7eee8c6f6d3b6880c5456227d2"}, @@ -1976,6 +2314,7 @@ version = "7.1.3" description = "Jupyter Notebook - A web-based notebook environment for interactive computing" optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "notebook-7.1.3-py3-none-any.whl", hash = "sha256:919b911e59f41f6e3857ce93c9d93535ba66bb090059712770e5968c07e1004d"}, {file = "notebook-7.1.3.tar.gz", hash = "sha256:41fcebff44cf7bb9377180808bcbae066629b55d8c7722f1ebbe75ca44f9cfc1"}, @@ -1991,7 +2330,7 @@ tornado = ">=6.2.0" [package.extras] dev = ["hatch", "pre-commit"] docs = ["myst-parser", "nbsphinx", "pydata-sphinx-theme", "sphinx (>=1.3.6)", "sphinxcontrib-github-alt", "sphinxcontrib-spelling"] -test = ["importlib-resources (>=5.0)", "ipykernel", "jupyter-server[test] (>=2.4.0,<3)", "jupyterlab-server[test] (>=2.22.1,<3)", "nbval", "pytest (>=7.0)", "pytest-console-scripts", "pytest-timeout", "pytest-tornasync", "requests"] +test = ["importlib-resources (>=5.0) ; python_version < \"3.10\"", "ipykernel", "jupyter-server[test] (>=2.4.0,<3)", "jupyterlab-server[test] (>=2.22.1,<3)", "nbval", "pytest (>=7.0)", "pytest-console-scripts", "pytest-timeout", "pytest-tornasync", "requests"] [[package]] name = "notebook-shim" @@ -1999,6 +2338,7 @@ version = "0.2.4" description = "A shim layer for notebook traits and config" optional = false python-versions = ">=3.7" +groups = ["lab"] files = [ {file = "notebook_shim-0.2.4-py3-none-any.whl", hash = "sha256:411a5be4e9dc882a074ccbcae671eda64cceb068767e9a3419096986560e1cef"}, {file = "notebook_shim-0.2.4.tar.gz", hash = "sha256:b4b2cfa1b65d98307ca24361f5b30fe785b53c3fd07b7a47e89acb5e6ac638cb"}, @@ -2016,6 +2356,7 @@ version = "2024.4.15" description = "Flexible test automation." optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "nox-2024.4.15-py3-none-any.whl", hash = "sha256:6492236efa15a460ecb98e7b67562a28b70da006ab0be164e8821177577c0565"}, {file = "nox-2024.4.15.tar.gz", hash = "sha256:ecf6700199cdfa9e5ea0a41ff5e6ef4641d09508eda6edb89d9987864115817f"}, @@ -2038,6 +2379,7 @@ version = "0.59.1" description = "compiling Python code using LLVM" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "numba-0.59.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:97385a7f12212c4f4bc28f648720a92514bee79d7063e40ef66c2d30600fd18e"}, {file = "numba-0.59.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0b77aecf52040de2a1eb1d7e314497b9e56fba17466c80b457b971a25bb1576d"}, @@ -2072,6 +2414,7 @@ version = "1.26.4" description = "Fundamental package for array computing in Python" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, @@ -2117,6 +2460,7 @@ version = "7.7.0" description = "A decorator to automatically detect mismatch when overriding a method." optional = false python-versions = ">=3.6" +groups = ["lab"] files = [ {file = "overrides-7.7.0-py3-none-any.whl", hash = "sha256:c7ed9d062f78b8e4c1a7b70bd8796b35ead4d9f510227ef9c5dc7626c60d7e49"}, {file = "overrides-7.7.0.tar.gz", hash = "sha256:55158fa3d93b98cc75299b1e67078ad9003ca27945c76162c1c0766d6f91820a"}, @@ -2124,13 +2468,14 @@ files = [ [[package]] name = "packaging" -version = "24.0" +version = "26.0" description = "Core utilities for Python packages" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +groups = ["main", "dev", "lab"] files = [ - {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, - {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, + {file = "packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529"}, + {file = "packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4"}, ] [[package]] @@ -2139,6 +2484,7 @@ version = "2.2.2" description = "Powerful data structures for data analysis, time series, and statistics" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "pandas-2.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:90c6fca2acf139569e74e8781709dccb6fe25940488755716d1d354d6bc58bce"}, {file = "pandas-2.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c7adfc142dac335d8c1e0dcbd37eb8617eac386596eb9e1a1b77791cf2498238"}, @@ -2211,6 +2557,7 @@ version = "1.5.1" description = "Utilities for writing pandoc filters in python" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +groups = ["lab"] files = [ {file = "pandocfilters-1.5.1-py2.py3-none-any.whl", hash = "sha256:93be382804a9cdb0a7267585f157e5d1731bbe5545a85b268d6f5fe6232de2bc"}, {file = "pandocfilters-1.5.1.tar.gz", hash = "sha256:002b4a555ee4ebc03f8b66307e287fa492e4a77b4ea14d3f934328297bb4939e"}, @@ -2222,6 +2569,7 @@ version = "0.8.4" description = "A Python Parser" optional = false python-versions = ">=3.6" +groups = ["lab"] files = [ {file = "parso-0.8.4-py2.py3-none-any.whl", hash = "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18"}, {file = "parso-0.8.4.tar.gz", hash = "sha256:eb3a7b58240fb99099a345571deecc0f9540ea5f4dd2fe14c2a99d6b281ab92d"}, @@ -2237,6 +2585,7 @@ version = "1.4.2" description = "Appendable key-value storage" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "partd-1.4.2-py3-none-any.whl", hash = "sha256:978e4ac767ec4ba5b86c6eaa52e5a2a3bc748a2ca839e8cc798f1cc6ce6efb0f"}, {file = "partd-1.4.2.tar.gz", hash = "sha256:d022c33afbdc8405c226621b015e8067888173d85f7f5ecebb3cafed9a20f02c"}, @@ -2255,6 +2604,7 @@ version = "0.12.1" description = "Utility library for gitignore style pattern matching of file paths." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, @@ -2266,6 +2616,8 @@ version = "4.9.0" description = "Pexpect allows easy control of interactive console applications." optional = false python-versions = "*" +groups = ["lab"] +markers = "sys_platform != \"win32\" and sys_platform != \"emscripten\"" files = [ {file = "pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523"}, {file = "pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f"}, @@ -2280,6 +2632,7 @@ version = "0.23" description = "Physical quantities module" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "Pint-0.23-py3-none-any.whl", hash = "sha256:df79b6b5f1beb7ed0cd55d91a0766fc55f972f757a9364e844958c05e8eb66f9"}, {file = "Pint-0.23.tar.gz", hash = "sha256:e1509b91606dbc52527c600a4ef74ffac12fff70688aff20e9072409346ec9b4"}, @@ -2306,6 +2659,7 @@ version = "0.3" description = "Physical units interface to xarray using Pint" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "pint-xarray-0.3.tar.gz", hash = "sha256:3545dfa78bee3f98eba29b8bd17500e3b5cb7c7b03a2c2781c4d4d59b6a82841"}, {file = "pint_xarray-0.3-py3-none-any.whl", hash = "sha256:a7d87c792a2e981cbff464bd1c875e872ef7a0c882a9395cfbc34512b3dcb1ab"}, @@ -2322,6 +2676,7 @@ version = "4.2.1" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.8" +groups = ["main", "dev", "lab"] files = [ {file = "platformdirs-4.2.1-py3-none-any.whl", hash = "sha256:17d5a1161b3fd67b390023cb2d3b026bbd40abde6fdb052dfbd3a29c3ba22ee1"}, {file = "platformdirs-4.2.1.tar.gz", hash = "sha256:031cd18d4ec63ec53e82dceaac0417d218a6863f7745dfcc9efe7793b7039bdf"}, @@ -2338,6 +2693,7 @@ version = "1.5.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, @@ -2353,6 +2709,7 @@ version = "3.7.0" description = "A framework for managing and maintaining multi-language pre-commit hooks." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "pre_commit-3.7.0-py2.py3-none-any.whl", hash = "sha256:5eae9e10c2b5ac51577c3452ec0a490455c45a0533f7960f993a0d01e59decab"}, {file = "pre_commit-3.7.0.tar.gz", hash = "sha256:e209d61b8acdcf742404408531f0c37d49d2c734fd7cff2d6076083d191cb060"}, @@ -2371,6 +2728,7 @@ version = "0.20.0" description = "Python client for the Prometheus monitoring system." optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "prometheus_client-0.20.0-py3-none-any.whl", hash = "sha256:cde524a85bce83ca359cc837f28b8c0db5cac7aa653a588fd7e84ba061c329e7"}, {file = "prometheus_client-0.20.0.tar.gz", hash = "sha256:287629d00b147a32dcb2be0b9df905da599b2d82f80377083ec8463309a4bb89"}, @@ -2385,6 +2743,7 @@ version = "3.0.43" description = "Library for building powerful interactive command lines in Python" optional = false python-versions = ">=3.7.0" +groups = ["lab"] files = [ {file = "prompt_toolkit-3.0.43-py3-none-any.whl", hash = "sha256:a11a29cb3bf0a28a387fe5122cdb649816a957cd9261dcedf8c9f1fef33eacf6"}, {file = "prompt_toolkit-3.0.43.tar.gz", hash = "sha256:3527b7af26106cbc65a040bcc84839a3566ec1b051bb0bfe953631e704b0ff7d"}, @@ -2399,6 +2758,7 @@ version = "5.9.8" description = "Cross-platform lib for process and system monitoring in Python." optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +groups = ["lab"] files = [ {file = "psutil-5.9.8-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:26bd09967ae00920df88e0352a91cff1a78f8d69b3ecabbfe733610c0af486c8"}, {file = "psutil-5.9.8-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:05806de88103b25903dff19bb6692bd2e714ccf9e668d050d144012055cbca73"}, @@ -2419,7 +2779,7 @@ files = [ ] [package.extras] -test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] +test = ["enum34 ; python_version <= \"3.4\"", "ipaddress ; python_version < \"3.0\"", "mock ; python_version < \"3.0\"", "pywin32 ; sys_platform == \"win32\"", "wmi ; sys_platform == \"win32\""] [[package]] name = "ptyprocess" @@ -2427,6 +2787,8 @@ version = "0.7.0" description = "Run a subprocess in a pseudo terminal" optional = false python-versions = "*" +groups = ["lab"] +markers = "sys_platform != \"win32\" and sys_platform != \"emscripten\" or os_name != \"nt\"" files = [ {file = "ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35"}, {file = "ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"}, @@ -2438,6 +2800,7 @@ version = "0.2.2" description = "Safely evaluate AST nodes without side effects" optional = false python-versions = "*" +groups = ["lab"] files = [ {file = "pure_eval-0.2.2-py3-none-any.whl", hash = "sha256:01eaab343580944bc56080ebe0a674b39ec44a945e6d09ba7db3cb8cec289350"}, {file = "pure_eval-0.2.2.tar.gz", hash = "sha256:2b45320af6dfaa1750f543d714b6d1c520a1688dec6fd24d339063ce0aaa9ac3"}, @@ -2448,13 +2811,14 @@ tests = ["pytest"] [[package]] name = "pycodestyle" -version = "2.11.1" +version = "2.12.1" description = "Python style guide checker" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ - {file = "pycodestyle-2.11.1-py2.py3-none-any.whl", hash = "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67"}, - {file = "pycodestyle-2.11.1.tar.gz", hash = "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f"}, + {file = "pycodestyle-2.12.1-py2.py3-none-any.whl", hash = "sha256:46f0fb92069a7c28ab7bb558f05bfc0110dac69a0cd23c61ea0040283a9d78b3"}, + {file = "pycodestyle-2.12.1.tar.gz", hash = "sha256:6838eae08bbce4f6accd5d5572075c63626a15ee3e6f842df996bf62f6d73521"}, ] [[package]] @@ -2463,137 +2827,190 @@ version = "2.22" description = "C parser in Python" optional = false python-versions = ">=3.8" +groups = ["main", "lab"] files = [ {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, ] +markers = {main = "platform_python_implementation != \"PyPy\""} [[package]] name = "pydantic" -version = "2.7.1" +version = "2.12.5" description = "Data validation using Python type hints" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["main", "dev"] files = [ - {file = "pydantic-2.7.1-py3-none-any.whl", hash = "sha256:e029badca45266732a9a79898a15ae2e8b14840b1eabbb25844be28f0b33f3d5"}, - {file = "pydantic-2.7.1.tar.gz", hash = "sha256:e9dbb5eada8abe4d9ae5f46b9939aead650cd2b68f249bb3a8139dbe125803cc"}, + {file = "pydantic-2.12.5-py3-none-any.whl", hash = "sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d"}, + {file = "pydantic-2.12.5.tar.gz", hash = "sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49"}, ] [package.dependencies] -annotated-types = ">=0.4.0" -pydantic-core = "2.18.2" -typing-extensions = ">=4.6.1" +annotated-types = ">=0.6.0" +pydantic-core = "2.41.5" +typing-extensions = ">=4.14.1" +typing-inspection = ">=0.4.2" [package.extras] email = ["email-validator (>=2.0.0)"] +timezone = ["tzdata ; python_version >= \"3.9\" and platform_system == \"Windows\""] [[package]] name = "pydantic-core" -version = "2.18.2" +version = "2.41.5" description = "Core functionality for Pydantic validation and serialization" optional = false -python-versions = ">=3.8" -files = [ - {file = "pydantic_core-2.18.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:9e08e867b306f525802df7cd16c44ff5ebbe747ff0ca6cf3fde7f36c05a59a81"}, - {file = "pydantic_core-2.18.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f0a21cbaa69900cbe1a2e7cad2aa74ac3cf21b10c3efb0fa0b80305274c0e8a2"}, - {file = "pydantic_core-2.18.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0680b1f1f11fda801397de52c36ce38ef1c1dc841a0927a94f226dea29c3ae3d"}, - {file = "pydantic_core-2.18.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:95b9d5e72481d3780ba3442eac863eae92ae43a5f3adb5b4d0a1de89d42bb250"}, - {file = "pydantic_core-2.18.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4fcf5cd9c4b655ad666ca332b9a081112cd7a58a8b5a6ca7a3104bc950f2038"}, - {file = "pydantic_core-2.18.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b5155ff768083cb1d62f3e143b49a8a3432e6789a3abee8acd005c3c7af1c74"}, - {file = "pydantic_core-2.18.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:553ef617b6836fc7e4df130bb851e32fe357ce36336d897fd6646d6058d980af"}, - {file = "pydantic_core-2.18.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b89ed9eb7d616ef5714e5590e6cf7f23b02d0d539767d33561e3675d6f9e3857"}, - {file = "pydantic_core-2.18.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:75f7e9488238e920ab6204399ded280dc4c307d034f3924cd7f90a38b1829563"}, - {file = "pydantic_core-2.18.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ef26c9e94a8c04a1b2924149a9cb081836913818e55681722d7f29af88fe7b38"}, - {file = "pydantic_core-2.18.2-cp310-none-win32.whl", hash = "sha256:182245ff6b0039e82b6bb585ed55a64d7c81c560715d1bad0cbad6dfa07b4027"}, - {file = "pydantic_core-2.18.2-cp310-none-win_amd64.whl", hash = "sha256:e23ec367a948b6d812301afc1b13f8094ab7b2c280af66ef450efc357d2ae543"}, - {file = "pydantic_core-2.18.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:219da3f096d50a157f33645a1cf31c0ad1fe829a92181dd1311022f986e5fbe3"}, - {file = "pydantic_core-2.18.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cc1cfd88a64e012b74e94cd00bbe0f9c6df57049c97f02bb07d39e9c852e19a4"}, - {file = "pydantic_core-2.18.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:05b7133a6e6aeb8df37d6f413f7705a37ab4031597f64ab56384c94d98fa0e90"}, - {file = "pydantic_core-2.18.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:224c421235f6102e8737032483f43c1a8cfb1d2f45740c44166219599358c2cd"}, - {file = "pydantic_core-2.18.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b14d82cdb934e99dda6d9d60dc84a24379820176cc4a0d123f88df319ae9c150"}, - {file = "pydantic_core-2.18.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2728b01246a3bba6de144f9e3115b532ee44bd6cf39795194fb75491824a1413"}, - {file = "pydantic_core-2.18.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:470b94480bb5ee929f5acba6995251ada5e059a5ef3e0dfc63cca287283ebfa6"}, - {file = "pydantic_core-2.18.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:997abc4df705d1295a42f95b4eec4950a37ad8ae46d913caeee117b6b198811c"}, - {file = "pydantic_core-2.18.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:75250dbc5290e3f1a0f4618db35e51a165186f9034eff158f3d490b3fed9f8a0"}, - {file = "pydantic_core-2.18.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4456f2dca97c425231d7315737d45239b2b51a50dc2b6f0c2bb181fce6207664"}, - {file = "pydantic_core-2.18.2-cp311-none-win32.whl", hash = "sha256:269322dcc3d8bdb69f054681edff86276b2ff972447863cf34c8b860f5188e2e"}, - {file = "pydantic_core-2.18.2-cp311-none-win_amd64.whl", hash = "sha256:800d60565aec896f25bc3cfa56d2277d52d5182af08162f7954f938c06dc4ee3"}, - {file = "pydantic_core-2.18.2-cp311-none-win_arm64.whl", hash = "sha256:1404c69d6a676245199767ba4f633cce5f4ad4181f9d0ccb0577e1f66cf4c46d"}, - {file = "pydantic_core-2.18.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:fb2bd7be70c0fe4dfd32c951bc813d9fe6ebcbfdd15a07527796c8204bd36242"}, - {file = "pydantic_core-2.18.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6132dd3bd52838acddca05a72aafb6eab6536aa145e923bb50f45e78b7251043"}, - {file = "pydantic_core-2.18.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7d904828195733c183d20a54230c0df0eb46ec746ea1a666730787353e87182"}, - {file = "pydantic_core-2.18.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c9bd70772c720142be1020eac55f8143a34ec9f82d75a8e7a07852023e46617f"}, - {file = "pydantic_core-2.18.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2b8ed04b3582771764538f7ee7001b02e1170223cf9b75dff0bc698fadb00cf3"}, - {file = "pydantic_core-2.18.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e6dac87ddb34aaec85f873d737e9d06a3555a1cc1a8e0c44b7f8d5daeb89d86f"}, - {file = "pydantic_core-2.18.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ca4ae5a27ad7a4ee5170aebce1574b375de390bc01284f87b18d43a3984df72"}, - {file = "pydantic_core-2.18.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:886eec03591b7cf058467a70a87733b35f44707bd86cf64a615584fd72488b7c"}, - {file = "pydantic_core-2.18.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ca7b0c1f1c983e064caa85f3792dd2fe3526b3505378874afa84baf662e12241"}, - {file = "pydantic_core-2.18.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4b4356d3538c3649337df4074e81b85f0616b79731fe22dd11b99499b2ebbdf3"}, - {file = "pydantic_core-2.18.2-cp312-none-win32.whl", hash = "sha256:8b172601454f2d7701121bbec3425dd71efcb787a027edf49724c9cefc14c038"}, - {file = "pydantic_core-2.18.2-cp312-none-win_amd64.whl", hash = "sha256:b1bd7e47b1558ea872bd16c8502c414f9e90dcf12f1395129d7bb42a09a95438"}, - {file = "pydantic_core-2.18.2-cp312-none-win_arm64.whl", hash = "sha256:98758d627ff397e752bc339272c14c98199c613f922d4a384ddc07526c86a2ec"}, - {file = "pydantic_core-2.18.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:9fdad8e35f278b2c3eb77cbdc5c0a49dada440657bf738d6905ce106dc1de439"}, - {file = "pydantic_core-2.18.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1d90c3265ae107f91a4f279f4d6f6f1d4907ac76c6868b27dc7fb33688cfb347"}, - {file = "pydantic_core-2.18.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:390193c770399861d8df9670fb0d1874f330c79caaca4642332df7c682bf6b91"}, - {file = "pydantic_core-2.18.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:82d5d4d78e4448683cb467897fe24e2b74bb7b973a541ea1dcfec1d3cbce39fb"}, - {file = "pydantic_core-2.18.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4774f3184d2ef3e14e8693194f661dea5a4d6ca4e3dc8e39786d33a94865cefd"}, - {file = "pydantic_core-2.18.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4d938ec0adf5167cb335acb25a4ee69a8107e4984f8fbd2e897021d9e4ca21b"}, - {file = "pydantic_core-2.18.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e0e8b1be28239fc64a88a8189d1df7fad8be8c1ae47fcc33e43d4be15f99cc70"}, - {file = "pydantic_core-2.18.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:868649da93e5a3d5eacc2b5b3b9235c98ccdbfd443832f31e075f54419e1b96b"}, - {file = "pydantic_core-2.18.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:78363590ef93d5d226ba21a90a03ea89a20738ee5b7da83d771d283fd8a56761"}, - {file = "pydantic_core-2.18.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:852e966fbd035a6468fc0a3496589b45e2208ec7ca95c26470a54daed82a0788"}, - {file = "pydantic_core-2.18.2-cp38-none-win32.whl", hash = "sha256:6a46e22a707e7ad4484ac9ee9f290f9d501df45954184e23fc29408dfad61350"}, - {file = "pydantic_core-2.18.2-cp38-none-win_amd64.whl", hash = "sha256:d91cb5ea8b11607cc757675051f61b3d93f15eca3cefb3e6c704a5d6e8440f4e"}, - {file = "pydantic_core-2.18.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:ae0a8a797a5e56c053610fa7be147993fe50960fa43609ff2a9552b0e07013e8"}, - {file = "pydantic_core-2.18.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:042473b6280246b1dbf530559246f6842b56119c2926d1e52b631bdc46075f2a"}, - {file = "pydantic_core-2.18.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a388a77e629b9ec814c1b1e6b3b595fe521d2cdc625fcca26fbc2d44c816804"}, - {file = "pydantic_core-2.18.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e25add29b8f3b233ae90ccef2d902d0ae0432eb0d45370fe315d1a5cf231004b"}, - {file = "pydantic_core-2.18.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f459a5ce8434614dfd39bbebf1041952ae01da6bed9855008cb33b875cb024c0"}, - {file = "pydantic_core-2.18.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eff2de745698eb46eeb51193a9f41d67d834d50e424aef27df2fcdee1b153845"}, - {file = "pydantic_core-2.18.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8309f67285bdfe65c372ea3722b7a5642680f3dba538566340a9d36e920b5f0"}, - {file = "pydantic_core-2.18.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f93a8a2e3938ff656a7c1bc57193b1319960ac015b6e87d76c76bf14fe0244b4"}, - {file = "pydantic_core-2.18.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:22057013c8c1e272eb8d0eebc796701167d8377441ec894a8fed1af64a0bf399"}, - {file = "pydantic_core-2.18.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:cfeecd1ac6cc1fb2692c3d5110781c965aabd4ec5d32799773ca7b1456ac636b"}, - {file = "pydantic_core-2.18.2-cp39-none-win32.whl", hash = "sha256:0d69b4c2f6bb3e130dba60d34c0845ba31b69babdd3f78f7c0c8fae5021a253e"}, - {file = "pydantic_core-2.18.2-cp39-none-win_amd64.whl", hash = "sha256:d9319e499827271b09b4e411905b24a426b8fb69464dfa1696258f53a3334641"}, - {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a1874c6dd4113308bd0eb568418e6114b252afe44319ead2b4081e9b9521fe75"}, - {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:ccdd111c03bfd3666bd2472b674c6899550e09e9f298954cfc896ab92b5b0e6d"}, - {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e18609ceaa6eed63753037fc06ebb16041d17d28199ae5aba0052c51449650a9"}, - {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e5c584d357c4e2baf0ff7baf44f4994be121e16a2c88918a5817331fc7599d7"}, - {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:43f0f463cf89ace478de71a318b1b4f05ebc456a9b9300d027b4b57c1a2064fb"}, - {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:e1b395e58b10b73b07b7cf740d728dd4ff9365ac46c18751bf8b3d8cca8f625a"}, - {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:0098300eebb1c837271d3d1a2cd2911e7c11b396eac9661655ee524a7f10587b"}, - {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:36789b70d613fbac0a25bb07ab3d9dba4d2e38af609c020cf4d888d165ee0bf3"}, - {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3f9a801e7c8f1ef8718da265bba008fa121243dfe37c1cea17840b0944dfd72c"}, - {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:3a6515ebc6e69d85502b4951d89131ca4e036078ea35533bb76327f8424531ce"}, - {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20aca1e2298c56ececfd8ed159ae4dde2df0781988c97ef77d5c16ff4bd5b400"}, - {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:223ee893d77a310a0391dca6df00f70bbc2f36a71a895cecd9a0e762dc37b349"}, - {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2334ce8c673ee93a1d6a65bd90327588387ba073c17e61bf19b4fd97d688d63c"}, - {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:cbca948f2d14b09d20268cda7b0367723d79063f26c4ffc523af9042cad95592"}, - {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:b3ef08e20ec49e02d5c6717a91bb5af9b20f1805583cb0adfe9ba2c6b505b5ae"}, - {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:c6fdc8627910eed0c01aed6a390a252fe3ea6d472ee70fdde56273f198938374"}, - {file = "pydantic_core-2.18.2.tar.gz", hash = "sha256:2e29d20810dfc3043ee13ac7d9e25105799817683348823f305ab3f349b9386e"}, -] - -[package.dependencies] -typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" +python-versions = ">=3.9" +groups = ["main", "dev"] +files = [ + {file = "pydantic_core-2.41.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:77b63866ca88d804225eaa4af3e664c5faf3568cea95360d21f4725ab6e07146"}, + {file = "pydantic_core-2.41.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dfa8a0c812ac681395907e71e1274819dec685fec28273a28905df579ef137e2"}, + {file = "pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5921a4d3ca3aee735d9fd163808f5e8dd6c6972101e4adbda9a4667908849b97"}, + {file = "pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e25c479382d26a2a41b7ebea1043564a937db462816ea07afa8a44c0866d52f9"}, + {file = "pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f547144f2966e1e16ae626d8ce72b4cfa0caedc7fa28052001c94fb2fcaa1c52"}, + {file = "pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f52298fbd394f9ed112d56f3d11aabd0d5bd27beb3084cc3d8ad069483b8941"}, + {file = "pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:100baa204bb412b74fe285fb0f3a385256dad1d1879f0a5cb1499ed2e83d132a"}, + {file = "pydantic_core-2.41.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:05a2c8852530ad2812cb7914dc61a1125dc4e06252ee98e5638a12da6cc6fb6c"}, + {file = "pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:29452c56df2ed968d18d7e21f4ab0ac55e71dc59524872f6fc57dcf4a3249ed2"}, + {file = "pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:d5160812ea7a8a2ffbe233d8da666880cad0cbaf5d4de74ae15c313213d62556"}, + {file = "pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:df3959765b553b9440adfd3c795617c352154e497a4eaf3752555cfb5da8fc49"}, + {file = "pydantic_core-2.41.5-cp310-cp310-win32.whl", hash = "sha256:1f8d33a7f4d5a7889e60dc39856d76d09333d8a6ed0f5f1190635cbec70ec4ba"}, + {file = "pydantic_core-2.41.5-cp310-cp310-win_amd64.whl", hash = "sha256:62de39db01b8d593e45871af2af9e497295db8d73b085f6bfd0b18c83c70a8f9"}, + {file = "pydantic_core-2.41.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6"}, + {file = "pydantic_core-2.41.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b"}, + {file = "pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a"}, + {file = "pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b576130c69225432866fe2f4a469a85a54ade141d96fd396dffcf607b558f8"}, + {file = "pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6cb58b9c66f7e4179a2d5e0f849c48eff5c1fca560994d6eb6543abf955a149e"}, + {file = "pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:88942d3a3dff3afc8288c21e565e476fc278902ae4d6d134f1eeda118cc830b1"}, + {file = "pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f31d95a179f8d64d90f6831d71fa93290893a33148d890ba15de25642c5d075b"}, + {file = "pydantic_core-2.41.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c1df3d34aced70add6f867a8cf413e299177e0c22660cc767218373d0779487b"}, + {file = "pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4009935984bd36bd2c774e13f9a09563ce8de4abaa7226f5108262fa3e637284"}, + {file = "pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:34a64bc3441dc1213096a20fe27e8e128bd3ff89921706e83c0b1ac971276594"}, + {file = "pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c9e19dd6e28fdcaa5a1de679aec4141f691023916427ef9bae8584f9c2fb3b0e"}, + {file = "pydantic_core-2.41.5-cp311-cp311-win32.whl", hash = "sha256:2c010c6ded393148374c0f6f0bf89d206bf3217f201faa0635dcd56bd1520f6b"}, + {file = "pydantic_core-2.41.5-cp311-cp311-win_amd64.whl", hash = "sha256:76ee27c6e9c7f16f47db7a94157112a2f3a00e958bc626e2f4ee8bec5c328fbe"}, + {file = "pydantic_core-2.41.5-cp311-cp311-win_arm64.whl", hash = "sha256:4bc36bbc0b7584de96561184ad7f012478987882ebf9f9c389b23f432ea3d90f"}, + {file = "pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7"}, + {file = "pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0"}, + {file = "pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69"}, + {file = "pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75"}, + {file = "pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05"}, + {file = "pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc"}, + {file = "pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c"}, + {file = "pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5"}, + {file = "pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c"}, + {file = "pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294"}, + {file = "pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1"}, + {file = "pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d"}, + {file = "pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815"}, + {file = "pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3"}, + {file = "pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9"}, + {file = "pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34"}, + {file = "pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0"}, + {file = "pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33"}, + {file = "pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e"}, + {file = "pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2"}, + {file = "pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586"}, + {file = "pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d"}, + {file = "pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740"}, + {file = "pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e"}, + {file = "pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858"}, + {file = "pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36"}, + {file = "pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11"}, + {file = "pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd"}, + {file = "pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a"}, + {file = "pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14"}, + {file = "pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1"}, + {file = "pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66"}, + {file = "pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869"}, + {file = "pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2"}, + {file = "pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375"}, + {file = "pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553"}, + {file = "pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90"}, + {file = "pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07"}, + {file = "pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb"}, + {file = "pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23"}, + {file = "pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf"}, + {file = "pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c"}, + {file = "pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008"}, + {file = "pydantic_core-2.41.5-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:8bfeaf8735be79f225f3fefab7f941c712aaca36f1128c9d7e2352ee1aa87bdf"}, + {file = "pydantic_core-2.41.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:346285d28e4c8017da95144c7f3acd42740d637ff41946af5ce6e5e420502dd5"}, + {file = "pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a75dafbf87d6276ddc5b2bf6fae5254e3d0876b626eb24969a574fff9149ee5d"}, + {file = "pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7b93a4d08587e2b7e7882de461e82b6ed76d9026ce91ca7915e740ecc7855f60"}, + {file = "pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e8465ab91a4bd96d36dde3263f06caa6a8a6019e4113f24dc753d79a8b3a3f82"}, + {file = "pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:299e0a22e7ae2b85c1a57f104538b2656e8ab1873511fd718a1c1c6f149b77b5"}, + {file = "pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:707625ef0983fcfb461acfaf14de2067c5942c6bb0f3b4c99158bed6fedd3cf3"}, + {file = "pydantic_core-2.41.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f41eb9797986d6ebac5e8edff36d5cef9de40def462311b3eb3eeded1431e425"}, + {file = "pydantic_core-2.41.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0384e2e1021894b1ff5a786dbf94771e2986ebe2869533874d7e43bc79c6f504"}, + {file = "pydantic_core-2.41.5-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:f0cd744688278965817fd0839c4a4116add48d23890d468bc436f78beb28abf5"}, + {file = "pydantic_core-2.41.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:753e230374206729bf0a807954bcc6c150d3743928a73faffee51ac6557a03c3"}, + {file = "pydantic_core-2.41.5-cp39-cp39-win32.whl", hash = "sha256:873e0d5b4fb9b89ef7c2d2a963ea7d02879d9da0da8d9d4933dee8ee86a8b460"}, + {file = "pydantic_core-2.41.5-cp39-cp39-win_amd64.whl", hash = "sha256:e4f4a984405e91527a0d62649ee21138f8e3d0ef103be488c1dc11a80d7f184b"}, + {file = "pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:b96d5f26b05d03cc60f11a7761a5ded1741da411e7fe0909e27a5e6a0cb7b034"}, + {file = "pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:634e8609e89ceecea15e2d61bc9ac3718caaaa71963717bf3c8f38bfde64242c"}, + {file = "pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:93e8740d7503eb008aa2df04d3b9735f845d43ae845e6dcd2be0b55a2da43cd2"}, + {file = "pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f15489ba13d61f670dcc96772e733aad1a6f9c429cc27574c6cdaed82d0146ad"}, + {file = "pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd"}, + {file = "pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc"}, + {file = "pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56"}, + {file = "pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b"}, + {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b5819cd790dbf0c5eb9f82c73c16b39a65dd6dd4d1439dcdea7816ec9adddab8"}, + {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:5a4e67afbc95fa5c34cf27d9089bca7fcab4e51e57278d710320a70b956d1b9a"}, + {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ece5c59f0ce7d001e017643d8d24da587ea1f74f6993467d85ae8a5ef9d4f42b"}, + {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:16f80f7abe3351f8ea6858914ddc8c77e02578544a0ebc15b4c2e1a0e813b0b2"}, + {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:33cb885e759a705b426baada1fe68cbb0a2e68e34c5d0d0289a364cf01709093"}, + {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:c8d8b4eb992936023be7dee581270af5c6e0697a8559895f527f5b7105ecd36a"}, + {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:242a206cd0318f95cd21bdacff3fcc3aab23e79bba5cac3db5a841c9ef9c6963"}, + {file = "pydantic_core-2.41.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d3a978c4f57a597908b7e697229d996d77a6d3c94901e9edee593adada95ce1a"}, + {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26"}, + {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808"}, + {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc"}, + {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:287dad91cfb551c363dc62899a80e9e14da1f0e2b6ebde82c806612ca2a13ef1"}, + {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:03b77d184b9eb40240ae9fd676ca364ce1085f203e1b1256f8ab9984dca80a84"}, + {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:a668ce24de96165bb239160b3d854943128f4334822900534f2fe947930e5770"}, + {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f14f8f046c14563f8eb3f45f499cc658ab8d10072961e07225e507adb700e93f"}, + {file = "pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51"}, + {file = "pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e"}, +] + +[package.dependencies] +typing-extensions = ">=4.14.1" [[package]] name = "pydantic-settings" -version = "2.2.1" +version = "2.13.1" description = "Settings management using Pydantic" optional = false -python-versions = ">=3.8" +python-versions = ">=3.10" +groups = ["main", "dev"] files = [ - {file = "pydantic_settings-2.2.1-py3-none-any.whl", hash = "sha256:0235391d26db4d2190cb9b31051c4b46882d28a51533f97440867f012d4da091"}, - {file = "pydantic_settings-2.2.1.tar.gz", hash = "sha256:00b9f6a5e95553590434c0fa01ead0b216c3e10bc54ae02e37f359948643c5ed"}, + {file = "pydantic_settings-2.13.1-py3-none-any.whl", hash = "sha256:d56fd801823dbeae7f0975e1f8c8e25c258eb75d278ea7abb5d9cebb01b56237"}, + {file = "pydantic_settings-2.13.1.tar.gz", hash = "sha256:b4c11847b15237fb0171e1462bf540e294affb9b86db4d9aa5c01730bdbe4025"}, ] [package.dependencies] -pydantic = ">=2.3.0" +pydantic = ">=2.7.0" python-dotenv = ">=0.21.0" +typing-inspection = ">=0.4.0" [package.extras] +aws-secrets-manager = ["boto3 (>=1.35.0)", "boto3-stubs[secretsmanager]"] +azure-key-vault = ["azure-identity (>=1.16.0)", "azure-keyvault-secrets (>=4.8.0)"] +gcp-secret-manager = ["google-cloud-secret-manager (>=2.23.1)"] toml = ["tomli (>=2.0.1)"] yaml = ["pyyaml (>=6.0.1)"] @@ -2603,6 +3020,7 @@ version = "3.2.0" description = "passive checker of Python programs" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "pyflakes-3.2.0-py2.py3-none-any.whl", hash = "sha256:84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a"}, {file = "pyflakes-3.2.0.tar.gz", hash = "sha256:1c61603ff154621fb2a9172037d84dca3500def8c8b630657d1701f026f8af3f"}, @@ -2614,13 +3032,14 @@ version = "2.17.2" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.7" +groups = ["main", "dev", "lab"] files = [ {file = "pygments-2.17.2-py3-none-any.whl", hash = "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c"}, {file = "pygments-2.17.2.tar.gz", hash = "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"}, ] [package.extras] -plugins = ["importlib-metadata"] +plugins = ["importlib-metadata ; python_version < \"3.8\""] windows-terminal = ["colorama (>=0.4.6)"] [[package]] @@ -2629,13 +3048,14 @@ version = "3.1.0" description = "python code static checker" optional = false python-versions = ">=3.8.0" +groups = ["dev"] files = [ {file = "pylint-3.1.0-py3-none-any.whl", hash = "sha256:507a5b60953874766d8a366e8e8c7af63e058b26345cfcb5f91f89d987fd6b74"}, {file = "pylint-3.1.0.tar.gz", hash = "sha256:6a69beb4a6f63debebaab0a3477ecd0f559aa726af4954fc948c51f7a2549e23"}, ] [package.dependencies] -astroid = ">=3.1.0,<=3.2.0-dev0" +astroid = ">=3.1.0,<=3.2.0.dev0" colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} dill = [ {version = ">=0.2", markers = "python_version < \"3.11\""}, @@ -2657,6 +3077,7 @@ version = "24.1.0" description = "Python wrapper module around the OpenSSL library" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "pyOpenSSL-24.1.0-py3-none-any.whl", hash = "sha256:17ed5be5936449c5418d1cd269a1a9e9081bc54c17aed272b45856a3d3dc86ad"}, {file = "pyOpenSSL-24.1.0.tar.gz", hash = "sha256:cabed4bfaa5df9f1a16c0ef64a0cb65318b5cd077a7eda7d6970131ca2f41a6f"}, @@ -2669,12 +3090,28 @@ cryptography = ">=41.0.5,<43" docs = ["sphinx (!=5.2.0,!=5.2.0.post0,!=7.2.5)", "sphinx-rtd-theme"] test = ["pretend", "pytest (>=3.0.1)", "pytest-rerunfailures"] +[[package]] +name = "pyparsing" +version = "3.3.2" +description = "pyparsing - Classes and methods to define and execute parsing grammars" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d"}, + {file = "pyparsing-3.3.2.tar.gz", hash = "sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc"}, +] + +[package.extras] +diagrams = ["jinja2", "railroad-diagrams"] + [[package]] name = "pytest" version = "7.4.4" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, @@ -2697,6 +3134,7 @@ version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main", "lab"] files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -2711,6 +3149,7 @@ version = "1.0.1" description = "Read key-value pairs from a .env file and set them as environment variables" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca"}, {file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"}, @@ -2725,6 +3164,7 @@ version = "2.0.7" description = "A python library adding a json log formatter" optional = false python-versions = ">=3.6" +groups = ["lab"] files = [ {file = "python-json-logger-2.0.7.tar.gz", hash = "sha256:23e7ec02d34237c5aa1e29a070193a4ea87583bb4e7f8fd06d3de8264c4b2e1c"}, {file = "python_json_logger-2.0.7-py3-none-any.whl", hash = "sha256:f380b826a991ebbe3de4d897aeec42760035ac760345e57b812938dc8b35e2bd"}, @@ -2736,6 +3176,7 @@ version = "2024.1" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, @@ -2747,6 +3188,8 @@ version = "306" description = "Python for Window Extensions" optional = false python-versions = "*" +groups = ["lab"] +markers = "sys_platform == \"win32\" and platform_python_implementation != \"PyPy\"" files = [ {file = "pywin32-306-cp310-cp310-win32.whl", hash = "sha256:06d3420a5155ba65f0b72f2699b5bacf3109f36acbe8923765c22938a69dfc8d"}, {file = "pywin32-306-cp310-cp310-win_amd64.whl", hash = "sha256:84f4471dbca1887ea3803d8848a1616429ac94a4a8d05f4bc9c5dcfd42ca99c8"}, @@ -2770,6 +3213,8 @@ version = "2.0.13" description = "Pseudo terminal support for Windows from Python." optional = false python-versions = ">=3.8" +groups = ["lab"] +markers = "os_name == \"nt\"" files = [ {file = "pywinpty-2.0.13-cp310-none-win_amd64.whl", hash = "sha256:697bff211fb5a6508fee2dc6ff174ce03f34a9a233df9d8b5fe9c8ce4d5eaf56"}, {file = "pywinpty-2.0.13-cp311-none-win_amd64.whl", hash = "sha256:b96fb14698db1284db84ca38c79f15b4cfdc3172065b5137383910567591fa99"}, @@ -2785,6 +3230,7 @@ version = "6.0.1" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.6" +groups = ["main", "dev", "lab"] files = [ {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, @@ -2845,6 +3291,7 @@ version = "26.0.2" description = "Python bindings for 0MQ" optional = false python-versions = ">=3.7" +groups = ["lab"] files = [ {file = "pyzmq-26.0.2-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:1a60a03b01e8c9c58932ec0cca15b1712d911c2800eb82d4281bc1ae5b6dad50"}, {file = "pyzmq-26.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:949067079e14ea1973bd740255e0840118c163d4bce8837f539d749f145cf5c3"}, @@ -2945,6 +3392,7 @@ version = "0.35.0" description = "JSON Referencing + Python" optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "referencing-0.35.0-py3-none-any.whl", hash = "sha256:8080727b30e364e5783152903672df9b6b091c926a146a759080b62ca3126cd6"}, {file = "referencing-0.35.0.tar.gz", hash = "sha256:191e936b0c696d0af17ad7430a3dc68e88bc11be6514f4757dc890f04ab05889"}, @@ -2960,6 +3408,7 @@ version = "2.32.2" description = "Python HTTP for Humans." optional = false python-versions = ">=3.8" +groups = ["main", "lab"] files = [ {file = "requests-2.32.2-py3-none-any.whl", hash = "sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c"}, {file = "requests-2.32.2.tar.gz", hash = "sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289"}, @@ -2981,6 +3430,7 @@ version = "1.2.0" description = "A persistent cache for python requests" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "requests_cache-1.2.0-py3-none-any.whl", hash = "sha256:490324301bf0cb924ff4e6324bd2613453e7e1f847353928b08adb0fdfb7f722"}, {file = "requests_cache-1.2.0.tar.gz", hash = "sha256:db1c709ca343cc1cd5b6c8b1a5387298eceed02306a6040760db538c885e3838"}, @@ -3011,6 +3461,7 @@ version = "0.1.4" description = "A pure python RFC3339 validator" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +groups = ["lab"] files = [ {file = "rfc3339_validator-0.1.4-py2.py3-none-any.whl", hash = "sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa"}, {file = "rfc3339_validator-0.1.4.tar.gz", hash = "sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b"}, @@ -3025,6 +3476,7 @@ version = "0.1.1" description = "Pure python rfc3986 validator" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +groups = ["lab"] files = [ {file = "rfc3986_validator-0.1.1-py2.py3-none-any.whl", hash = "sha256:2f235c432ef459970b4306369336b9d5dbdda31b510ca1e327636e01f528bfa9"}, {file = "rfc3986_validator-0.1.1.tar.gz", hash = "sha256:3d44bde7921b3b9ec3ae4e3adca370438eccebc676456449b145d533b240d055"}, @@ -3036,6 +3488,7 @@ version = "13.7.1" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false python-versions = ">=3.7.0" +groups = ["main", "dev"] files = [ {file = "rich-13.7.1-py3-none-any.whl", hash = "sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222"}, {file = "rich-13.7.1.tar.gz", hash = "sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432"}, @@ -3054,6 +3507,7 @@ version = "1.7.4" description = "Format click help output nicely with rich" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "rich-click-1.7.4.tar.gz", hash = "sha256:7ce5de8e4dc0333aec946113529b3eeb349f2e5d2fafee96b9edf8ee36a01395"}, {file = "rich_click-1.7.4-py3-none-any.whl", hash = "sha256:e363655475c60fec5a3e16a1eb618118ed79e666c365a36006b107c17c93ac4e"}, @@ -3073,6 +3527,7 @@ version = "0.18.0" description = "Python bindings to Rust's persistent data structures (rpds)" optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "rpds_py-0.18.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:5b4e7d8d6c9b2e8ee2d55c90b59c707ca59bc30058269b3db7b1f8df5763557e"}, {file = "rpds_py-0.18.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c463ed05f9dfb9baebef68048aed8dcdc94411e4bf3d33a39ba97e271624f8f7"}, @@ -3175,12 +3630,41 @@ files = [ {file = "rpds_py-0.18.0.tar.gz", hash = "sha256:42821446ee7a76f5d9f71f9e33a4fb2ffd724bb3e7f93386150b61a43115788d"}, ] +[[package]] +name = "ruff" +version = "0.11.11" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +groups = ["dev"] +files = [ + {file = "ruff-0.11.11-py3-none-linux_armv6l.whl", hash = "sha256:9924e5ae54125ed8958a4f7de320dab7380f6e9fa3195e3dc3b137c6842a0092"}, + {file = "ruff-0.11.11-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:c8a93276393d91e952f790148eb226658dd275cddfde96c6ca304873f11d2ae4"}, + {file = "ruff-0.11.11-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d6e333dbe2e6ae84cdedefa943dfd6434753ad321764fd937eef9d6b62022bcd"}, + {file = "ruff-0.11.11-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7885d9a5e4c77b24e8c88aba8c80be9255fa22ab326019dac2356cff42089fc6"}, + {file = "ruff-0.11.11-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1b5ab797fcc09121ed82e9b12b6f27e34859e4227080a42d090881be888755d4"}, + {file = "ruff-0.11.11-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e231ff3132c1119ece836487a02785f099a43992b95c2f62847d29bace3c75ac"}, + {file = "ruff-0.11.11-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:a97c9babe1d4081037a90289986925726b802d180cca784ac8da2bbbc335f709"}, + {file = "ruff-0.11.11-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d8c4ddcbe8a19f59f57fd814b8b117d4fcea9bee7c0492e6cf5fdc22cfa563c8"}, + {file = "ruff-0.11.11-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6224076c344a7694c6fbbb70d4f2a7b730f6d47d2a9dc1e7f9d9bb583faf390b"}, + {file = "ruff-0.11.11-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:882821fcdf7ae8db7a951df1903d9cb032bbe838852e5fc3c2b6c3ab54e39875"}, + {file = "ruff-0.11.11-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:dcec2d50756463d9df075a26a85a6affbc1b0148873da3997286caf1ce03cae1"}, + {file = "ruff-0.11.11-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:99c28505ecbaeb6594701a74e395b187ee083ee26478c1a795d35084d53ebd81"}, + {file = "ruff-0.11.11-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9263f9e5aa4ff1dec765e99810f1cc53f0c868c5329b69f13845f699fe74f639"}, + {file = "ruff-0.11.11-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:64ac6f885e3ecb2fdbb71de2701d4e34526651f1e8503af8fb30d4915a3fe345"}, + {file = "ruff-0.11.11-py3-none-win32.whl", hash = "sha256:1adcb9a18802268aaa891ffb67b1c94cd70578f126637118e8099b8e4adcf112"}, + {file = "ruff-0.11.11-py3-none-win_amd64.whl", hash = "sha256:748b4bb245f11e91a04a4ff0f96e386711df0a30412b9fe0c74d5bdc0e4a531f"}, + {file = "ruff-0.11.11-py3-none-win_arm64.whl", hash = "sha256:6c51f136c0364ab1b774767aa8b86331bd8e9d414e2d107db7a2189f35ea1f7b"}, + {file = "ruff-0.11.11.tar.gz", hash = "sha256:7774173cc7c1980e6bf67569ebb7085989a78a103922fb83ef3dfe230cd0687d"}, +] + [[package]] name = "scipy" version = "1.13.1" description = "Fundamental algorithms for scientific computing in Python" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "scipy-1.13.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:20335853b85e9a49ff7572ab453794298bcf0354d8068c5f6775a0eabf350aca"}, {file = "scipy-1.13.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:d605e9c23906d1994f55ace80e0125c587f96c020037ea6aa98d01b4bd2e222f"}, @@ -3223,15 +3707,16 @@ version = "1.8.3" description = "Send file to trash natively under Mac OS X, Windows and Linux" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +groups = ["lab"] files = [ {file = "Send2Trash-1.8.3-py3-none-any.whl", hash = "sha256:0c31227e0bd08961c7665474a3d1ef7193929fedda4233843689baa056be46c9"}, {file = "Send2Trash-1.8.3.tar.gz", hash = "sha256:b18e7a3966d99871aefeb00cfbcfdced55ce4871194810fc71f4aa484b953abf"}, ] [package.extras] -nativelib = ["pyobjc-framework-Cocoa", "pywin32"] -objc = ["pyobjc-framework-Cocoa"] -win32 = ["pywin32"] +nativelib = ["pyobjc-framework-Cocoa ; sys_platform == \"darwin\"", "pywin32 ; sys_platform == \"win32\""] +objc = ["pyobjc-framework-Cocoa ; sys_platform == \"darwin\""] +win32 = ["pywin32 ; sys_platform == \"win32\""] [[package]] name = "setuptools" @@ -3239,6 +3724,7 @@ version = "69.5.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "setuptools-69.5.1-py3-none-any.whl", hash = "sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32"}, {file = "setuptools-69.5.1.tar.gz", hash = "sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987"}, @@ -3246,7 +3732,7 @@ files = [ [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov ; platform_python_implementation != \"PyPy\"", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] @@ -3255,6 +3741,7 @@ version = "2.0.4" description = "Manipulation and analysis of geometric objects" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "shapely-2.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:011b77153906030b795791f2fdfa2d68f1a8d7e40bce78b029782ade3afe4f2f"}, {file = "shapely-2.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9831816a5d34d5170aa9ed32a64982c3d6f4332e7ecfe62dc97767e163cb0b17"}, @@ -3312,6 +3799,7 @@ version = "1.5.4" description = "Tool to Detect Surrounding Shell" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"}, {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, @@ -3323,6 +3811,7 @@ version = "1.16.0" description = "Python 2 and 3 compatibility utilities" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +groups = ["main", "lab"] files = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, @@ -3334,6 +3823,7 @@ version = "1.3.1" description = "Sniff out which async library your code is running under" optional = false python-versions = ">=3.7" +groups = ["main", "lab"] files = [ {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, @@ -3345,6 +3835,7 @@ version = "2.5" description = "A modern CSS selector implementation for Beautiful Soup." optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "soupsieve-2.5-py3-none-any.whl", hash = "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"}, {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"}, @@ -3356,6 +3847,7 @@ version = "0.15.4" description = "Sparse n-dimensional arrays for the PyData ecosystem" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "sparse-0.15.4-py2.py3-none-any.whl", hash = "sha256:76ec76fee2aee82a84eb97155dd530a9644e3b1fdea2406bc4b454698b36d938"}, {file = "sparse-0.15.4.tar.gz", hash = "sha256:d4b1c57d24ff0f64f2fd5b5a95b49b7fb84ed207a26d7d58ce2764dcc5c72b84"}, @@ -3372,12 +3864,115 @@ docs = ["sphinx", "sphinx-rtd-theme"] tests = ["dask[array]", "pre-commit", "pytest (>=3.5)", "pytest-cov"] tox = ["sparse[tests]", "tox"] +[[package]] +name = "sqlalchemy" +version = "2.0.48" +description = "Database Abstraction Library" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "sqlalchemy-2.0.48-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7001dc9d5f6bb4deb756d5928eaefe1930f6f4179da3924cbd95ee0e9f4dce89"}, + {file = "sqlalchemy-2.0.48-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1a89ce07ad2d4b8cfc30bd5889ec40613e028ed80ef47da7d9dd2ce969ad30e0"}, + {file = "sqlalchemy-2.0.48-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10853a53a4a00417a00913d270dddda75815fcb80675874285f41051c094d7dd"}, + {file = "sqlalchemy-2.0.48-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:fac0fa4e4f55f118fd87177dacb1c6522fe39c28d498d259014020fec9164c29"}, + {file = "sqlalchemy-2.0.48-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3713e21ea67bca727eecd4a24bf68bcd414c403faae4989442be60994301ded0"}, + {file = "sqlalchemy-2.0.48-cp310-cp310-win32.whl", hash = "sha256:d404dc897ce10e565d647795861762aa2d06ca3f4a728c5e9a835096c7059018"}, + {file = "sqlalchemy-2.0.48-cp310-cp310-win_amd64.whl", hash = "sha256:841a94c66577661c1f088ac958cd767d7c9bf507698f45afffe7a4017049de76"}, + {file = "sqlalchemy-2.0.48-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b4c575df7368b3b13e0cebf01d4679f9a28ed2ae6c1cd0b1d5beffb6b2007dc"}, + {file = "sqlalchemy-2.0.48-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e83e3f959aaa1c9df95c22c528096d94848a1bc819f5d0ebf7ee3df0ca63db6c"}, + {file = "sqlalchemy-2.0.48-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f7b7243850edd0b8b97043f04748f31de50cf426e939def5c16bedb540698f7"}, + {file = "sqlalchemy-2.0.48-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:82745b03b4043e04600a6b665cb98697c4339b24e34d74b0a2ac0a2488b6f94d"}, + {file = "sqlalchemy-2.0.48-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e5e088bf43f6ee6fec7dbf1ef7ff7774a616c236b5c0cb3e00662dd71a56b571"}, + {file = "sqlalchemy-2.0.48-cp311-cp311-win32.whl", hash = "sha256:9c7d0a77e36b5f4b01ca398482230ab792061d243d715299b44a0b55c89fe617"}, + {file = "sqlalchemy-2.0.48-cp311-cp311-win_amd64.whl", hash = "sha256:583849c743e0e3c9bb7446f5b5addeacedc168d657a69b418063dfdb2d90081c"}, + {file = "sqlalchemy-2.0.48-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:348174f228b99f33ca1f773e85510e08927620caa59ffe7803b37170df30332b"}, + {file = "sqlalchemy-2.0.48-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:53667b5f668991e279d21f94ccfa6e45b4e3f4500e7591ae59a8012d0f010dcb"}, + {file = "sqlalchemy-2.0.48-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34634e196f620c7a61d18d5cf7dc841ca6daa7961aed75d532b7e58b309ac894"}, + {file = "sqlalchemy-2.0.48-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:546572a1793cc35857a2ffa1fe0e58571af1779bcc1ffa7c9fb0839885ed69a9"}, + {file = "sqlalchemy-2.0.48-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:07edba08061bc277bfdc772dd2a1a43978f5a45994dd3ede26391b405c15221e"}, + {file = "sqlalchemy-2.0.48-cp312-cp312-win32.whl", hash = "sha256:908a3fa6908716f803b86896a09a2c4dde5f5ce2bb07aacc71ffebb57986ce99"}, + {file = "sqlalchemy-2.0.48-cp312-cp312-win_amd64.whl", hash = "sha256:68549c403f79a8e25984376480959975212a670405e3913830614432b5daa07a"}, + {file = "sqlalchemy-2.0.48-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e3070c03701037aa418b55d36532ecb8f8446ed0135acb71c678dbdf12f5b6e4"}, + {file = "sqlalchemy-2.0.48-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2645b7d8a738763b664a12a1542c89c940daa55196e8d73e55b169cc5c99f65f"}, + {file = "sqlalchemy-2.0.48-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b19151e76620a412c2ac1c6f977ab1b9fa7ad43140178345136456d5265b32ed"}, + {file = "sqlalchemy-2.0.48-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5b193a7e29fd9fa56e502920dca47dffe60f97c863494946bd698c6058a55658"}, + {file = "sqlalchemy-2.0.48-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:36ac4ddc3d33e852da9cb00ffb08cea62ca05c39711dc67062ca2bb1fae35fd8"}, + {file = "sqlalchemy-2.0.48-cp313-cp313-win32.whl", hash = "sha256:389b984139278f97757ea9b08993e7b9d1142912e046ab7d82b3fbaeb0209131"}, + {file = "sqlalchemy-2.0.48-cp313-cp313-win_amd64.whl", hash = "sha256:d612c976cbc2d17edfcc4c006874b764e85e990c29ce9bd411f926bbfb02b9a2"}, + {file = "sqlalchemy-2.0.48-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:69f5bc24904d3bc3640961cddd2523e361257ef68585d6e364166dfbe8c78fae"}, + {file = "sqlalchemy-2.0.48-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fd08b90d211c086181caed76931ecfa2bdfc83eea3cfccdb0f82abc6c4b876cb"}, + {file = "sqlalchemy-2.0.48-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:1ccd42229aaac2df431562117ac7e667d702e8e44afdb6cf0e50fa3f18160f0b"}, + {file = "sqlalchemy-2.0.48-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f0dcbc588cd5b725162c076eb9119342f6579c7f7f55057bb7e3c6ff27e13121"}, + {file = "sqlalchemy-2.0.48-cp313-cp313t-win32.whl", hash = "sha256:9764014ef5e58aab76220c5664abb5d47d5bc858d9debf821e55cfdd0f128485"}, + {file = "sqlalchemy-2.0.48-cp313-cp313t-win_amd64.whl", hash = "sha256:e2f35b4cccd9ed286ad62e0a3c3ac21e06c02abc60e20aa51a3e305a30f5fa79"}, + {file = "sqlalchemy-2.0.48-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e2d0d88686e3d35a76f3e15a34e8c12d73fc94c1dea1cd55782e695cc14086dd"}, + {file = "sqlalchemy-2.0.48-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49b7bddc1eebf011ea5ab722fdbe67a401caa34a350d278cc7733c0e88fecb1f"}, + {file = "sqlalchemy-2.0.48-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:426c5ca86415d9b8945c7073597e10de9644802e2ff502b8e1f11a7a2642856b"}, + {file = "sqlalchemy-2.0.48-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:288937433bd44e3990e7da2402fabc44a3c6c25d3704da066b85b89a85474ae0"}, + {file = "sqlalchemy-2.0.48-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8183dc57ae7d9edc1346e007e840a9f3d6aa7b7f165203a99e16f447150140d2"}, + {file = "sqlalchemy-2.0.48-cp314-cp314-win32.whl", hash = "sha256:1182437cb2d97988cfea04cf6cdc0b0bb9c74f4d56ec3d08b81e23d621a28cc6"}, + {file = "sqlalchemy-2.0.48-cp314-cp314-win_amd64.whl", hash = "sha256:144921da96c08feb9e2b052c5c5c1d0d151a292c6135623c6b2c041f2a45f9e0"}, + {file = "sqlalchemy-2.0.48-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5aee45fd2c6c0f2b9cdddf48c48535e7471e42d6fb81adfde801da0bd5b93241"}, + {file = "sqlalchemy-2.0.48-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7cddca31edf8b0653090cbb54562ca027c421c58ddde2c0685f49ff56a1690e0"}, + {file = "sqlalchemy-2.0.48-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7a936f1bb23d370b7c8cc079d5fce4c7d18da87a33c6744e51a93b0f9e97e9b3"}, + {file = "sqlalchemy-2.0.48-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e004aa9248e8cb0a5f9b96d003ca7c1c0a5da8decd1066e7b53f59eb8ce7c62b"}, + {file = "sqlalchemy-2.0.48-cp314-cp314t-win32.whl", hash = "sha256:b8438ec5594980d405251451c5b7ea9aa58dda38eb7ac35fb7e4c696712ee24f"}, + {file = "sqlalchemy-2.0.48-cp314-cp314t-win_amd64.whl", hash = "sha256:d854b3970067297f3a7fbd7a4683587134aa9b3877ee15aa29eea478dc68f933"}, + {file = "sqlalchemy-2.0.48-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f8649a14caa5f8a243628b1d61cf530ad9ae4578814ba726816adb1121fc493e"}, + {file = "sqlalchemy-2.0.48-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6bb85c546591569558571aa1b06aba711b26ae62f111e15e56136d69920e1616"}, + {file = "sqlalchemy-2.0.48-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a6b764fb312bd35e47797ad2e63f0d323792837a6ac785a4ca967019357d2bc7"}, + {file = "sqlalchemy-2.0.48-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:7c998f2ace8bf76b453b75dbcca500d4f4b9dd3908c13e89b86289b37784848b"}, + {file = "sqlalchemy-2.0.48-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:d64177f443594c8697369c10e4bbcac70ef558e0f7921a1de7e4a3d1734bcf67"}, + {file = "sqlalchemy-2.0.48-cp38-cp38-win32.whl", hash = "sha256:01f6bbd4308b23240cf7d3ef117557c8fd097ec9549d5d8a52977544e35b40ad"}, + {file = "sqlalchemy-2.0.48-cp38-cp38-win_amd64.whl", hash = "sha256:858e433f12b0e5b3ed2f8da917433b634f4937d0e8793e5cb33c54a1a01df565"}, + {file = "sqlalchemy-2.0.48-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4599a95f9430ae0de82b52ff0d27304fe898c17cb5f4099f7438a51b9998ac77"}, + {file = "sqlalchemy-2.0.48-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f27f9da0a7d22b9f981108fd4b62f8b5743423388915a563e651c20d06c1f457"}, + {file = "sqlalchemy-2.0.48-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d8fcccbbc0c13c13702c471da398b8cd72ba740dca5859f148ae8e0e8e0d3e7e"}, + {file = "sqlalchemy-2.0.48-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:a5b429eb84339f9f05e06083f119ad814e6d85e27ecbdf9c551dfdbb128eaf8a"}, + {file = "sqlalchemy-2.0.48-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:bcb8ebbf2e2c36cfe01a94f2438012c6a9d494cf80f129d9753bcdf33bfc35a6"}, + {file = "sqlalchemy-2.0.48-cp39-cp39-win32.whl", hash = "sha256:e214d546c8ecb5fc22d6e6011746082abf13a9cf46eefb45769c7b31407c97b5"}, + {file = "sqlalchemy-2.0.48-cp39-cp39-win_amd64.whl", hash = "sha256:b8fc3454b4f3bd0a368001d0e968852dad45a873f8b4babd41bc302ec851a099"}, + {file = "sqlalchemy-2.0.48-py3-none-any.whl", hash = "sha256:a66fe406437dd65cacd96a72689a3aaaecaebbcd62d81c5ac1c0fdbeac835096"}, + {file = "sqlalchemy-2.0.48.tar.gz", hash = "sha256:5ca74f37f3369b45e1f6b7b06afb182af1fd5dde009e4ffd831830d98cbe5fe7"}, +] + +[package.dependencies] +greenlet = {version = ">=1", markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\""} +typing-extensions = ">=4.6.0" + +[package.extras] +aiomysql = ["aiomysql (>=0.2.0)", "greenlet (>=1)"] +aioodbc = ["aioodbc", "greenlet (>=1)"] +aiosqlite = ["aiosqlite", "greenlet (>=1)", "typing_extensions (!=3.10.0.1)"] +asyncio = ["greenlet (>=1)"] +asyncmy = ["asyncmy (>=0.2.3,!=0.2.4,!=0.2.6)", "greenlet (>=1)"] +mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2,!=1.1.5,!=1.1.10)"] +mssql = ["pyodbc"] +mssql-pymssql = ["pymssql"] +mssql-pyodbc = ["pyodbc"] +mypy = ["mypy (>=0.910)"] +mysql = ["mysqlclient (>=1.4.0)"] +mysql-connector = ["mysql-connector-python"] +oracle = ["cx_oracle (>=8)"] +oracle-oracledb = ["oracledb (>=1.0.1)"] +postgresql = ["psycopg2 (>=2.7)"] +postgresql-asyncpg = ["asyncpg", "greenlet (>=1)"] +postgresql-pg8000 = ["pg8000 (>=1.29.1)"] +postgresql-psycopg = ["psycopg (>=3.0.7)"] +postgresql-psycopg2binary = ["psycopg2-binary"] +postgresql-psycopg2cffi = ["psycopg2cffi"] +postgresql-psycopgbinary = ["psycopg[binary] (>=3.0.7)"] +pymysql = ["pymysql"] +sqlcipher = ["sqlcipher3_binary"] + [[package]] name = "stack-data" version = "0.6.3" description = "Extract data from python stack frames and tracebacks for informative displays" optional = false python-versions = "*" +groups = ["lab"] files = [ {file = "stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695"}, {file = "stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9"}, @@ -3397,6 +3992,7 @@ version = "0.18.1" description = "Tornado websocket backend for the Xterm.js Javascript terminal emulator library." optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "terminado-0.18.1-py3-none-any.whl", hash = "sha256:a4468e1b37bb318f8a86514f65814e1afc977cf29b3992a4500d9dd305dcceb0"}, {file = "terminado-0.18.1.tar.gz", hash = "sha256:de09f2c4b85de4765f7714688fff57d3e75bad1f909b589fde880460c753fd2e"}, @@ -3418,6 +4014,7 @@ version = "1.3.0" description = "A tiny CSS parser" optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "tinycss2-1.3.0-py3-none-any.whl", hash = "sha256:54a8dbdffb334d536851be0226030e9505965bb2f30f21a4a82c55fb2a80fae7"}, {file = "tinycss2-1.3.0.tar.gz", hash = "sha256:152f9acabd296a8375fbca5b84c961ff95971fcfc32e79550c8df8e29118c54d"}, @@ -3436,6 +4033,8 @@ version = "2.0.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.7" +groups = ["main", "dev", "lab"] +markers = "python_version == \"3.10\"" files = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, @@ -3447,6 +4046,7 @@ version = "0.12.4" description = "Style preserving TOML library" optional = false python-versions = ">=3.7" +groups = ["main", "dev"] files = [ {file = "tomlkit-0.12.4-py3-none-any.whl", hash = "sha256:5cd82d48a3dd89dee1f9d64420aa20ae65cfbd00668d6f094d7578a78efbb77b"}, {file = "tomlkit-0.12.4.tar.gz", hash = "sha256:7ca1cfc12232806517a8515047ba66a19369e71edf2439d0f5824f91032b6cc3"}, @@ -3458,6 +4058,7 @@ version = "0.12.1" description = "List processing tools and functional utilities" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "toolz-0.12.1-py3-none-any.whl", hash = "sha256:d22731364c07d72eea0a0ad45bafb2c2937ab6fd38a3507bf55eae8744aa7d85"}, {file = "toolz-0.12.1.tar.gz", hash = "sha256:ecca342664893f177a13dac0e6b41cbd8ac25a358e5f215316d43e2100224f4d"}, @@ -3469,6 +4070,7 @@ version = "6.4" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." optional = false python-versions = ">= 3.8" +groups = ["lab"] files = [ {file = "tornado-6.4-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:02ccefc7d8211e5a7f9e8bc3f9e5b0ad6262ba2fbb683a6443ecc804e5224ce0"}, {file = "tornado-6.4-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:27787de946a9cffd63ce5814c33f734c627a87072ec7eed71f7fc4417bb16263"}, @@ -3489,6 +4091,7 @@ version = "4.66.4" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "tqdm-4.66.4-py3-none-any.whl", hash = "sha256:b75ca56b413b030bc3f00af51fd2c1a1a5eac6a0c1cca83cbb37a5c52abce644"}, {file = "tqdm-4.66.4.tar.gz", hash = "sha256:e4d936c9de8727928f3be6079590e97d9abfe8d39a590be678eb5919ffc186bb"}, @@ -3509,6 +4112,7 @@ version = "5.14.3" description = "Traitlets Python configuration system" optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "traitlets-5.14.3-py3-none-any.whl", hash = "sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f"}, {file = "traitlets-5.14.3.tar.gz", hash = "sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7"}, @@ -3524,6 +4128,7 @@ version = "0.12.5" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "typer-0.12.5-py3-none-any.whl", hash = "sha256:62fe4e471711b147e3365034133904df3e235698399bc4de2b36c8579298d52b"}, {file = "typer-0.12.5.tar.gz", hash = "sha256:f592f089bedcc8ec1b974125d64851029c3b1af145f04aca64d69410f0c9b722"}, @@ -3541,6 +4146,7 @@ version = "2.9.0.20240316" description = "Typing stubs for python-dateutil" optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "types-python-dateutil-2.9.0.20240316.tar.gz", hash = "sha256:5d2f2e240b86905e40944dd787db6da9263f0deabef1076ddaed797351ec0202"}, {file = "types_python_dateutil-2.9.0.20240316-py3-none-any.whl", hash = "sha256:6b8cb66d960771ce5ff974e9dd45e38facb81718cc1e208b10b1baccbfdbee3b"}, @@ -3548,21 +4154,38 @@ files = [ [[package]] name = "typing-extensions" -version = "4.11.0" -description = "Backported and Experimental Type Hints for Python 3.8+" +version = "4.15.0" +description = "Backported and Experimental Type Hints for Python 3.9+" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["main", "dev", "lab"] files = [ - {file = "typing_extensions-4.11.0-py3-none-any.whl", hash = "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"}, - {file = "typing_extensions-4.11.0.tar.gz", hash = "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0"}, + {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, + {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, ] +[[package]] +name = "typing-inspection" +version = "0.4.2" +description = "Runtime typing introspection tools" +optional = false +python-versions = ">=3.9" +groups = ["main", "dev"] +files = [ + {file = "typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7"}, + {file = "typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464"}, +] + +[package.dependencies] +typing-extensions = ">=4.12.0" + [[package]] name = "tzdata" version = "2024.1" description = "Provider of IANA time zone data" optional = false python-versions = ">=2" +groups = ["main"] files = [ {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, @@ -3574,6 +4197,7 @@ version = "0.1.1" description = "Transforms tokens into original source code (while preserving whitespace)." optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "untokenize-0.1.1.tar.gz", hash = "sha256:3865dbbbb8efb4bb5eaa72f1be7f3e0be00ea8b7f125c69cbd1f5fda926f37a2"}, ] @@ -3584,6 +4208,7 @@ version = "1.3.0" description = "RFC 6570 URI Template Processor" optional = false python-versions = ">=3.7" +groups = ["lab"] files = [ {file = "uri-template-1.3.0.tar.gz", hash = "sha256:0e00f8eb65e18c7de20d595a14336e9f337ead580c70934141624b6d1ffdacc7"}, {file = "uri_template-1.3.0-py3-none-any.whl", hash = "sha256:a44a133ea12d44a0c0f06d7d42a52d71282e77e2f937d8abd5655b8d56fc1363"}, @@ -3598,6 +4223,7 @@ version = "1.4.3" description = "URL normalization for Python" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +groups = ["main"] files = [ {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, @@ -3612,23 +4238,48 @@ version = "2.2.1" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.8" +groups = ["main", "lab"] files = [ {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] +[[package]] +name = "validators" +version = "0.22.0" +description = "Python Data Validation for Humans™" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "validators-0.22.0-py3-none-any.whl", hash = "sha256:61cf7d4a62bbae559f2e54aed3b000cea9ff3e2fdbe463f51179b92c58c9585a"}, + {file = "validators-0.22.0.tar.gz", hash = "sha256:77b2689b172eeeb600d9605ab86194641670cdb73b60afd577142a9397873370"}, +] + +[package.extras] +docs-offline = ["myst-parser (>=2.0.0)", "pypandoc-binary (>=1.11)", "sphinx (>=7.1.1)"] +docs-online = ["mkdocs (>=1.5.2)", "mkdocs-git-revision-date-localized-plugin (>=1.2.0)", "mkdocs-material (>=9.2.6)", "mkdocstrings[python] (>=0.22.0)", "pyaml (>=23.7.0)"] +hooks = ["pre-commit (>=3.3.3)"] +package = ["build (>=1.0.0)", "twine (>=4.0.2)"] +runner = ["tox (>=4.11.1)"] +sast = ["bandit[toml] (>=1.7.5)"] +testing = ["pytest (>=7.4.0)"] +tooling = ["black (>=23.7.0)", "pyright (>=1.1.325)", "ruff (>=0.0.287)"] +tooling-extras = ["pyaml (>=23.7.0)", "pypandoc-binary (>=1.11)", "pytest (>=7.4.0)"] + [[package]] name = "virtualenv" version = "20.26.0" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "virtualenv-20.26.0-py3-none-any.whl", hash = "sha256:0846377ea76e818daaa3e00a4365c018bc3ac9760cbb3544de542885aad61fb3"}, {file = "virtualenv-20.26.0.tar.gz", hash = "sha256:ec25a9671a5102c8d2657f62792a27b48f016664c6873f6beed3800008577210"}, @@ -3641,7 +4292,7 @@ platformdirs = ">=3.9.1,<5" [package.extras] docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2,!=7.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] -test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8)", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10)"] +test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8) ; platform_python_implementation == \"PyPy\"", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10) ; platform_python_implementation == \"CPython\""] [[package]] name = "wcwidth" @@ -3649,6 +4300,7 @@ version = "0.2.13" description = "Measures the displayed width of unicode strings in a terminal" optional = false python-versions = "*" +groups = ["lab"] files = [ {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"}, {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"}, @@ -3660,6 +4312,7 @@ version = "1.13" description = "A library for working with the color formats defined by HTML and CSS." optional = false python-versions = ">=3.7" +groups = ["lab"] files = [ {file = "webcolors-1.13-py3-none-any.whl", hash = "sha256:29bc7e8752c0a1bd4a1f03c14d6e6a72e93d82193738fa860cbff59d0fcc11bf"}, {file = "webcolors-1.13.tar.gz", hash = "sha256:c225b674c83fa923be93d235330ce0300373d02885cef23238813b0d5668304a"}, @@ -3675,6 +4328,7 @@ version = "0.5.1" description = "Character encoding aliases for legacy web content" optional = false python-versions = "*" +groups = ["lab"] files = [ {file = "webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78"}, {file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"}, @@ -3686,6 +4340,7 @@ version = "1.8.7" description = "WSGI request and response object" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*" +groups = ["main"] files = [ {file = "WebOb-1.8.7-py2.py3-none-any.whl", hash = "sha256:73aae30359291c14fa3b956f8b5ca31960e420c28c1bec002547fb04928cf89b"}, {file = "WebOb-1.8.7.tar.gz", hash = "sha256:b64ef5141be559cfade448f044fa45c2260351edcb6a8ef6b7e00c7dcef0c323"}, @@ -3701,6 +4356,7 @@ version = "1.8.0" description = "WebSocket client for Python with low level API options" optional = false python-versions = ">=3.8" +groups = ["lab"] files = [ {file = "websocket_client-1.8.0-py3-none-any.whl", hash = "sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526"}, {file = "websocket_client-1.8.0.tar.gz", hash = "sha256:3239df9f44da632f96012472805d40a23281a991027ce11d2f45a6f24ac4c3da"}, @@ -3711,12 +4367,100 @@ docs = ["Sphinx (>=6.0)", "myst-parser (>=2.0.0)", "sphinx-rtd-theme (>=1.1.0)"] optional = ["python-socks", "wsaccel"] test = ["websockets"] +[[package]] +name = "wrapt" +version = "2.1.1" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "wrapt-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7e927375e43fd5a985b27a8992327c22541b6dede1362fc79df337d26e23604f"}, + {file = "wrapt-2.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e1c99544b6a7d40ca22195563b6d8bc3986ee8bb82f272f31f0670fe9440c869"}, + {file = "wrapt-2.1.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b2be3fa5f4efaf16ee7c77d0556abca35f5a18ad4ac06f0ef3904c3399010ce9"}, + {file = "wrapt-2.1.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:67c90c1ae6489a6cb1a82058902caa8006706f7b4e8ff766f943e9d2c8e608d0"}, + {file = "wrapt-2.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:05c0db35ccffd7480143e62df1e829d101c7b86944ae3be7e4869a7efa621f53"}, + {file = "wrapt-2.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:0c2ec9f616755b2e1e0bf4d0961f59bb5c2e7a77407e7e2c38ef4f7d2fdde12c"}, + {file = "wrapt-2.1.1-cp310-cp310-win32.whl", hash = "sha256:203ba6b3f89e410e27dbd30ff7dccaf54dcf30fda0b22aa1b82d560c7f9fe9a1"}, + {file = "wrapt-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:6f9426d9cfc2f8732922fc96198052e55c09bb9db3ddaa4323a18e055807410e"}, + {file = "wrapt-2.1.1-cp310-cp310-win_arm64.whl", hash = "sha256:69c26f51b67076b40714cff81bdd5826c0b10c077fb6b0678393a6a2f952a5fc"}, + {file = "wrapt-2.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6c366434a7fb914c7a5de508ed735ef9c133367114e1a7cb91dfb5cd806a1549"}, + {file = "wrapt-2.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5d6a2068bd2e1e19e5a317c8c0b288267eec4e7347c36bc68a6e378a39f19ee7"}, + {file = "wrapt-2.1.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:891ab4713419217b2aed7dd106c9200f64e6a82226775a0d2ebd6bef2ebd1747"}, + {file = "wrapt-2.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c8ef36a0df38d2dc9d907f6617f89e113c5892e0a35f58f45f75901af0ce7d81"}, + {file = "wrapt-2.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:76e9af3ebd86f19973143d4d592cbf3e970cf3f66ddee30b16278c26ae34b8ab"}, + {file = "wrapt-2.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ff562067485ebdeaef2fa3fe9b1876bc4e7b73762e0a01406ad81e2076edcebf"}, + {file = "wrapt-2.1.1-cp311-cp311-win32.whl", hash = "sha256:9e60a30aa0909435ec4ea2a3c53e8e1b50ac9f640c0e9fe3f21fd248a22f06c5"}, + {file = "wrapt-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:7d79954f51fcf84e5ec4878ab4aea32610d70145c5bbc84b3370eabfb1e096c2"}, + {file = "wrapt-2.1.1-cp311-cp311-win_arm64.whl", hash = "sha256:d3ffc6b0efe79e08fd947605fd598515aebefe45e50432dc3b5cd437df8b1ada"}, + {file = "wrapt-2.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ab8e3793b239db021a18782a5823fcdea63b9fe75d0e340957f5828ef55fcc02"}, + {file = "wrapt-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7c0300007836373d1c2df105b40777986accb738053a92fe09b615a7a4547e9f"}, + {file = "wrapt-2.1.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2b27c070fd1132ab23957bcd4ee3ba707a91e653a9268dc1afbd39b77b2799f7"}, + {file = "wrapt-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b0e36d845e8b6f50949b6b65fc6cd279f47a1944582ed4ec8258cd136d89a64"}, + {file = "wrapt-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4aeea04a9889370fcfb1ef828c4cc583f36a875061505cd6cd9ba24d8b43cc36"}, + {file = "wrapt-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d88b46bb0dce9f74b6817bc1758ff2125e1ca9e1377d62ea35b6896142ab6825"}, + {file = "wrapt-2.1.1-cp312-cp312-win32.whl", hash = "sha256:63decff76ca685b5c557082dfbea865f3f5f6d45766a89bff8dc61d336348833"}, + {file = "wrapt-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:b828235d26c1e35aca4107039802ae4b1411be0fe0367dd5b7e4d90e562fcbcd"}, + {file = "wrapt-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:75128507413a9f1bcbe2db88fd18fbdbf80f264b82fa33a6996cdeaf01c52352"}, + {file = "wrapt-2.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ce9646e17fa7c3e2e7a87e696c7de66512c2b4f789a8db95c613588985a2e139"}, + {file = "wrapt-2.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:428cfc801925454395aa468ba7ddb3ed63dc0d881df7b81626cdd433b4e2b11b"}, + {file = "wrapt-2.1.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5797f65e4d58065a49088c3b32af5410751cd485e83ba89e5a45e2aa8905af98"}, + {file = "wrapt-2.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5a2db44a71202c5ae4bb5f27c6d3afbc5b23053f2e7e78aa29704541b5dad789"}, + {file = "wrapt-2.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8d5350c3590af09c1703dd60ec78a7370c0186e11eaafb9dda025a30eee6492d"}, + {file = "wrapt-2.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d9b076411bed964e752c01b49fd224cc385f3a96f520c797d38412d70d08359"}, + {file = "wrapt-2.1.1-cp313-cp313-win32.whl", hash = "sha256:0bb7207130ce6486727baa85373503bf3334cc28016f6928a0fa7e19d7ecdc06"}, + {file = "wrapt-2.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:cbfee35c711046b15147b0ae7db9b976f01c9520e6636d992cd9e69e5e2b03b1"}, + {file = "wrapt-2.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:7d2756061022aebbf57ba14af9c16e8044e055c22d38de7bf40d92b565ecd2b0"}, + {file = "wrapt-2.1.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:4814a3e58bc6971e46baa910ecee69699110a2bf06c201e24277c65115a20c20"}, + {file = "wrapt-2.1.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:106c5123232ab9b9f4903692e1fa0bdc231510098f04c13c3081f8ad71c3d612"}, + {file = "wrapt-2.1.1-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1a40b83ff2535e6e56f190aff123821eea89a24c589f7af33413b9c19eb2c738"}, + {file = "wrapt-2.1.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:789cea26e740d71cf1882e3a42bb29052bc4ada15770c90072cb47bf73fb3dbf"}, + {file = "wrapt-2.1.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ba49c14222d5e5c0ee394495a8655e991dc06cbca5398153aefa5ac08cd6ccd7"}, + {file = "wrapt-2.1.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ac8cda531fe55be838a17c62c806824472bb962b3afa47ecbd59b27b78496f4e"}, + {file = "wrapt-2.1.1-cp313-cp313t-win32.whl", hash = "sha256:b8af75fe20d381dd5bcc9db2e86a86d7fcfbf615383a7147b85da97c1182225b"}, + {file = "wrapt-2.1.1-cp313-cp313t-win_amd64.whl", hash = "sha256:45c5631c9b6c792b78be2d7352129f776dd72c605be2c3a4e9be346be8376d83"}, + {file = "wrapt-2.1.1-cp313-cp313t-win_arm64.whl", hash = "sha256:da815b9263947ac98d088b6414ac83507809a1d385e4632d9489867228d6d81c"}, + {file = "wrapt-2.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9aa1765054245bb01a37f615503290d4e207e3fd59226e78341afb587e9c1236"}, + {file = "wrapt-2.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:feff14b63a6d86c1eee33a57f77573649f2550935981625be7ff3cb7342efe05"}, + {file = "wrapt-2.1.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:81fc5f22d5fcfdbabde96bb3f5379b9f4476d05c6d524d7259dc5dfb501d3281"}, + {file = "wrapt-2.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:951b228ecf66def855d22e006ab9a1fc12535111ae7db2ec576c728f8ddb39e8"}, + {file = "wrapt-2.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0ddf582a95641b9a8c8bd643e83f34ecbbfe1b68bc3850093605e469ab680ae3"}, + {file = "wrapt-2.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:fc5c500966bf48913f795f1984704e6d452ba2414207b15e1f8c339a059d5b16"}, + {file = "wrapt-2.1.1-cp314-cp314-win32.whl", hash = "sha256:4aa4baadb1f94b71151b8e44a0c044f6af37396c3b8bcd474b78b49e2130a23b"}, + {file = "wrapt-2.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:860e9d3fd81816a9f4e40812f28be4439ab01f260603c749d14be3c0a1170d19"}, + {file = "wrapt-2.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:3c59e103017a2c1ea0ddf589cbefd63f91081d7ce9d491d69ff2512bb1157e23"}, + {file = "wrapt-2.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:9fa7c7e1bee9278fc4f5dd8275bc8d25493281a8ec6c61959e37cc46acf02007"}, + {file = "wrapt-2.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:39c35e12e8215628984248bd9c8897ce0a474be2a773db207eb93414219d8469"}, + {file = "wrapt-2.1.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:94ded4540cac9125eaa8ddf5f651a7ec0da6f5b9f248fe0347b597098f8ec14c"}, + {file = "wrapt-2.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:da0af328373f97ed9bdfea24549ac1b944096a5a71b30e41c9b8b53ab3eec04a"}, + {file = "wrapt-2.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:4ad839b55f0bf235f8e337ce060572d7a06592592f600f3a3029168e838469d3"}, + {file = "wrapt-2.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0d89c49356e5e2a50fa86b40e0510082abcd0530f926cbd71cf25bee6b9d82d7"}, + {file = "wrapt-2.1.1-cp314-cp314t-win32.whl", hash = "sha256:f4c7dd22cf7f36aafe772f3d88656559205c3af1b7900adfccb70edeb0d2abc4"}, + {file = "wrapt-2.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:f76bc12c583ab01e73ba0ea585465a41e48d968f6d1311b4daec4f8654e356e3"}, + {file = "wrapt-2.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:7ea74fc0bec172f1ae5f3505b6655c541786a5cabe4bbc0d9723a56ac32eb9b9"}, + {file = "wrapt-2.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9e03b3d486eb39f5d3f562839f59094dcee30c4039359ea15768dc2214d9e07c"}, + {file = "wrapt-2.1.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0fdf3073f488ce4d929929b7799e3b8c52b220c9eb3f4a5a51e2dc0e8ff07881"}, + {file = "wrapt-2.1.1-cp39-cp39-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0cb4f59238c6625fae2eeb72278da31c9cfba0ff4d9cbe37446b73caa0e9bcf7"}, + {file = "wrapt-2.1.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7f794a1c148871b714cb566f5466ec8288e0148a1c417550983864b3981737cd"}, + {file = "wrapt-2.1.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:95ef3866631c6da9ce1fc0f1e17b90c4c0aa6d041fc70a11bc90733aee122e1a"}, + {file = "wrapt-2.1.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:66bc1b2446f01cbbd3c56b79a3a8435bcd4178ac4e06b091913f7751a7f528b8"}, + {file = "wrapt-2.1.1-cp39-cp39-win32.whl", hash = "sha256:1b9e08e57cabc32972f7c956d10e85093c5da9019faa24faf411e7dd258e528c"}, + {file = "wrapt-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:e75ad48c3cca739f580b5e14c052993eb644c7fa5b4c90aa51193280b30875ae"}, + {file = "wrapt-2.1.1-cp39-cp39-win_arm64.whl", hash = "sha256:9ccd657873b7f964711447d004563a2bc08d1476d7a1afcad310f3713e6f50f4"}, + {file = "wrapt-2.1.1-py3-none-any.whl", hash = "sha256:3b0f4629eb954394a3d7c7a1c8cca25f0b07cefe6aa8545e862e9778152de5b7"}, + {file = "wrapt-2.1.1.tar.gz", hash = "sha256:5fdcb09bf6db023d88f312bd0767594b414655d58090fc1c46b3414415f67fac"}, +] + +[package.extras] +dev = ["pytest", "setuptools"] + [[package]] name = "xarray" version = "2024.9.0" description = "N-D labeled arrays and datasets in Python" optional = false python-versions = ">=3.10" +groups = ["main"] files = [ {file = "xarray-2024.9.0-py3-none-any.whl", hash = "sha256:4fd534abdf12d5fa75dd566c56483d5081f77864462cf3d6ad53e13f9db48222"}, {file = "xarray-2024.9.0.tar.gz", hash = "sha256:e796a6b3eaec11da24f33e4bb14af41897011660a0516fa4037d3ae4bbd1d378"}, @@ -3731,7 +4475,7 @@ pandas = ">=2.1" accel = ["bottleneck", "flox", "numbagg", "opt-einsum", "scipy"] complete = ["xarray[accel,dev,io,parallel,viz]"] dev = ["hypothesis", "mypy", "pre-commit", "pytest", "pytest-cov", "pytest-env", "pytest-timeout", "pytest-xdist", "ruff", "xarray[complete]"] -io = ["cftime", "fsspec", "h5netcdf", "netCDF4", "pooch", "pydap", "scipy", "zarr"] +io = ["cftime", "fsspec", "h5netcdf", "netCDF4", "pooch", "pydap ; python_version < \"3.10\"", "scipy", "zarr"] parallel = ["dask[complete]"] viz = ["matplotlib", "nc-time-axis", "seaborn"] @@ -3741,6 +4485,7 @@ version = "2.0.1" description = "A collection of various tools for data analysis built on top of xarray and xgcm" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "xarrayutils-2.0.1-py3-none-any.whl", hash = "sha256:dc8dd6f603d0184cb2bd210c7b8e131297467521e30915fc1d0e215d8bc8bcee"}, {file = "xarrayutils-2.0.1.tar.gz", hash = "sha256:e157d6cf6a7e166e5a50a15d64cb2022af46bc30211a11e3f5a63f6985b7ceb9"}, @@ -3763,6 +4508,7 @@ version = "0.8.5" description = "Universal Regridder for Geospatial Data" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "xesmf-0.8.5-py3-none-any.whl", hash = "sha256:f142c05974e815d58a26ff54faca21873ba157109e8eace34d2188a554357691"}, {file = "xesmf-0.8.5.tar.gz", hash = "sha256:6767752f75d1900f9d28ce7e70d979964bed0ad7f1a16e1b34fac049cc26f88b"}, @@ -3782,6 +4528,7 @@ version = "0.6.1" description = "General Circulation Model Postprocessing with xarray" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "xgcm-0.6.1-py3-none-any.whl", hash = "sha256:b58772c1f7a529881dbc609aa471382b0d2f4a3dff6f502e163f56d602b7aa42"}, {file = "xgcm-0.6.1.tar.gz", hash = "sha256:6dfa1c57a6c660d9457ca0f563d4ff613c00bfaee5399917cc22a62ebc179b6c"}, @@ -3804,6 +4551,7 @@ version = "0.7.2" description = "Analysis ready CMIP6 data the easy way" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "xmip-0.7.2-py3-none-any.whl", hash = "sha256:1a500f6b5041da2e06ce643b99a0191e417e9f0dec33074e5ad2c34f444b9286"}, {file = "xmip-0.7.2.tar.gz", hash = "sha256:a145b084e48ce1a40c1727c95b4e251b0a1904266aecf0138f383ddacbbfb1c6"}, @@ -3826,6 +4574,7 @@ version = "3.19.0" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "zipp-3.19.0-py3-none-any.whl", hash = "sha256:96dc6ad62f1441bcaccef23b274ec471518daf4fbbc580341204936a5a3dddec"}, {file = "zipp-3.19.0.tar.gz", hash = "sha256:952df858fb3164426c976d9338d3961e8e8b3758e2e059e0f754b8c4262625ee"}, @@ -3836,6 +4585,6 @@ docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.link testing = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] [metadata] -lock-version = "2.0" +lock-version = "2.1" python-versions = ">=3.10,<3.12" -content-hash = "f022cd016ae910e8f019261294ca552fd34ebbc217b804d9be4aa5fe24d1446d" +content-hash = "a51a6b576fa2757d24446c0ec36df1ae29ec5364cac87971c00946a5e639beec" diff --git a/pyproject.toml b/pyproject.toml index 8503b54..b75a73e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ esgf-pyclient = "^0.3.1" myproxyclient = "^2.1.1" h5netcdf = "^1.3.0" typer = "^0.12.5" +esgpull = "^0.9.6" [tool.poetry.group.secondary.dependencies] jinja2 = ">=3.1.4" @@ -39,7 +40,10 @@ pre-commit = "^3.7.0" flake8-pyproject = "^1.2.3" black = "^24.4.2" nox = "^2024.4.15" -docformatter = {extras = ["tomli"], version = "^1.7.5"} +docformatter = { extras = ["tomli"], version = "^1.7.5" } +autoflake = "^2.3.1" +autopep8 = "^2.3.2" +ruff = "^0.11.11" [tool.poetry.group.lab.dependencies] jupyterlab = "^4.0.10" @@ -98,21 +102,25 @@ replace = ''' ''' [tool.pylint] -disable = "C0114,C0115,C0116,R0903,R1710,W1203,W0511,W0718,C0302" +disable = [ + "C0114", + "C0115", + "C0116", + "R0903", + "R1710", + "W1203", + "W0511", # TO DO warnings +] max-line-length = 120 -max-locals = 20 +max-locals = 30 max-args = 16 max-attributes = 20 -ignore = [ - ".git", - "migrations", - "__pycache__" -] [tool.flake8] max-line-length = 120 ignore = ["E203", "E266", "E501", "W503"] -max-complexity = 20 +docstring-convention = "numpy" +max-complexity = 18 per-file-ignores = [] exclude = [ ".git", @@ -120,6 +128,20 @@ exclude = [ "__pycache__" ] +[tool.autoflake] +remove-all-unused-imports = true +in-place = true +ignore-init-module-imports = true +remove-unused-variables = true +recursive = true + +[tool.autopep8] +max_line_length = 120 +in-place = true +aggressive = 2 +ignore = ["W503", "E203", "E501"] +recursive = true + [tool.black] line-length = 120 target-version = ["py311"] @@ -132,17 +154,72 @@ line-length = 120 transform-concats = true verbose = true -[tool.pytest.ini_options] -testpaths = [ - "tests", -] -markers = ["offline: mark a test as needing to be run offline.", ] - [tool.docformatter] -style = "google" +style = "numpy" pre-summary-newline = true wrap-descriptions = 120 wrap-summaries = 120 blank = false exclude = [] recursive = true + +[tool.ruff] +line-length = 120 + +target-version = "py311" + +exclude = [ + ".git", + "migrations", + "__pycache__", +] + +[tool.ruff.lint] +select = [ + "A", # Flake8 Built ins + "E", # Error (Flake8) + "F", # Pyflakes (Flake8) + "W", # Warning (Flake8) + "I", # isort (import sorting) + "N", # Naming conventions (Pylint, etc.) + "C90", # mccabe complexity (replaces flake8 --max-complexity and mccabe) + "B", # Bugbear (common linting issues) + "UP", # pyupgrade (suggests modern Python syntax) + "PLR", # Pylint refactor + "PLE", # Pylint error (specific Pylint error rules) + "PLW", # Pylint warning (specific Pylint warning rules) + "PLC", # Pylint convention (specific Pylint convention rules) + "R", # Refactor (Pylint refactoring suggestions) + "TID", # TO DO comments + "FAST",# FastAPI + "C4", # List and dict comprehensions + "DJ", # Django + "PIE", # Returns and unnecessary returns + "Q", # Double quotes + "RET", # Fix return statements + "PTH", # Enforce pathlib + "ARG", # Unused argument + "FLY", # Flynt + "NPY", # Numpy specific + "PD", # Pandas specific + "RUF", # Ruff specific +] + +ignore = [ + "E203", # whitespace before ':', Black already handles this + "E266", # too many leading '#' for comments + "E501", # line too long (we enforce via line-length instead) + "RET504", + "RUF013", + "PTH123" +] + +[tool.ruff.lint.pydocstyle] +convention = "numpy" # Corresponds to flake8's docstring-convention and docformatter style. + +[tool.ruff.lint.pylint] +max-args = 16 + +[tool.ruff.lint.mccabe] +# cyclomatic complexity +max-complexity = 18 diff --git a/scripts/download_example.py b/scripts/download_example.py new file mode 100644 index 0000000..418992d --- /dev/null +++ b/scripts/download_example.py @@ -0,0 +1,108 @@ +import typer + +from climateset import CONFIGS +from climateset.download import download_from_config_file, downloader_config +from climateset.download.cmip6_downloader import CMIP6Downloader, CMIP6DownloaderV2 +from climateset.download.input4mips_downloader import ( + Input4MipsDownloader, + Input4MipsDownloaderV2, +) + +app = typer.Typer(no_args_is_help=True) + +CONFIG_PATH = CONFIGS / "minimal_dataset.yaml" + + +@app.command( + name="download-basic", + help="Download ClimateSet data by building the config objects. See function content for more details.", +) +def basic_download(): + """ + By default, will download to the DATA_DIR folder. You can override this behavior modifying the config objects or by + adding the `data_dir` key in the config file under each project. + + ex. + CMIP6: + models: [ "NorESM2-LM" ] + variables: [ "tas" ] + experiments: [ "historical", "ssp126" ] + data_dir: "PATH/TO/DATA_DIR" + input4MIPs: + variables: [ "CO2", "CH4" ] + experiments: [ "historical","ssp126" ] + data_dir: "PATH/TO/DATA_DIR" + """ + input4mips_config = downloader_config.create_input4mips_downloader_config_from_file(CONFIG_PATH) + cmip6_config = downloader_config.create_cmip6_downloader_config_from_file(CONFIG_PATH) + + # If you want to specify where data will be downloaded, change the following: + # input4mips_config.data_dir = "PATH_TO_DATA_DIR" + # cmip6_config.data_dir = "PATH_TO_DATA_DIR" + + input4mips_downloader = Input4MipsDownloader(input4mips_config) + input4mips_downloader.download() + + cmip6_downloader = CMIP6Downloader(cmip6_config) + cmip6_downloader.download() + + +@app.command( + name="download-basic-v2", + help="Download ClimateSet data by building the config objects. See function content for more details.", +) +def basic_download_v2(): + """ + By default, will download to the DATA_DIR folder. You can override this behavior modifying the config objects or by + adding the `data_dir` key in the config file under each project. + + ex. + CMIP6: + models: [ "NorESM2-LM" ] + variables: [ "tas" ] + experiments: [ "historical", "ssp126" ] + data_dir: "PATH/TO/DATA_DIR" + input4MIPs: + variables: [ "CO2", "CH4" ] + experiments: [ "historical","ssp126" ] + data_dir: "PATH/TO/DATA_DIR" + """ + input4mips_config = downloader_config.create_input4mips_downloader_config_from_file(CONFIG_PATH) + cmip6_config = downloader_config.create_cmip6_downloader_config_from_file(CONFIG_PATH) + + # If you want to specify where data will be downloaded, change the following: + # input4mips_config.data_dir = "PATH_TO_DATA_DIR" + # cmip6_config.data_dir = "PATH_TO_DATA_DIR" + + input4mips_downloader = Input4MipsDownloaderV2(input4mips_config) + input4mips_downloader.download() + + cmip6_downloader = CMIP6DownloaderV2(cmip6_config) + cmip6_downloader.download() + + +@app.command( + name="download-from-config", + help="Download ClimateSet data via download_from_config_file() function. See function content for more details.", +) +def alternative_approach(): + """ + By default, will download to the DATA_DIR folder. You can override this behavior by adding the `data_dir` key in the + config file under each project. + + ex. + CMIP6: + models: [ "NorESM2-LM" ] + variables: [ "tas" ] + experiments: [ "historical", "ssp126" ] + data_dir: "PATH/TO/DATA_DIR" + input4MIPs: + variables: [ "CO2", "CH4" ] + experiments: [ "historical","ssp126" ] + data_dir: "PATH/TO/DATA_DIR" + """ + download_from_config_file(CONFIG_PATH) + + +if __name__ == "__main__": + app() diff --git a/tests/resources/test_minimal_dataset.yaml b/tests/resources/test_minimal_dataset.yaml index 86bf9c0..a09f942 100644 --- a/tests/resources/test_minimal_dataset.yaml +++ b/tests/resources/test_minimal_dataset.yaml @@ -1,4 +1,7 @@ -models: ["NorESM2-LM"] -downloader_kwargs: - variables: ["tas", "CO2", "CH4"] - experiments: ["historical", "ssp126"] \ No newline at end of file +CMIP6: + models: ["NorESM2-LM"] + variables: ["tas"] + experiments: ["ssp126"] +input4MIPs: + variables: ["CO2", "CH4"] + experiments: ["historical","ssp126"] \ No newline at end of file diff --git a/tests/test_download/test_constraints.py b/tests/test_download/test_constraints.py new file mode 100644 index 0000000..241d8af --- /dev/null +++ b/tests/test_download/test_constraints.py @@ -0,0 +1,73 @@ +from climateset.download.constraints import ( + BaseSearchConstraints, + CMIP6Constraints, + Input4MIPsConstraints, +) + + +def test_base_constraints_to_esgf_params(): + """Test that BaseSearchConstraints correctly converts to ESGF parameters.""" + constraints = BaseSearchConstraints(project="CMIP6", variable="tas", frequency="mon") + + params = constraints.to_esgf_params() + + assert params == {"project": "CMIP6", "variable": "tas", "frequency": "mon"} + + # Verify strict None filtering + assert "grid_label" not in params + assert "version" not in params + + +def test_cmip6_constraints_inheritance(): + """Test that CMIP6Constraints includes fields from Base and its own.""" + constraints = CMIP6Constraints( + project="CMIP6", experiment_id="ssp585", source_id="NorESM2-LM", variant_label="r1i1p1f1" + ) + + params = constraints.to_esgf_params() + + expected = {"project": "CMIP6", "experiment_id": "ssp585", "source_id": "NorESM2-LM", "variant_label": "r1i1p1f1"} + assert params == expected + + +def test_input4mips_constraints_inheritance(): + """Test Input4MIPsConstraints serialization.""" + constraints = Input4MIPsConstraints( + project="input4MIPs", + target_mip="CMIP", + institution_id="UoM", + ) + + params = constraints.to_esgf_params() + + expected = { + "project": "input4MIPs", + "target_mip": "CMIP", + "institution_id": "UoM", + } + assert params == expected + + +def test_constraints_immutability(): + """Test that constraints are immutable (frozen).""" + constraints = BaseSearchConstraints(project="CMIP6") + + try: + constraints.project = "CMIP5" + assert False, "Should have raised AttributeError" + except AttributeError: + pass # Expected behavior + + +def test_base_constraints_multi_value_esgpull(): + """Test that constraints support list values and serialization to esgpull queries.""" + constraints = BaseSearchConstraints(project=["CMIP6", "input4MIPs"], variable=["tas", "pr"], frequency="mon") + + # The serialization should output the list values directly. + esgf_params = constraints.to_esgf_params() + esgpull_params = constraints.to_esgpull_query() + + expected = {"project": ["CMIP6", "input4MIPs"], "variable": ["tas", "pr"], "frequency": "mon"} + + assert esgf_params == expected + assert esgpull_params == expected diff --git a/tests/test_download/test_downloader.py b/tests/test_download/test_downloader.py index 83a25ae..6f544c3 100644 --- a/tests/test_download/test_downloader.py +++ b/tests/test_download/test_downloader.py @@ -1,20 +1,39 @@ -from unittest.mock import call, patch +import shutil +from unittest.mock import ANY, call, patch import pytest from climateset import TEST_DIR -from climateset.download.downloader import Downloader, download_from_config_file -from climateset.utils import get_yaml_config +from climateset.download.cmip6_downloader import CMIP6Downloader +from climateset.download.constants.esgf import CMIP6, INPUT4MIPS +from climateset.download.downloader import download_from_config_file +from climateset.download.downloader_config import ( + create_cmip6_downloader_config_from_file, + create_input4mips_downloader_config_from_file, +) +from climateset.download.input4mips_downloader import Input4MipsDownloader MINIMAL_DATASET_CONFIG_PATH = TEST_DIR / "resources/test_minimal_dataset.yaml" +TEST_TMP_DIR = TEST_DIR / "resources/.tmp" +MAX_ENSEMBLE_MEMBERS = 10 -DOWNLOAD_RAW_INPUT_SINGLE_VAR = "climateset.download.downloader.Downloader.download_raw_input_single_var" +DOWNLOAD_RAW_INPUT_SINGLE_VAR = ( + "climateset.download.input4mips_downloader.Input4MipsDownloader.download_raw_input_single_var" +) DOWNLOAD_META_HISTORIC_SINGLE_VAR = ( - "climateset.download.downloader.Downloader.download_meta_historic_biomassburning_single_var" + "climateset.download.input4mips_downloader.Input4MipsDownloader.download_meta_historic_biomassburning_single_var" ) -DOWNLOAD_MODEL_SINGLE_VAR = "climateset.download.downloader.Downloader.download_from_model_single_var" +DOWNLOAD_MODEL_SINGLE_VAR = "climateset.download.cmip6_downloader.CMIP6Downloader.download_from_model_single_var" SUBPROCESS_RUN = "subprocess.run" +DOWNLOAD_RAW_INPUT_SINGLE_VAR_V2 = ( + "climateset.download.input4mips_downloader.Input4MipsDownloaderV2.download_raw_input_single_var" +) +DOWNLOAD_META_HISTORIC_SINGLE_VAR_V2 = ( + "climateset.download.input4mips_downloader.Input4MipsDownloaderV2.download_meta_historic_biomassburning_single_var" +) +DOWNLOAD_MODEL_SINGLE_VAR_V2 = "climateset.download.cmip6_downloader.CMIP6DownloaderV2.download_from_model_single_var" + EXPECTED_MINIMAL_RAW_INPUT_CALLS = [ call(variable="CO2_em_anthro", institution_id="PNNL-JGCRI"), call(variable="CO2_em_AIR_anthro", institution_id="PNNL-JGCRI"), @@ -25,6 +44,7 @@ call(variable="CH4", institution_id="VUA"), call(variable="CH4_openburning_share", institution_id="IAMC"), ] +RAW_INPUT_NUM_OF_CALLS = 8 EXPECTED_MINIMAL_META_HISTORIC_CALLS = [ call(variable="CH4_percentage_AGRI", institution_id="VUA"), @@ -34,13 +54,42 @@ call(variable="CH4_percentage_SAVA", institution_id="VUA"), call(variable="CH4_percentage_TEMF", institution_id="VUA"), ] +META_HISTORIC_NUM_OF_CALLS = 6 EXPECTED_MINIMAL_MODEL_CALLS = [ - call(variable="tas", experiment="historical"), - call(variable="tas", experiment="ssp126"), + call(model="NorESM2-LM", project="CMIP6", variable="tas", experiment="ssp126"), +] +MODEL_SINGLE_NUM_OF_CALLS = 1 + +EXPECTED_MINIMAL_RAW_INPUT_CALLS_V2 = [ + call(esg=ANY, variable="CO2_em_anthro", institution_id="PNNL-JGCRI"), + call(esg=ANY, variable="CO2_em_AIR_anthro", institution_id="PNNL-JGCRI"), + call(esg=ANY, variable="CH4_em_openburning", institution_id="IAMC"), + call(esg=ANY, variable="CH4_em_anthro", institution_id="PNNL-JGCRI"), + call(esg=ANY, variable="CH4_em_AIR_anthro", institution_id="PNNL-JGCRI"), + call(esg=ANY, variable="CO2", institution_id="VUA"), + call(esg=ANY, variable="CH4", institution_id="VUA"), + call(esg=ANY, variable="CH4_openburning_share", institution_id="IAMC"), +] + +EXPECTED_MINIMAL_META_HISTORIC_CALLS_V2 = [ + call(esg=ANY, variable="CH4_percentage_AGRI", institution_id="VUA"), + call(esg=ANY, variable="CH4_percentage_BORF", institution_id="VUA"), + call(esg=ANY, variable="CH4_percentage_DEFO", institution_id="VUA"), + call(esg=ANY, variable="CH4_percentage_PEAT", institution_id="VUA"), + call(esg=ANY, variable="CH4_percentage_SAVA", institution_id="VUA"), + call(esg=ANY, variable="CH4_percentage_TEMF", institution_id="VUA"), +] + +EXPECTED_MINIMAL_MODEL_CALLS_V2 = [ + call(esg=ANY, model="NorESM2-LM", project="CMIP6", variable="tas", experiment="ssp126"), ] +def delete_tmp_dir(): + shutil.rmtree(TEST_TMP_DIR, ignore_errors=True) + + @pytest.fixture() def mock_raw_input_single_var(): with patch(DOWNLOAD_RAW_INPUT_SINGLE_VAR) as mock_function: @@ -66,38 +115,51 @@ def mock_subprocess_run(): @pytest.fixture -def simple_downloader_object(): +def input4mips_downloader_object(): + config_source = MINIMAL_DATASET_CONFIG_PATH + config = create_input4mips_downloader_config_from_file(config_source) + config.data_dir = TEST_TMP_DIR + yield Input4MipsDownloader(config=config) + delete_tmp_dir() + + +@pytest.fixture +def cmip6_downloader_object(): config_source = MINIMAL_DATASET_CONFIG_PATH - config = get_yaml_config(config_source) - model = config["models"][0] - downloader_kwargs = config["downloader_kwargs"] - return Downloader(model=model, **downloader_kwargs) + config = create_cmip6_downloader_config_from_file(config_source) + config.data_dir = TEST_TMP_DIR + yield CMIP6Downloader(config=config) + delete_tmp_dir() -def test_downloader_init(simple_downloader_object): - assert isinstance(simple_downloader_object, Downloader) +def test_downloader_init(input4mips_downloader_object, cmip6_downloader_object): + assert isinstance(input4mips_downloader_object, Input4MipsDownloader) + assert isinstance(cmip6_downloader_object, CMIP6Downloader) -def test_downloader_base_params(simple_downloader_object): - assert simple_downloader_object.model == "NorESM2-LM" - assert simple_downloader_object.experiments == ["historical", "ssp126"] +def test_downloader_base_params(input4mips_downloader_object, cmip6_downloader_object): + assert input4mips_downloader_object.config.project == INPUT4MIPS + assert input4mips_downloader_object.config.experiments == ["historical", "ssp126"] + assert cmip6_downloader_object.config.project == CMIP6 + assert cmip6_downloader_object.config.models == ["NorESM2-LM"] + assert cmip6_downloader_object.config.experiments == ["ssp126"] -def test_downloader_max_possible_member_number(simple_downloader_object): - assert simple_downloader_object.max_ensemble_members == 1 +def test_downloader_max_possible_member_number(cmip6_downloader_object): + assert cmip6_downloader_object.config.max_ensemble_members == MAX_ENSEMBLE_MEMBERS -def test_downloader_variables(simple_downloader_object): - assert simple_downloader_object.raw_vars == [ +def test_downloader_variables(input4mips_downloader_object, cmip6_downloader_object): + assert cmip6_downloader_object.config.variables == ["tas"] + assert input4mips_downloader_object.config.variables == [ "CO2_em_anthro", "CO2_em_AIR_anthro", "CH4_em_openburning", "CH4_em_anthro", "CH4_em_AIR_anthro", ] - assert simple_downloader_object.biomass_vars == ["CO2", "CH4"] - assert simple_downloader_object.model_vars == ["tas"] - assert simple_downloader_object.meta_vars_percentage == [ + assert input4mips_downloader_object.config.biomass_vars == ["CO2", "CH4"] + assert input4mips_downloader_object.config.meta_vars_percentage == [ "CH4_percentage_AGRI", "CH4_percentage_BORF", "CH4_percentage_DEFO", @@ -105,39 +167,42 @@ def test_downloader_variables(simple_downloader_object): "CH4_percentage_SAVA", "CH4_percentage_TEMF", ] - assert simple_downloader_object.meta_vars_share == ["CH4_openburning_share"] + assert input4mips_downloader_object.config.meta_vars_share == ["CH4_openburning_share"] -def test_downloader_model_params(simple_downloader_object): - assert simple_downloader_object.model_node_link == "https://esgf-data.dkrz.de/esg-search" - assert simple_downloader_object.model_source_center == "NCC" +def test_downloader_model_params(cmip6_downloader_object): + # TODO refactor this test for new Node list + assert cmip6_downloader_object.config.node_link in "https://esgf-node.llnl.gov/esg-search/" -def test_download_raw_input(simple_downloader_object, mock_raw_input_single_var, mock_meta_historic_single_var): - simple_downloader_object.download_raw_input() +def test_download_raw_input(input4mips_downloader_object, mock_raw_input_single_var, mock_meta_historic_single_var): + input4mips_downloader_object.download() assert mock_raw_input_single_var.call_args_list == EXPECTED_MINIMAL_RAW_INPUT_CALLS - assert mock_raw_input_single_var.call_count == 8 + assert mock_raw_input_single_var.call_count == RAW_INPUT_NUM_OF_CALLS assert mock_meta_historic_single_var.call_args_list == EXPECTED_MINIMAL_META_HISTORIC_CALLS - assert mock_meta_historic_single_var.call_count == 6 + assert mock_meta_historic_single_var.call_count == META_HISTORIC_NUM_OF_CALLS -def test_download_from_model(simple_downloader_object, mock_model_single_var): - simple_downloader_object.download_from_model() +def test_download_from_model(cmip6_downloader_object, mock_model_single_var): + cmip6_downloader_object.download() assert mock_model_single_var.call_args_list == EXPECTED_MINIMAL_MODEL_CALLS - assert mock_model_single_var.call_count == 2 + assert mock_model_single_var.call_count == 1 +@patch(DOWNLOAD_MODEL_SINGLE_VAR_V2) +@patch(DOWNLOAD_META_HISTORIC_SINGLE_VAR_V2) +@patch(DOWNLOAD_RAW_INPUT_SINGLE_VAR_V2) def test_download_from_config_file( - simple_downloader_object, mock_raw_input_single_var, mock_meta_historic_single_var, mock_model_single_var + mock_raw_input_single_var_v2, mock_meta_historic_single_var_v2, mock_model_single_var_v2 ): - download_from_config_file(config=MINIMAL_DATASET_CONFIG_PATH) + download_from_config_file(config_file=MINIMAL_DATASET_CONFIG_PATH) - assert mock_raw_input_single_var.call_args_list == EXPECTED_MINIMAL_RAW_INPUT_CALLS - assert mock_raw_input_single_var.call_count == 8 - assert mock_meta_historic_single_var.call_args_list == EXPECTED_MINIMAL_META_HISTORIC_CALLS - assert mock_meta_historic_single_var.call_count == 6 - assert mock_model_single_var.call_args_list == EXPECTED_MINIMAL_MODEL_CALLS - assert mock_model_single_var.call_count == 2 + assert mock_raw_input_single_var_v2.call_args_list == EXPECTED_MINIMAL_RAW_INPUT_CALLS_V2 + assert mock_raw_input_single_var_v2.call_count == RAW_INPUT_NUM_OF_CALLS + assert mock_meta_historic_single_var_v2.call_args_list == EXPECTED_MINIMAL_META_HISTORIC_CALLS_V2 + assert mock_meta_historic_single_var_v2.call_count == META_HISTORIC_NUM_OF_CALLS + assert mock_model_single_var_v2.call_args_list == EXPECTED_MINIMAL_MODEL_CALLS_V2 + assert mock_model_single_var_v2.call_count == MODEL_SINGLE_NUM_OF_CALLS def _assert_content_is_in_wget_script(mock_call, string_content): @@ -145,34 +210,40 @@ def _assert_content_is_in_wget_script(mock_call, string_content): # With the provided inputs, there should be only 1 call. # We then access the call's arguments. We are interested in # the content of the wget script that is generated, and we - # want to make sure that for there inputs, we get the same files + # want to make sure that for the same inputs, we get the same files call_list = mock_call.call_args_list first_and_only_call = call_list[0] call_arguments = first_and_only_call.args[0] wget_script_content = call_arguments[2] + print(string_content) + print(wget_script_content) assert string_content in wget_script_content -def test_download_raw_input_single_var(simple_downloader_object, mock_subprocess_run): +def test_download_raw_input_single_var(input4mips_downloader_object, mock_subprocess_run): download_subprocess = mock_subprocess_run - simple_downloader_object.download_raw_input_single_var(variable="CO2_em_anthro", institution_id="PNNL-JGCRI") + input4mips_downloader_object.download_raw_input_single_var(variable="CO2_em_anthro", institution_id="PNNL-JGCRI") + # These are partial file strings. Since we download multiple variables at the same time, it's very complicated + # to specify versions for each without becoming cubbersome. Therefore, this test just looks for the file parts + # That don't change once a new version gets published (which made this test crash and required updating this + # variable expected_files = [ - "CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_175001-179912.nc", - "CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_180001-184912.nc", - "CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_185001-185012.nc", - "CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_185101-189912.nc", - "CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_190001-194912.nc", - "CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_195001-199912.nc", - "CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_200001-201412.nc", + "CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-CMIP-", + "_gn_175001-179912.nc", + "_gn_180001-184912.nc", + "_gn_185001-189912.nc", + "_gn_190001-194912.nc", + "_gn_195001-199912.nc", ] download_subprocess.assert_called_once() for f in expected_files: _assert_content_is_in_wget_script(download_subprocess, f) -def test_download_meta_historic_biomassburning_single_var(simple_downloader_object, mock_subprocess_run): - simple_downloader_object.download_meta_historic_biomassburning_single_var( +@pytest.mark.xfail +def test_download_meta_historic_biomassburning_single_var(input4mips_downloader_object, mock_subprocess_run): + input4mips_downloader_object.download_meta_historic_biomassburning_single_var( variable="CH4_percentage_AGRI", institution_id="VUA" ) @@ -184,8 +255,8 @@ def test_download_meta_historic_biomassburning_single_var(simple_downloader_obje _assert_content_is_in_wget_script(mock_call=mock_subprocess_run, string_content=f) -def test_download_from_model_single_var(simple_downloader_object, mock_subprocess_run): - simple_downloader_object.download_from_model_single_var(variable="tas", experiment="ssp126") +def test_download_from_model_single_var(cmip6_downloader_object, mock_subprocess_run): + cmip6_downloader_object.download() expected_files = [ "tas_Amon_NorESM2-LM_ssp126_r1i1p1f1_gn_201501-202012.nc", @@ -198,6 +269,35 @@ def test_download_from_model_single_var(simple_downloader_object, mock_subproces "tas_Amon_NorESM2-LM_ssp126_r1i1p1f1_gn_208101-209012.nc", "tas_Amon_NorESM2-LM_ssp126_r1i1p1f1_gn_209101-210012.nc", ] - mock_subprocess_run.assert_called_once() + mock_subprocess_run.assert_called() for f in expected_files: _assert_content_is_in_wget_script(mock_call=mock_subprocess_run, string_content=f) + + +@patch("climateset.download.input4mips_downloader.isolated_esgpull_context") +def test_input4mips_downloader_v2_context_lifecycle(mock_context, input4mips_downloader_object): + from climateset.download.input4mips_downloader import Input4MipsDownloaderV2 + + mock_context.return_value.__enter__.return_value = "mock_esg" + downloader = Input4MipsDownloaderV2(config=input4mips_downloader_object.config) + with ( + patch.object(downloader, "download_raw_input_single_var"), + patch.object(downloader, "download_meta_historic_biomassburning_single_var"), + ): + downloader.download() + + # Assert context manager was called exactly once + assert mock_context.call_count == 1 + + +@patch("climateset.download.cmip6_downloader.isolated_esgpull_context") +def test_cmip6_downloader_v2_context_lifecycle(mock_context, cmip6_downloader_object): + from climateset.download.cmip6_downloader import CMIP6DownloaderV2 + + mock_context.return_value.__enter__.return_value = "mock_esg" + downloader = CMIP6DownloaderV2(config=cmip6_downloader_object.config) + with patch.object(downloader, "download_from_model_single_var"): + downloader.download() + + # Assert context manager was called exactly once + assert mock_context.call_count == 1 diff --git a/tests/test_download/test_esgpull_context_reuse.py b/tests/test_download/test_esgpull_context_reuse.py new file mode 100644 index 0000000..83fa240 --- /dev/null +++ b/tests/test_download/test_esgpull_context_reuse.py @@ -0,0 +1,111 @@ +import shutil +from unittest.mock import AsyncMock, patch + +import pytest +import sqlalchemy as sa +from esgpull.models import File + +from climateset.download.esgpull_utils import ( + esgpull_search_and_download_esgf_model_single_var, + isolated_esgpull_context, +) + + +@pytest.fixture +def clean_tmp_path(tmp_path): + yield tmp_path + if tmp_path.exists(): + shutil.rmtree(tmp_path, ignore_errors=True) + + +class TestEsgpullContextReuse: + @pytest.mark.integration + @patch("climateset.download.esgpull_utils.Esgpull.download", new_callable=AsyncMock) + def test_context_reuse_prevents_facet_bleed(self, mock_download, clean_tmp_path): + """ + Verify that reusing a single esg context across sequential calls does not cause search facets (like variable or + experiment) to bleed between queries. + + We mock the actual download to save time/bandwidth, but let the search hit the network. + """ + mock_download.return_value = ([], []) + + with isolated_esgpull_context(clean_tmp_path) as esg: + # First call: Search for 'tas' in 'historical' + _ = esgpull_search_and_download_esgf_model_single_var( + esg=esg, + model="NorESM2-LM", + variable="tas", + experiment="historical", + project="CMIP6", + default_grid_label="gn", + default_frequency="mon", + preferred_version="latest", + max_ensemble_members=1, + ensemble_members=[], + data_dir=clean_tmp_path, + distrib=True, + ) + + # Retrieve files added to the DB in the first call + files_1 = list(esg.db.session.scalars(sa.select(File))) + assert len(files_1) > 0, "First search should return results" + + # Verify constraints were respected + for file in files_1: + assert "tas" in file.file_id + assert "historical" in file.file_id + + # Clear DB to cleanly assert on the second query + for file in files_1: + esg.db.delete(file) + + # Second call: Search for 'pr' in 'ssp126' + _ = esgpull_search_and_download_esgf_model_single_var( + esg=esg, + model="NorESM2-LM", + variable="pr", + experiment="ssp126", + project="CMIP6", + default_grid_label="gn", + default_frequency="mon", + preferred_version="latest", + max_ensemble_members=1, + ensemble_members=[], + data_dir=clean_tmp_path, + distrib=True, + ) + + # Retrieve files added to the DB in the second call + files_2 = list(esg.db.session.scalars(sa.select(File))) + assert len(files_2) > 0, "Second search should return results" + + # Crucial assertion: ensure no bleed from the first query ('tas', 'historical') + for file in files_2: + assert "pr" in file.file_id + assert "ssp126" in file.file_id + assert "tas" not in file.file_id, "Facet bleed detected: 'tas' found in 'pr' query results" + assert ( + "historical" not in file.file_id + ), "Facet bleed detected: 'historical' found in 'ssp126' query results" + + def test_isolated_context_lifecycle(self, clean_tmp_path): + """Verify that isolated_esgpull_context creates exactly one UUID directory and cleans it up afterwards.""" + jobs_dir = clean_tmp_path / ".esgpull_jobs" + + assert not jobs_dir.exists() or len(list(jobs_dir.iterdir())) == 0 + + with isolated_esgpull_context(clean_tmp_path) as esg: + assert jobs_dir.exists(), ".esgpull_jobs directory should be created" + + uuid_dirs = list(jobs_dir.iterdir()) + assert len(uuid_dirs) == 1, "Exactly one UUID directory should be created" + + uuid_dir = uuid_dirs[0] + assert uuid_dir.is_dir(), "The item should be a directory" + # In the latest esgpull the root path might not be exposed as `root` directly, + # but rather data/db paths are inside it. + assert uuid_dir.name in str(esg.config.paths.data), "Esgpull data path should be inside the UUID dir" + + # After the context manager exits, the UUID directory should be removed + assert not uuid_dir.exists(), "UUID directory should be cleaned up on exit" diff --git a/tests/test_download/test_search_client.py b/tests/test_download/test_search_client.py new file mode 100644 index 0000000..09b3fd1 --- /dev/null +++ b/tests/test_download/test_search_client.py @@ -0,0 +1,105 @@ +from unittest.mock import MagicMock, patch + +import pytest + +from climateset.download.client import SearchClient +from climateset.download.constraints import BaseSearchConstraints + + +@pytest.fixture +def mock_search_connection(): + with patch("climateset.download.client.SearchConnection") as mock: + yield mock + + +def test_search_client_context_manager(): + """Test that SearchClient works as a context manager.""" + with SearchClient() as client: + assert isinstance(client, SearchClient) + + +def test_search_session_initial_connection(mock_search_connection): + """Test that a new session establishes a connection to the first node.""" + mock_conn_instance = MagicMock() + mock_search_connection.return_value = mock_conn_instance + + client = SearchClient(node_urls=["http://node1", "http://node2"]) + session = client.new_session() + + mock_search_connection.assert_called_with(url="http://node1", distrib=True) + mock_conn_instance.new_context.assert_called_once() + assert session._connection == mock_conn_instance + + +def test_search_session_failover(mock_search_connection): + """Test that session fails over to the next node if the first one fails.""" + # First call raises error, second returns mock + mock_conn_instance = MagicMock() + mock_search_connection.side_effect = [Exception("Connection failed"), mock_conn_instance] + + client = SearchClient(node_urls=["http://node1", "http://node2"]) + session = client.new_session() + + # Should have tried node1 then node2 + assert mock_search_connection.call_count == 2 + mock_search_connection.assert_any_call(url="http://node1", distrib=True) + mock_search_connection.assert_any_call(url="http://node2", distrib=True) + assert session._connection == mock_conn_instance + + +def test_search_session_constrain_replay(mock_search_connection): + """ + Test that constraints are replayed when failing over to a new node. + + Scenario: + 1. Connect to Node 1 successfully. + 2. Apply constraint A. + 3. Apply constraint B (fails on Node 1). + 4. Session should rotate to Node 2 and replay A and B. + """ + # Setup mocks + node1_conn = MagicMock() + node2_conn = MagicMock() + + node1_ctx = MagicMock() + node2_ctx = MagicMock() + + node1_conn.new_context.return_value = node1_ctx + node2_conn.new_context.return_value = node2_ctx + + # Node 1 context dies on second constraint + node1_ctx.constrain.side_effect = [ + node1_ctx, # First constraint OK + Exception("Node 1 died"), # Second constraint fails + ] + + # Node 2 context succeeds + node2_ctx.constrain.return_value = node2_ctx + + mock_search_connection.side_effect = [node1_conn, node2_conn] + + client = SearchClient(node_urls=["http://node1", "http://node2"]) + session = client.new_session() + + # 1. Connected to Node 1 + constraints_a = BaseSearchConstraints(project="CMIP6") + session.constrain(constraints_a) + + # Verify Node 1 constrained + node1_ctx.constrain.assert_called_with(project="CMIP6") + + # 2. Convert to params and constrain again -> Logic inside constrain() handles exceptions? + # Actually, constrain() calls _context.constrain. + # If that raises, it should catch, rotate, and re-ensure connection (replaying all). + + constraints_b = BaseSearchConstraints(variable="tas") + session.constrain(constraints_b) + + # Only verify we moved to Node 2 + assert session._current_node_index == 1 # 0-indexed, so 1 is second node + + # Verify Node 2 was initialized and constrained with BOTH A and B + node2_conn.new_context.assert_called() + assert node2_ctx.constrain.call_count >= 2 + # call_args_list should verify replay order + # Note: dictionary ordering might vary but we passed simple kwargs diff --git a/tests/test_download/test_utils.py b/tests/test_download/test_utils.py new file mode 100644 index 0000000..7d5ae55 --- /dev/null +++ b/tests/test_download/test_utils.py @@ -0,0 +1,16 @@ +from climateset.download.esgpull_utils import isolated_esgpull_context + + +def test_isolated_esgpull_context(tmp_path): + with isolated_esgpull_context(tmp_path) as esg: + assert esg is not None + # the path should be tmp_path / .esgpull_jobs / + esg_path = esg.path + assert esg_path.parent.name == ".esgpull_jobs" + assert esg_path.parent.parent == tmp_path + + # It should exist during the context + assert esg_path.exists() + + # After the context, it should be deleted + assert not esg_path.exists()